From 430f4fc214b85aed1ac7b4b4916e56ce91803979 Mon Sep 17 00:00:00 2001 From: Jeremy Daer Date: Tue, 18 Aug 2026 03:59:24 -0700 Subject: [PATCH 01/51] Event feed connector: the run loop, catch-up, recovery, and the tier-2 driver MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The other half of the connector, stacked on the foundations PR. Everything here is the running machine and its harness: connector.go New, the options, Events, Close — the consumer surface loop.go the state machine: states, transitions, timers, live buffer catchup.go the poll walk, its page boundary, and the drain recovery.go the 400/409/410 recovery matrix scenario_* the tier-2 conformance driver, its fixture model, harness, and self-tests export_test.go the bridge from the external test package to the internal observation points Rebasing #705's 29 commits onto the foundations PR was attempted and abandoned: 8 of them touch ONLY foundation files and would replay empty, and 21 of the remaining 22 mix both halves, so every one conflicts against the four fixes the foundations PR applies (the credential boundary, the dedupe contract, the file store, the redactor). Resolving 22 interleaved conflicts by hand is a worse guarantee than re-cutting the tree, which is byte-exact by construction: the foundation paths come from that PR's tip and these 17 from #705's. The development history is not lost. It is tagged `pre-split/705-head` at 1646f2c1f, which is what the SHAs cited in #705's review threads still resolve against, and the per-file reasoning it carries is unchanged in the code and its comments. --- CONTRIBUTING.md | 6 + go/README.md | 134 ++ go/pkg/basecamp/eventfeed/buffer_test.go | 36 + go/pkg/basecamp/eventfeed/catchup.go | 994 ++++++++++ go/pkg/basecamp/eventfeed/catchup_test.go | 1612 +++++++++++++++ go/pkg/basecamp/eventfeed/connector.go | 478 +++++ go/pkg/basecamp/eventfeed/connector_test.go | 622 ++++++ go/pkg/basecamp/eventfeed/export_test.go | 131 ++ go/pkg/basecamp/eventfeed/loop.go | 1306 ++++++++++++ go/pkg/basecamp/eventfeed/loop_test.go | 1759 +++++++++++++++++ go/pkg/basecamp/eventfeed/reconnect_test.go | 453 +++++ go/pkg/basecamp/eventfeed/recovery.go | 181 ++ go/pkg/basecamp/eventfeed/recovery_test.go | 760 +++++++ .../eventfeed/scenario_conformance_test.go | 1077 ++++++++++ .../eventfeed/scenario_fixture_test.go | 807 ++++++++ .../eventfeed/scenario_harness_test.go | 626 ++++++ .../eventfeed/scenario_selftest_test.go | 378 ++++ 17 files changed, 11360 insertions(+) create mode 100644 go/pkg/basecamp/eventfeed/buffer_test.go create mode 100644 go/pkg/basecamp/eventfeed/catchup.go create mode 100644 go/pkg/basecamp/eventfeed/catchup_test.go create mode 100644 go/pkg/basecamp/eventfeed/connector.go create mode 100644 go/pkg/basecamp/eventfeed/connector_test.go create mode 100644 go/pkg/basecamp/eventfeed/export_test.go create mode 100644 go/pkg/basecamp/eventfeed/loop.go create mode 100644 go/pkg/basecamp/eventfeed/loop_test.go create mode 100644 go/pkg/basecamp/eventfeed/reconnect_test.go create mode 100644 go/pkg/basecamp/eventfeed/recovery.go create mode 100644 go/pkg/basecamp/eventfeed/recovery_test.go create mode 100644 go/pkg/basecamp/eventfeed/scenario_conformance_test.go create mode 100644 go/pkg/basecamp/eventfeed/scenario_fixture_test.go create mode 100644 go/pkg/basecamp/eventfeed/scenario_harness_test.go create mode 100644 go/pkg/basecamp/eventfeed/scenario_selftest_test.go diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 09f09d7ddc..2fedae8740 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -308,6 +308,12 @@ All SDKs are generated from a single Smithy specification. When adding support f - **Python:** `make py-generate` - **Rust:** `make rs-generate` + `go/pkg/basecamp/eventfeed/` is outside that drift check by design: it is + hand-written §23 infrastructure rather than a wrapper over generated + operations, so nothing about it is derivable from the spec. It is verified + instead by the tier-2 conformance driver in the package, which replays every + fixture under `conformance/event-feed/fixtures/`. + 4. **Add tests** for each SDK 5. **Add conformance tests** (`conformance/tests/`) covering the new operations diff --git a/go/README.md b/go/README.md index bdcc30d552..a246790525 100644 --- a/go/README.md +++ b/go/README.md @@ -779,6 +779,140 @@ webhooks, err := account.Webhooks().List(ctx, bucketID, nil) err = account.Webhooks().Delete(ctx, webhookID) ``` +## Event Feed (experimental) + +The `eventfeed` package is the account-wide event feed connector: an Action Cable +subscription for live push, plus a poll lane that catches up on entry, repairs on a +timer, and resumes after a disconnect. You consume it as one serial, deduplicated +stream of events; the connector owns reconnection, backoff, staleness detection, and +the durable position. + +**Experimental: the Layer-1 seam adapters have not landed yet.** The connector performs +no wire I/O of its own — every HTTP exchange reaches the wire through a seam backed by +a generated operation — and the adapters that build those seams over the generated +`CreateStreamTicket` and `PollEvents` operations are still to come. Until they do, a +consumer must supply the `TicketMinter` and `PollSource` implementations itself, and the +exported surface may still change as they land. + +```go +import ( + "context" + "errors" + "fmt" + "log" + + "github.com/basecamp/basecamp-sdk/go/pkg/basecamp/eventfeed" +) + +// The two seams the host supplies until the Layer-1 adapters land. Each call is +// exactly one generated operation: +// +// MintStreamTicket(ctx) (eventfeed.StreamTicket, error) // CreateStreamTicket +// Poll(ctx, cursor, filters) (eventfeed.PollPage, error) // PollEvents +var minter eventfeed.TicketMinter +var polls eventfeed.PollSource + +ctx := context.Background() + +feed, err := eventfeed.New("https://3.basecampapi.com", "5951425", minter, polls, + eventfeed.WithFilters(eventfeed.Filters{Types: []string{"message.created"}}), + eventfeed.WithCheckpointStore(eventfeed.NewFileCheckpointStore("/var/lib/myapp/feed.json")), + eventfeed.WithConsumerNamespace("myapp"), + eventfeed.WithSignalHandler(func(sig eventfeed.Signal) eventfeed.Disposition { + switch s := sig.(type) { + case eventfeed.FeedGap: + // History before this id is gone. Accept resumes at the server's + // resume URL, having acknowledged what it skips. + log.Printf("feed gap: history before %d is gone", s.EpochAfterID) + return eventfeed.Accept + case eventfeed.BufferOverflow: + log.Printf("live buffer dropped %d events: %v", s.DroppedCount, s.DroppedIDs) + return eventfeed.Terminate + } + return eventfeed.Terminate + }), +) +if err != nil { + return err // *eventfeed.TerminalError: a usage-coded construction error, zero wire attempts +} +defer feed.Close() + +for ev, err := range feed.Events(ctx) { + if err != nil { + var te *eventfeed.TerminalError + if errors.As(err, &te) { + return fmt.Errorf("event feed terminated (%s): %w", te.Reason, te) + } + return err + } + // A feed row is a wake-up signal — enough to route, not enough to act on. + // Refetch the recording through the canonical resource API before acting. + handle(ev.BucketID, ev.RecordingID, ev.EventType) +} +``` + +Construction validates and does no I/O. The base origin must be `https://` — cleartext +`http://` is accepted only for localhost/loopback, the same carve-out the client's base +URL and the connector's cable URL make — because that origin is the trust anchor every +continuation and resume URL is validated against before an authenticated poll follows +it. The checkpoint identity's text inputs (origin, account id, consumer namespace, +filter types) must be valid UTF-8, since the identity encoding is one-to-one only over +valid UTF-8. Either violation is a `ReasonUsage` construction error with zero wire +attempts. + +`Events` is single-shot: consuming it twice yields one `ReasonUsage` error element. +`Close` stops the feed without draining, and cancelling the context, calling `Close`, or +breaking out of the loop all end iteration with **no** error element — a clean stop, and +the feed is resumable by design. + +### Checkpointing + +`FileCheckpointStore` is the built-in `CheckpointStore`: one JSON file holding every +lineage's position, keyed by the four-part checkpoint identity (origin, account, +consumer namespace, filter key). It writes temp-file-plus-rename at 0600, and it is safe +for concurrent use within one process but deliberately not across processes. A store +requires `WithConsumerNamespace` — two independent consumers in one account must not +share a lineage — and changing filters starts a new lineage, because positions are +filter-bound. + +Only poll pages ever advance the durable position; live event ids never do. What the +connector publishes is SPEC.md §23's conjunctive save-ordering invariant, and nothing +stronger: a position is saved only after the retained events it covers have been +delivered **and** every loss condition in that window has been explicitly accepted. +Terminate — or no handler — means no save. A load failure is terminal +(`ReasonCheckpointLoad`, before any wire attempt: silently starting at the present would +skip history), while a save failure is reported through `Observer.CheckpointSaveFailed` +and the feed continues. + +### Semantic signals + +A semantic signal is a condition that changes what the feed can promise, and there are +exactly two: `BufferOverflow` (the live buffer dropped events, naming the exact ids) and +`FeedGap` (a 410 — history before `EpochAfterID` is gone). The handler registered with +`WithSignalHandler` is invoked exactly once per signal, synchronously, on your own +execution context, and returns `Accept` or `Terminate`. + +**With no handler registered, every semantic signal is terminal** — `ReasonBufferOverflow` +or `ReasonFeedGap` — so an unhandled signal cannot disappear, and a 410 never silently +auto-continues. `Accept` on a `FeedGap` resumes via the server's resume URL; `Accept` on a +`BufferOverflow` means you own the acknowledged incompleteness. The `Observer.Gap` and +`Observer.BufferOverflow` callbacks see the same conditions but are observability only: +the disposition lives exclusively in the handler. + +### Terminal versus continuable + +Continuable failures — a dropped socket, a staleness expiry, a throttled or transient +mint or poll, a rejected position (400-position or 409) — never reach the consumer as +errors. They ride the reconnect cycle (full-jitter backoff) or the poll-retry timer, and +you see them through `Observer` callbacks if you register any. + +A terminal condition ends the iteration with exactly one final `*eventfeed.TerminalError` +element carrying a `TerminalReason`: `subscription_rejected`, `protocol_fatal`, +`filter_invalid`, `authorization_failed`, `checkpoint_load`, `usage`, `buffer_overflow`, +`feed_gap`, `invalid_continuation`, `poll_failed`, `mint_failed`, or `invalid_cable_url`. +Switch on `te.Reason` rather than on message text; `errors.Unwrap` reaches the generated +error behind `mint_failed` and `poll_failed`. + ## Pagination List methods auto-paginate: they follow the API's `Link: rel="next"` headers and diff --git a/go/pkg/basecamp/eventfeed/buffer_test.go b/go/pkg/basecamp/eventfeed/buffer_test.go new file mode 100644 index 0000000000..6928536261 --- /dev/null +++ b/go/pkg/basecamp/eventfeed/buffer_test.go @@ -0,0 +1,36 @@ +package eventfeed + +import "testing" + +// TestLiveBufferAddClearsEvictedSlots pins the eviction half of the live +// buffer's memory ceiling. SPEC.md §23 publishes the connector's worst case +// as (pump depth + EVENT_FEED_LIVE_BUFFER_CAPACITY) × EVENT_FEED_MAX_FRAME_BYTES; +// a reslice alone removes the evicted event LOGICALLY while the slice that +// results still points into the same backing array, whose prefix keeps that +// event's strings reachable until a later reallocation. Under sustained +// overflow — the one condition eviction happens under — that is a second +// buffer's worth of payload held by events which no longer count toward +// occupancy. shift already zeroes for exactly this reason. +func TestLiveBufferAddClearsEvictedSlots(t *testing.T) { + b := newLiveBuffer(2, nil) + b.add(Event{ID: 1, Kind: "message", EventType: "message.created"}) + b.add(Event{ID: 2, Kind: "message", EventType: "message.created"}) + // Captured before the eviction: base still spans the backing array the + // live slice is resliced from, so base[0] IS the vacated slot. + base := b.events + + dropped := b.add(Event{ID: 3, Kind: "message", EventType: "message.created"}) + if len(dropped) != 1 || dropped[0] != 1 { + t.Fatalf("dropped = %v, want [1]", dropped) + } + if base[0] != (Event{}) { + t.Errorf("evicted slot = %+v, want the zero Event — a bare reslice keeps its payload reachable", base[0]) + } + // The eviction itself is unchanged. + if got := len(b.events); got != 2 { + t.Fatalf("occupancy = %d, want 2", got) + } + if b.events[0].ID != 2 || b.events[1].ID != 3 { + t.Fatalf("buffer = %d,%d, want 2,3", b.events[0].ID, b.events[1].ID) + } +} diff --git a/go/pkg/basecamp/eventfeed/catchup.go b/go/pkg/basecamp/eventfeed/catchup.go new file mode 100644 index 0000000000..3db2290311 --- /dev/null +++ b/go/pkg/basecamp/eventfeed/catchup.go @@ -0,0 +1,994 @@ +package eventfeed + +import ( + "errors" + "fmt" + "time" +) + +// The catch-up walk, the entry boundary, the drain, and streaming steady +// state (SPEC.md §23 transitions 16 and 20–26, plus the out-of-inventory +// invalid_continuation / poll_failed edges). A confirmed subscription enters +// here at the catchUpHandoff boundary and stays until the cycle ends; the +// whole walk runs on the consumer's goroutine, so a page's deliveries and its +// save are strictly ordered by construction rather than by a protocol. +// +// The dispositions that hand the walk a NEW cursor — a 410's gated resume, a +// 400-position's and a 409's reset cursor (transitions 17/18/19) — and the +// repair poll a fired `repair-poll` timer drives (transition 24) live in +// recovery.go; both re-enter this file's walk. + +// Timer kinds this slice arms (SPEC.md §23 "Clock, Timers, and Virtual +// Time"; the other four arm in the lifecycle slice). +const ( + timerRepairPoll = "repair-poll" + timerPollRetry = "poll-retry" +) + +// runCatchUp is the post-confirmation continuation: the catch-up walk, the +// drain, and then streaming. It owns the attempt from the hand-off onward — +// every exit path disposes it. +func (l *loop) runCatchUp(h catchUpHandoff) cycleOutcome { + l.setState(stateCatchingUp) + if l.hooks.catchUpEntered != nil { + l.hooks.catchUpEntered(h) + } + if l.cfg.observer.CatchUpStarted != nil { + l.cfg.observer.CatchUpStarted(h.entry) + } + // The consecutive-poll-failure index is per walk: a socket teardown ends + // the previous walk, and from there the reconnect cycle's own counter + // governs the wait. + l.pollFailures = 0 + + if out, done := l.walkThenDrain(h.at, h.entry, h.presentClass); done { + return out + } + return l.stream(h.at) +} + +// walkThenDrain runs one walk to its frozen head, drains the live buffer, and +// saves a held present-class entry position: transitions 16/22/23, shared by +// the post-confirmation catch-up and the repair poll (transition 24), which +// differ only in how they enter and what they return to. +func (l *loop) walkThenDrain(at *attempt, cursor Cursor, presentClass bool) (cycleOutcome, bool) { + out, held, done := l.walk(at, cursor, presentClass) + if done { + return out, true + } + + // Transition 22 → Draining: the walk reached its frozen head. Draining is + // a bounded in-memory replay — no polls, no wire waits, no failure edge of + // its own. + l.setState(stateDraining) + if out, done := l.drain(at); done { + return out, true + } + // Transition 23's present-class amendment: the held entry position saves + // only now — after every retained pre-cut event has been accepted. A + // non-empty held position is exactly the walk's report that its FINAL + // entry was present-class, which a mid-walk re-entry can change. The + // invariant's other conjunct (every pre-cut loss condition explicitly + // accepted) is structural: an overflow with no handler, or one whose + // handler returned Terminate, took the Terminal(buffer_overflow) edge in + // the goroutine that observed the drop and never reached this line. + if held != "" { + l.acceptPosition(held) + l.saveCheckpoint(held) + } + if l.cfg.observer.CaughtUp != nil { + l.cfg.observer.CaughtUp() + } + // The walk's last dispatch point, and by construction a SOCKET outcome + // only: nothing else is ever deferred — an overflowing admission is + // dispatched where it is observed, and the drain's scan already took the + // protocol-fatal carve-out. What is left defers here, after the drain and + // the held save have completed — the same "finish what the page started, + // then observe the socket" ordering the page boundary applies, and §23's + // one deliberate deferred-consumption case. + return l.dispatchDeferred(at) +} + +// walk is the catch-up page loop (transition 16): validate any continuation +// URL, poll, deliver the page, then take the page's position — saved per page +// on a position-resume entry, HELD on a present-class one — and follow `next` +// until the walk reaches its frozen head (an absent `next`). +// +// It returns the held present-class position (empty for position-resume +// entries, and empty once a re-entry has superseded a held one), and done=true +// with the cycle's outcome when the walk ended the cycle instead of reaching +// the head. +// +// Two details of the present-class case, both consequences of the invariant +// rather than extra policy: the ownership cut is taken once, right after the +// ENTRY page is accepted, and every later page's position is held too — so a +// walk torn down mid-flight has moved nothing durable and simply re-enters at +// the present, which is exactly what a held position means. +func (l *loop) walk(at *attempt, cursor Cursor, presentClass bool) (out cycleOutcome, held string, done bool) { + cutTaken := false + for { + if l.runCtx.Err() != nil { + l.disposeAttempt(at, nil) + return cycleOutcome{kind: outcomeClosed}, "", true + } + // A frame the previous page's in-flight servicing deferred is + // dispatched here, before anything else this iteration does: the page + // that was in flight has been fully accepted, delivered and saved, so + // this IS transition 21's page boundary. + if out, done := l.dispatchDeferred(at); done { + return out, "", true + } + // Continuation validation precedes the seam call, always: the poll + // carries the caller's bearer, so a cross-origin or downgraded URL + // must never be requested at all. + if cursor.PageURL != "" { + if terr := checkContinuation(l.cfg.origin, cursor.PageURL); terr != nil { + l.disposeAttempt(at, nil) + return cycleOutcome{kind: outcomeTerminal, term: terr}, "", true + } + } + p := l.pollPage(at, cursor) + if p.ended { + // Servicing the socket during the call ended the cycle: an + // overflowing admission whose disposition was not Accept, dispatched + // where it was observed rather than parked for a cut this call may + // never reach. The attempt is already disposed. + return p.out, "", true + } + if l.runCtx.Err() != nil { + l.disposeAttempt(at, nil) + return cycleOutcome{kind: outcomeClosed}, "", true + } + if p.superseded { + // The socket's own outcome, deferred while the seam call was in + // flight, outlived the staleness window with the call still + // outstanding. It IS the disposition — transition 21 — and its + // teardown is what cancels the abandoned call. + if out, done := l.dispatchDeferred(at); done { + return out, "", true + } + // Structurally unreachable: only a socket outcome supersedes a + // call, and every one of them ends the cycle. Fail closed onto + // transition 21 rather than walk on over a spoken-for socket. + l.disposeAttempt(at, nil) + l.observeDisconnected("", errStaleConnection) + return cycleOutcome{kind: outcomeFailed}, "", true + } + if p.err != nil { + // A socket outcome deferred while THIS call was in flight is + // dispatched before the poll error is classified. Transition 21's + // finish-the-page ordering is about a page that SUCCEEDED: it + // exists so an accepted page's deliveries and save are not + // stranded by the socket's death. A failed poll has no such page, + // and recoverPoll's terminal and re-entry branches dispose the + // attempt — which discards the deferral — so leaving it parked + // loses the server's own verdict. Concretely: an + // `invalid_event_stream_command` observed during CatchingUp, + // reported instead as authorization_failed or poll_failed, or + // retried indefinitely behind poll-retry waits when the error is + // transient. + if out, done := l.dispatchDeferred(at); done { + return out, "", true + } + step, out, done := l.recoverPoll(at, cursor, p.err) + if done { + return out, "", true + } + cursor = step.cursor + if step.reentry { + // A re-entry IS an entry (transitions 17/18/19): the previous + // entry's held position is dropped — nothing durable moved — + // its class no longer governs, and a present-class re-entry + // takes its own ownership cut on the new entry page. + presentClass, held, cutTaken = step.presentClass, "", false + } + continue + } + // A successful page resets both consecutive-failure counters: the + // poll-retry index, and the shared connection-level authorization + // counter — the ONLY thing that resets the latter (a confirmation + // proves the ticket, not the bearer). + l.pollFailures = 0 + l.authFailures = 0 + // The page's rows advance the poll-lane reset cursor — served, not + // delivered: dedupe suppression and a consumer break are both + // irrelevant to what the poll lane has served. + page := p.page + for _, ev := range page.Events { + if ev.ID > l.lastPollServedID { + l.lastPollServedID = ev.ID + } + } + + for _, ev := range page.Events { + if !l.deliver(ev) { + l.disposeAttempt(at, nil) + return cycleOutcome{kind: outcomeClosed}, "", true + } + } + if l.cfg.observer.PageDelivered != nil { + l.cfg.observer.PageDelivered(len(page.Events), page.Position) + } + + if presentClass { + // Entry Boundary: the position is held, never saved mid-walk — + // the entry cursor sits at the server's present head, so an + // admitted straggler behind it is not poll-repairable and must be + // delivered before anything durable moves. + held = page.Position + if !cutTaken { + if out, done := l.ownershipCut(at); done { + return out, "", true + } + cutTaken = true + } + } else { + l.acceptPosition(page.Position) + l.saveCheckpoint(page.Position) + } + + if page.Next == "" { + return cycleOutcome{}, held, false + } + // Transition 21 from inside the walk: the page boundary is where a + // socket that died — or went stale — during the previous seam call is + // observed. Following `next` on a dead socket would walk the whole + // frozen head before noticing, delaying the reconnect cycle by the + // length of the walk. + if out, done := l.socketCheck(at); done { + return out, "", true + } + cursor = Cursor{PageURL: page.Next} + } +} + +// deferredFrame is one pump receive taken out of band while a poll seam call +// was in flight and left for the walk's ordinary dispatch point. closed +// records the pump's channel closing, which carries no item. +// +// Every deferral is a SOCKET outcome (or the pump's exit), which is transition +// 21: the page boundary deliberately observes it AFTER the in-flight page has +// been accepted, delivered and saved. Nothing else defers — in particular an +// admission that drops is a pre-cut loss condition whose disposition runs at +// drop time, in the goroutine that received the frame. +type deferredFrame struct { + item pumpItem + closed bool +} + +// pollResult is one poll seam call's outcome, carried back from the call's +// own goroutine. +type pollResult struct { + page PollPage + err error +} + +// pollAttempt is one poll-page attempt as the walk sees it: the page the seam +// call served, or the reason the walk has no page to work with. +type pollAttempt struct { + // page and err are the seam call's own outcome, when it completed. + page PollPage + err error + // ended reports that servicing the socket while the call was in flight + // ENDED THE CYCLE — an admission that dropped and whose disposition was + // Terminate, or which had no handler at all. out carries the outcome, the + // attempt is already disposed, and page/err are meaningless. + ended bool + out cycleOutcome + // superseded reports the call was abandoned: a deferred socket outcome + // outlived the staleness window with the call still outstanding, or the + // consumer went away. The caller dispatches the deferral, whose teardown + // cancels the abandoned call. + superseded bool +} + +// pollPage issues one poll seam call and keeps ADMITTING live events while it +// is in flight. The call runs on its own goroutine — it carries no state, so +// the state machine remains the only mutator — because an event arriving +// while the entry poll is outstanding is exactly the in-flight-at-entry +// straggler the live buffer exists to carry (SPEC.md §23 "Entry Boundary": +// "observed" is admission into the state-machine-owned buffer at or before +// the cut), and a state machine blocked inside the call cannot admit it. +// +// ONLY admissions are serviced here, and an admission that DROPS is serviced +// too: the drop's BufferOverflow signal is dispatched at drop time, on this +// goroutine — the consumer's — because that is §23's first consumer-context +// opportunity after the condition arises. "Before the next save" is the rule's +// outer bound, not an appointment to keep: an in-flight call can stall +// indefinitely or fail outright, so a signal parked until it returns is a +// signal that may never be dispatched at all. An Accept keeps awaiting the +// call; anything else ends the cycle right here (ended=true). +// +// The first receive that is not an admission — a socket failure, a disconnect +// frame, an invalid frame — is DEFERRED, and the call is still awaited to +// completion. That is what keeps transition 21's page boundary the place a +// dying socket is observed: the in-flight page is accepted, delivered and saved +// before the walk stops, and frame order is preserved because the servicing +// stops at the deferred receive. +// +// It reports superseded=true when it gave up on the call: a deferred SOCKET +// outcome is still awaited, but only for as long as the staleness window, +// because a PollSource that returns only on context cancellation would +// otherwise hold the consumer's goroutine forever behind a socket that has +// already spoken (and the staleness expiry that would tear that socket down +// is unobservable while the call is outstanding). The caller dispatches the +// deferred outcome, whose teardown cancels the abandoned call. +func (l *loop) pollPage(at *attempt, cursor Cursor) pollAttempt { + done := make(chan pollResult, 1) + go func() { + // Cloned on the way out for the same reason WithFilters clones on the + // way in: the seam is host code, and handing it the connector's own + // backing arrays would let an adapter that sorts or dedupes in place + // repoint the subscription's lineage from under a live feed. + p, perr := l.cfg.polls.Poll(at.ctx, cursor, l.cfg.filters.clone()) + done <- pollResult{page: p, err: perr} + }() + for { + select { + case r := <-done: + return pollAttempt{page: r.page, err: r.err} + case item, ok := <-at.lc.frames: + if !ok { + l.deferred = &deferredFrame{closed: true} + } else { + handled, out, ended := l.admitDuringPoll(at, item) + if ended { + // The drop's own disposition. Disposing the attempt is what + // returns the abandoned call, so nothing waits on it. + return pollAttempt{ended: true, out: out} + } + if handled { + continue + } + l.deferred = &deferredFrame{item: item} + } + // The superseded-poll bound starts HERE, at the deferral, and is + // read before the hook fires so nothing observing the deferral + // can race the deadline into existence behind it. + deadline := l.cfg.clock.Now().Add(l.cfg.staleAfter) + if l.hooks.frameDeferred != nil { + l.hooks.frameDeferred() + } + return l.awaitSupersededPoll(at, done, deadline) + } + } +} + +// awaitSupersededPoll waits out a seam call whose socket has already spoken. +// The in-flight page is still accepted, delivered and saved when the call +// completes — that ordering is what makes transition 21's page boundary the +// place a dying socket is observed — but the wait is BOUNDED: on the lapse the +// call is abandoned to the deferred outcome's teardown. +// +// The bound is `deadline`: a FIXED instant, read from the injected clock at +// the deferral and moved by nothing afterwards. It is deliberately NOT the staleness +// verdict this wait used to borrow, and the difference is the whole point. +// Every OTHER socket-open wait in the connector keeps DRAINING the hand-off +// queue while it waits, which is what makes §23's suspension rule sound +// there: a full queue really does prove the peer is outrunning a connector +// that is still consuming. This wait is the one that deliberately stops +// consuming — a frame is already parked in the deferral slot, and a second +// receive would have nowhere to go. So the queue it stopped draining fills, +// the pump blocks in handOff, and the suspension gets granted against a +// premise that is false by construction: every firing is disregarded and +// re-armed, forever, while a compliant PollSource that returns only on +// cancellation holds the consumer's goroutine. A peer that keeps sending is +// then enough to keep an already-dead socket's verdict from ever landing. +// +// The staleness firing and the re-arm wake are demoted to WAKE-UPS; the +// deadline alone decides. evaluate is still called on a firing, and still +// re-arms a suspended window — that re-arm is what guarantees the next wake +// when the pump is blocked and no frame can deliver one — so the lapse is +// observed at the first wake at or after the deadline. +// +// A dedicated timer would express this more directly, and is deliberately not +// used: SPEC.md §23 pins exactly six timer kinds AND every state's exact +// timer set, both asserted by the cross-SDK fixtures, so a seventh kind is a +// spec change across six SDKs rather than a fix to the Go reference. +func (l *loop) awaitSupersededPoll(at *attempt, done <-chan pollResult, deadline time.Time) pollAttempt { + for { + staleTimer, staleGen := at.lc.stale.current() + select { + case r := <-done: + return pollAttempt{page: r.page, err: r.err} + case <-l.runCtx.Done(): + return pollAttempt{superseded: true} + case <-at.lc.stale.rearmed(): + case <-staleTimer.C(): + if _, ok := at.lc.stale.evaluate(staleGen); ok { + return pollAttempt{superseded: true} + } + } + if !l.cfg.clock.Now().Before(deadline) { + return pollAttempt{superseded: true} + } + } +} + +// admitDuringPoll handles one pump item received out of band — while a poll +// seam call is in flight, or during the drain's scan — reporting whether it was +// fully handled and, when handling it ended the cycle, that outcome. A +// correlated event frame is admitted to the live buffer; a ping, welcome, +// unknown type, or a frame carrying a foreign identifier is liveness only and +// skipped, exactly as the ordinary dispatch would (the pump already reset +// staleness). Everything else is left for the caller. +// +// An admission that DROPS is handled here rather than handed back, because +// this goroutine is the consumer's and drop time is §23's first +// consumer-context opportunity: admitLive raises the signal and takes its +// disposition, which is Accept (handled, the walk continues) or the +// Terminal(buffer_overflow) that ends the cycle with nothing durable moved. +func (l *loop) admitDuringPoll(at *attempt, item pumpItem) (handled bool, out cycleOutcome, ended bool) { + if item.err != nil { + return false, cycleOutcome{}, false + } + f, err := parseFrame(item.data) + if err != nil { + return false, cycleOutcome{}, false + } + var ev Event + admit := false + switch f.kind { + case frameMessage: + if f.identifier != l.identifier { + break + } + e, derr := decodeMessageEvent(f.message) + if derr != nil { + return false, cycleOutcome{}, false + } + ev, admit = e, true + case frameDisconnect: + return false, cycleOutcome{}, false + case frameWelcome, framePing, frameConfirm, frameReject, frameUnknown: + // Liveness only in a post-confirmation state. + } + if l.hooks.frameHandled != nil { + l.hooks.frameHandled(f.kind) + } + if admit { + if out, done := l.admitLive(at, nil, ev); done { + return true, out, true + } + } + return true, cycleOutcome{}, false +} + +// dispatchDeferred dispatches the frame the in-flight-poll servicing deferred, +// if any, through the ordinary post-confirmation dispatch. +// +// Its two call sites are the walk's dispatch points — the page boundary and +// the walk's end — and deliberately not the `poll-retry` wait, which holds a +// timer this dispatch would not stop on teardown. Nothing is lost by the +// omission: a deferred receive is either the pump's terminating error, which +// nothing can follow, or a disconnect frame, the server's last word on the +// socket; both are dispatched at the next page boundary either way. +func (l *loop) dispatchDeferred(at *attempt) (cycleOutcome, bool) { + d := l.deferred + if d == nil { + return cycleOutcome{}, false + } + l.deferred = nil + if d.closed { + return l.pumpExited(at, nil), true + } + return l.handleLiveFrame(at, nil, d.item, false) +} + +// ownershipCut performs SPEC.md §23's bounded admission pass: after the +// entry-poll response is accepted, receive from the frame pump's queue +// WITHOUT blocking until the queue is momentarily empty or the pass has +// dequeued liveBufferCapacity frames of any kind, whichever comes first. The +// cut is the completion of that pass, and the snapshot is what the +// state-machine-owned buffer holds at it. +// +// The bound counts dequeued frames, not admitted events: pings and control +// frames dequeue without admitting, so an event-counting bound would spin +// forever under heartbeat replenishment. The pass is deliberately not a +// drain-until-empty barrier — that races a concurrent sender and, under +// sustained arrival, never completes, so the entry position would never save. +func (l *loop) ownershipCut(at *attempt) (cycleOutcome, bool) { + return l.admissionPass(at) +} + +// socketCheck consumes what the socket has to say without waiting on it: a +// staleness firing observed here is evaluated under the same rule as in any +// select (a firing whose window overlapped a blocked hand-off is disregarded +// and re-armed), and the bounded admission pass then dispatches whatever the +// pump queued — a socket failure, a disconnect frame, or live events for the +// buffer. +func (l *loop) socketCheck(at *attempt) (cycleOutcome, bool) { + staleTimer, staleGen := at.lc.stale.current() + select { + case <-staleTimer.C(): + if age, ok := at.lc.stale.evaluate(staleGen); ok { + l.disposeAttempt(at, nil) + if l.cfg.observer.StaleConnection != nil { + l.cfg.observer.StaleConnection(age) + } + l.observeDisconnected("", errStaleConnection) + return cycleOutcome{kind: outcomeFailed}, true + } + default: + } + return l.admissionPass(at) +} + +// admissionPass is the bounded non-blocking pass over the frame pump's queue +// the ownership cut is defined as: receive without blocking until the queue is +// momentarily empty or liveBufferCapacity frames of ANY kind have been +// dequeued. Live events are admitted to the buffer, and every other frame +// takes its ordinary dispatch — so a socket failure or a disconnect frame the +// pump already queued ends the cycle here. +func (l *loop) admissionPass(at *attempt) (cycleOutcome, bool) { + for dequeued := 0; dequeued < l.cfg.liveBufferCapacity; dequeued++ { + select { + case item, ok := <-at.lc.frames: + if !ok { + return l.pumpExited(at, nil), true + } + if out, done := l.handleLiveFrame(at, nil, item, false); done { + return out, true + } + default: + return cycleOutcome{}, false + } + } + return cycleOutcome{}, false +} + +// drain replays the live buffer through the dedupe LRU (transition 23). It is +// a bounded in-memory completion, not socket delivery: it dequeues no frames +// and takes no wire waits. Everything the buffer STILL HOLDS is replayed — +// the pre-cut snapshot the save-ordering invariant is stated over, plus any +// straggler admitted after it, less anything an overflowing admission evicted +// along the way, which is the one subtraction the invariant already allows +// for and which the BufferOverflow signal reports before the held save. +func (l *loop) drain(at *attempt) (cycleOutcome, bool) { + // The scan runs under ONE budget for the whole drain — pumpDepth+1 + // dequeues: the depth of the queue being scanned, plus the one frame the + // pump may already have READ and be blocked handing off. + // + // It is deliberately NOT the ownership cut's liveBufferCapacity bound. + // That bound answers a different question ("how large an admission pass + // may the cut take before the entry position must be allowed to save"), + // it is caller-configurable down to 1, and it measures the state + // machine's buffer rather than the pump's queue. Spending it on ordinary + // frames is exactly what let a fatal frame hide: one queued ping under + // WithLiveBufferCapacity(1) consumed the whole scan, and the + // invalid_event_stream_command behind it went unseen until Streaming — + // after the held save and the caught_up announcement the carve-out exists + // to prevent. + // + // pumpDepth+1 is the bound the carve-out's guarantee falls out of, and the + // +1 is load-bearing rather than slack. The drain-start boundary the + // guarantee is stated over is "every frame the pump had ALREADY READ", + // not "every frame sitting in the queue", and those differ by exactly + // one: the hand-off queue is FIFO and holds at most pumpDepth items, + // while the pump — a single goroutine, so at most one hand-off in flight + // — may be blocked inside handOff with one more. A budget of pumpDepth + // spends itself on that frame's predecessors; the first dequeue releases + // the blocked hand-off, the held frame enters at the TAIL, and if it is + // the `invalid_event_stream_command` the carve-out exists for, the scan + // stops one short of it. The drain then completes, the held entry + // position saves and `caught_up` announces — precisely what the carve-out + // forbids. pumpDepth+1 reaches it, and cannot grow further: one reader + // goroutine can hold no more than one frame outside the queue. + // + // It still terminates — at most pumpDepth+1 dequeues for the whole drain, + // after which the scan admits nothing, the buffer empties and the drain + // ends — which is what an unbounded scan-until-empty could not promise + // against a sustained sender, and the reason the held position would + // otherwise never save. + // + // What the bound does NOT promise is a fatal frame that both ARRIVES + // after the drain began and sits behind pumpDepth other frames that also + // arrived after it. That one is consumed at the Streaming boundary under + // §23's ordinary deferred-consumption rule: it was not observable when + // the drain started, so no ordering between it and the drain's completion + // was ever established. + // + // The replay dequeues ONE event at a time, and the scan runs before each + // one. Both halves matter. Scanning first is what keeps the + // protocol-fatal carve-out ahead of every delivery; dequeuing singly is + // what keeps the drain inside the live buffer's capacity, which is a + // bound on events held AT ONCE (SPEC.md §23 sizes the connector's whole + // memory ceiling off it). Taking the buffer's whole contents into a batch + // instead let the buffer read as empty while `capacity` events were still + // pending in that batch, so the scan could admit another full capacity + // without dropping anything — twice the configured retention and twice + // the published ceiling, during exactly the slow drain the bound is for. + // + // Because nothing is held outside the buffer, an admission that overflows + // evicts the oldest events still pending — including ones this drain has + // not replayed yet. That is the same pre-cut loss condition the buffer + // always had, taking its drop-time dispatch in drainScan, before the held + // save: the conjunctive invariant asks that every such loss be explicitly + // accepted, not that a drain be exempt from the capacity it was + // configured with. + // + // Termination is unchanged, and now falls out of two independently + // decreasing quantities: each turn either delivers one buffered event or + // finds the buffer empty and ends, while the scan's budget bounds total + // admissions. Total replay work stays the buffer's occupancy at entry + // plus pumpDepth+1. + budget := pumpDepth + 1 + for { + if out, done := l.drainScan(at, &budget); done { + return out, true + } + ev, ok := l.buffer.shift() + if !ok { + return cycleOutcome{}, false + } + if !l.deliver(ev) { + l.disposeAttempt(at, nil) + return cycleOutcome{kind: outcomeClosed}, true + } + } +} + +// drainScan is Draining's protocol-fatal carve-out, and the ONLY reason the +// drain looks at the frame queue at all. SPEC.md §23: "a raw +// `invalid_event_stream_command` observed during Draining is +// Terminal(`protocol_fatal`) immediately — the drain is not completed, the +// held entry position is NOT saved, and no `caught_up` is announced; only +// recoverable failures defer." Without the scan a fatal frame — one the walk +// left deferred, or one the pump has already queued — is seen only in +// Streaming, after the held save and the caught_up announcement have both +// happened. +// +// Everything else keeps Draining's deferred consumption, which is §23's one +// deliberate deferred-consumption case: a recoverable socket failure, a +// non-fatal disconnect, an invalid frame or a closed pump is parked in the +// same single deferral slot the in-flight-poll servicing uses and dispatched +// at the Streaming boundary, once the drain has completed. Correlated events +// are admitted, so the next take replays them in arrival order, and an +// admission that would drop takes its drop-time dispatch here — before the +// held save, like every other pre-cut loss condition. +// +// budget is the drain's shared dequeue allowance, sized at the scanned +// queue's own depth (see drain); an ordinary frame consuming it must never be +// able to end the scan short of a fatal frame the pump had already queued. +func (l *loop) drainScan(at *attempt, budget *int) (cycleOutcome, bool) { + if d := l.deferred; d != nil { + // A fatal frame already in the slot is dispatched now rather than + // after the save: the carve-out is about what governs, not about + // which queue the frame is sitting in. Everything the pump has + // queued arrived behind this one, so the scan stops here either way. + if !d.closed { + if f, ok := protocolFatalFrame(d.item); ok { + l.deferred = nil + return l.terminateProtocolFatal(at, f), true + } + } + return cycleOutcome{}, false + } + for *budget > 0 { + select { + case item, ok := <-at.lc.frames: + if !ok { + l.deferred = &deferredFrame{closed: true} + return cycleOutcome{}, false + } + *budget-- + handled, out, ended := l.admitDuringPoll(at, item) + switch { + case ended: + return out, true + case handled: + default: + if f, ok := protocolFatalFrame(item); ok { + return l.terminateProtocolFatal(at, f), true + } + l.deferred = &deferredFrame{item: item} + return cycleOutcome{}, false + } + default: + return cycleOutcome{}, false + } + } + return cycleOutcome{}, false +} + +// terminateProtocolFatal takes the raw disconnect's state-generic terminal +// through the ordinary dispatch, so the frame is counted as handled and the +// disconnect observation carries the server's reason verbatim. +func (l *loop) terminateProtocolFatal(at *attempt, f frame) cycleOutcome { + if l.hooks.frameHandled != nil { + l.hooks.frameHandled(f.kind) + } + return l.dispatchDisconnect(at, nil, f) +} + +// protocolFatalFrame reports whether one pump item is a raw +// `invalid_event_stream_command` disconnect — the one frame Draining is not +// allowed to defer. +func protocolFatalFrame(item pumpItem) (frame, bool) { + if item.err != nil { + return frame{}, false + } + f, err := parseFrame(item.data) + if err != nil || f.kind != frameDisconnect || f.reason != disconnectReasonProtocolFatal { + return frame{}, false + } + return f, true +} + +// stream is the steady state (transitions 24–26): live events are delivered +// as they arrive, deduped, and never save — live ids do not advance the +// durable position. The jittered repair-poll timer is armed BEFORE the state +// is announced, so Streaming's exact timer set is {staleness, repair-poll} +// from the first instant an observer can see the state. +func (l *loop) stream(at *attempt) cycleOutcome { + repair := l.cfg.clock.NewTimer(repairJitter(l.cfg.repairInterval, l.cfg.rand), timerRepairPoll) + l.setState(stateStreaming) + for { + staleTimer, staleGen := at.lc.stale.current() + select { + case <-l.runCtx.Done(): + l.disposeAttempt(at, repair) + return cycleOutcome{kind: outcomeClosed} + case <-repair.C(): + // Transition 24 → CatchingUp: one repair walk from the connector's + // current position, returning here through Draining. The next + // cadence is armed before Streaming is re-announced, for the same + // reason it is on first entry. + out, done := l.repairWalk(at) + if done { + return out + } + repair = l.cfg.clock.NewTimer(repairJitter(l.cfg.repairInterval, l.cfg.rand), timerRepairPoll) + l.setState(stateStreaming) + case <-at.lc.stale.rearmed(): + continue + case <-staleTimer.C(): + age, ok := at.lc.stale.evaluate(staleGen) + if !ok { + // Superseded by a frame the pump received first, or suspended + // by a blocked hand-off and re-armed. + continue + } + // Transition 25's staleness trigger. + l.disposeAttempt(at, repair) + if l.cfg.observer.StaleConnection != nil { + l.cfg.observer.StaleConnection(age) + } + l.observeDisconnected("", errStaleConnection) + return cycleOutcome{kind: outcomeFailed} + case item, ok := <-at.lc.frames: + if !ok { + return l.pumpExited(at, repair) + } + if out, done := l.handleLiveFrame(at, repair, item, true); done { + return out + } + } + } +} + +// deliver yields one event to the consumer unless the delivered-id LRU has +// already served it. Dedupe is symmetric across lanes — poll page, drain, and +// streaming all check before delivering and record the delivered id — so +// poll-vs-push duplication is suppressed by id regardless of which lane got +// there first. It reports whether the consumer is still consuming: a false +// yield (a `break`) ends iteration with no error element, so nothing is ever +// yielded after it. +// +// Cancellation is checked HERE, per delivery, rather than only at the loop's +// dispatch points, because Close is callable from the consumer's own loop +// body and from any observer or handler callback — all of which run on this +// goroutine, between one delivery and the next. Close cancels synchronously +// (Connector.Close), so this check is what makes "no delivery begins after +// Close returns" true mid-page and mid-drain, not merely at the next page +// boundary. It reports the same false every caller already routes onto the +// Closed edge, and deliberately does NOT latch `stopped`: cancellation is not +// a consumer break. +func (l *loop) deliver(ev Event) bool { + if l.stopped || l.runCtx.Err() != nil { + return false + } + if l.dedupe.Seen(ev.ID) { + return true + } + if !l.yield(ev, nil) { + l.stopped = true + return false + } + return true +} + +// handleLiveFrame dispatches one pump item in a post-confirmation state. +// deliverLive selects the state's live-event disposition: Streaming delivers +// immediately; CatchingUp and the entry window admit to the live buffer +// instead (per-state invariants: catch-up delivers poll pages only). pending +// is the state's own timer, stopped with the attempt on teardown. +func (l *loop) handleLiveFrame(at *attempt, pending Timer, item pumpItem, deliverLive bool) (cycleOutcome, bool) { + if item.err != nil { + // Socket failure: peer close, read error, or the transport's + // frame-size rejection (transitions 21/25). + l.disposeAttempt(at, pending) + l.observeDisconnected("", item.err) + return cycleOutcome{kind: outcomeFailed}, true + } + f, err := parseFrame(item.data) + if err != nil { + // Invalid-frame class, parse shape: a socket-failure dispatch, never + // terminal, never a silent skip. + l.disposeAttempt(at, pending) + l.observeDisconnected("", err) + return cycleOutcome{kind: outcomeFailed}, true + } + if l.hooks.frameHandled != nil { + l.hooks.frameHandled(f.kind) + } + switch f.kind { + case frameDisconnect: + // Reason-string dispatch applied state-generically: protocol-fatal is + // terminal from every socket-open state; everything else is a socket + // drop (transition 25). + return l.dispatchDisconnect(at, pending, f), true + case frameMessage: + if f.identifier != l.identifier { + return cycleOutcome{}, false + } + ev, derr := decodeMessageEvent(f.message) + if derr != nil { + // Invalid-frame class, decode shape. + l.disposeAttempt(at, pending) + l.observeDisconnected("", derr) + return cycleOutcome{kind: outcomeFailed}, true + } + if !deliverLive { + return l.admitLive(at, pending, ev) + } + if !l.deliver(ev) { + l.disposeAttempt(at, pending) + return cycleOutcome{kind: outcomeClosed}, true + } + return cycleOutcome{}, false + default: + // welcome, ping, unknown types, and a post-confirmation confirm or + // reject: liveness only — the pump already reset staleness. + return cycleOutcome{}, false + } +} + +// pumpExited handles the pump's channel closing without a terminating error — +// only possible under a cancellation race; treated as a socket failure. +func (l *loop) pumpExited(at *attempt, pending Timer) cycleOutcome { + l.disposeAttempt(at, pending) + l.observeDisconnected("", errors.New("event feed frame pump exited")) + return cycleOutcome{kind: outcomeFailed} +} + +// walkStep is the walk's next move after a failed poll: re-poll at cursor, +// with reentry marking transitions 17/18/19's NEW entry — the entry boundary +// restarts, under presentClass. A transient's re-poll of the same cursor is +// not a re-entry: nothing about the entry changed. +type walkStep struct { + cursor Cursor + presentClass bool + reentry bool +} + +// recoverPoll disposes a failed poll seam call onto its SPEC.md §23 outcome. +// It returns the walk's next step when the walk continues, or done=true with +// the outcome that ended the cycle. Seam errors are post-retry outcomes: +// the generated operation already spent its own §7 budget inside the call, so +// nothing here is a second per-request retry layer. +func (l *loop) recoverPoll(at *attempt, cursor Cursor, err error) (walkStep, cycleOutcome, bool) { + var pe *PollError + if !errors.As(err, &pe) { + // The seam contract maps every generated outcome onto exactly one + // kind; an unclassified failure is surfaced rather than retried, as + // on the mint lane. + l.disposeAttempt(at, nil) + return walkStep{}, cycleOutcome{kind: outcomeTerminal, term: &TerminalError{ + Reason: ReasonPollFailed, Msg: "unclassified poll failure", Err: err, + }}, true + } + switch pe.Kind { + case PollTransient, PollThrottled: + // The self-loop inside CatchingUp: a wait, not a state change. The + // same cursor is re-polled. + l.pollFailures++ + out, done := l.waitPollRetry(at, pollRetryDelay(pe.RetryAfter, l.pollFailures, l.cfg.rand)) + return walkStep{cursor: cursor}, out, done + case PollUnauthorized: + // Recovery rides the reconnect cycle — the fresh mint/token pass — + // incrementing the shared connection-level counter. + l.authFailures++ + l.disposeAttempt(at, nil) + l.observeDisconnected("", err) + if l.authFailures >= authFailureThreshold { + return walkStep{}, cycleOutcome{kind: outcomeTerminal, term: &TerminalError{ + Reason: ReasonAuthorizationFailed, + Msg: fmt.Sprintf("%d consecutive connection-level authorization failures", l.authFailures), + Err: err, + }}, true + } + return walkStep{}, cycleOutcome{kind: outcomeFailed, retryAfter: pe.RetryAfter}, true + case PollFilterInvalid: + // Transition 20: a configuration error a position reset won't help; + // the server's message naming the offending list is preserved. + l.disposeAttempt(at, nil) + return walkStep{}, cycleOutcome{kind: outcomeTerminal, term: &TerminalError{ + Reason: ReasonFilterInvalid, Msg: pe.Msg, Err: err, + }}, true + case PollRedirectRefused: + // A 3xx whose Location failed the seam's per-hop validation: the + // continuation edge, NEVER poll_failed. This edge exposes the rejected + // Location redacted to its ORIGIN — a hostile continuation's path and + // query are exactly what must not be echoed — so the seam error is not + // retained as the cause: PollError.Error renders its underlying Err and + // Unwrap hands it out, and the generated error's text routinely carries + // the request URL in full. The classification survives on a sanitized + // cause carrying the same redaction the terminal promises. + l.disposeAttempt(at, nil) + return walkStep{}, cycleOutcome{kind: outcomeTerminal, term: &TerminalError{ + Reason: ReasonInvalidContinuation, + Msg: "the poll refused a redirect to " + pe.LocationOrigin, + Err: &PollError{Kind: PollRedirectRefused, LocationOrigin: pe.LocationOrigin}, + }}, true + case PollUnrecoverable: + l.disposeAttempt(at, nil) + return walkStep{}, cycleOutcome{kind: outcomeTerminal, term: &TerminalError{ + Reason: ReasonPollFailed, Msg: pe.Msg, Err: err, + }}, true + case PollGone, PollPositionInvalid, PollFilterChanged: + // The re-entry matrix, whose disposition is a NEW cursor (recovery.go). + return l.reenterWalk(at, pe) + default: + // An unknown kind is an adapter bug — the seam contract maps every + // generated outcome onto exactly one kind. It is surfaced rather than + // guessed into a re-entry, as on the mint lane: re-entering on an + // unclassified failure would move the cursor off a condition nothing + // here understands. + l.disposeAttempt(at, nil) + return walkStep{}, cycleOutcome{kind: outcomeTerminal, term: &TerminalError{ + Reason: ReasonPollFailed, + Msg: fmt.Sprintf("unknown poll error kind %d", int(pe.Kind)), + Err: err, + }}, true + } +} + +// waitPollRetry waits out the `poll-retry` timer while keeping the socket +// live: inbound frames are still consumed (admitted, not delivered — the walk +// has not reached its head), staleness still tears a dead socket down, and +// close still ends the cycle promptly. +func (l *loop) waitPollRetry(at *attempt, d time.Duration) (cycleOutcome, bool) { + t := l.cfg.clock.NewTimer(d, timerPollRetry) + for { + staleTimer, staleGen := at.lc.stale.current() + select { + case <-l.runCtx.Done(): + l.disposeAttempt(at, t) + return cycleOutcome{kind: outcomeClosed}, true + case <-t.C(): + return cycleOutcome{}, false + case <-at.lc.stale.rearmed(): + continue + case <-staleTimer.C(): + age, ok := at.lc.stale.evaluate(staleGen) + if !ok { + continue + } + // Transition 21's staleness trigger. + l.disposeAttempt(at, t) + if l.cfg.observer.StaleConnection != nil { + l.cfg.observer.StaleConnection(age) + } + l.observeDisconnected("", errStaleConnection) + return cycleOutcome{kind: outcomeFailed}, true + case item, ok := <-at.lc.frames: + if !ok { + return l.pumpExited(at, t), true + } + if out, done := l.handleLiveFrame(at, t, item, false); done { + return out, true + } + } + } +} diff --git a/go/pkg/basecamp/eventfeed/catchup_test.go b/go/pkg/basecamp/eventfeed/catchup_test.go new file mode 100644 index 0000000000..529971f063 --- /dev/null +++ b/go/pkg/basecamp/eventfeed/catchup_test.go @@ -0,0 +1,1612 @@ +// Slice-4b acceptance tests: the catch-up walk, the entry boundary's +// ownership cut and conjunctive save ordering, the drain, and streaming +// steady state (SPEC.md §23 transitions 16/20–26 plus the out-of-inventory +// checkpoint_load / invalid_continuation / poll_failed edges). Structured to +// mirror the tier-2 scenario fixtures 01, 12, 19–22, 24, 26, 28, 29 so the +// fixture driver can later subsume them; all time flows through the injected +// feedtest.Clock, and delivery-vs-save ordering is asserted on one ledger +// both sides write from the consumer's goroutine. +package eventfeed_test + +import ( + "errors" + "fmt" + "runtime" + "strings" + "testing" + "time" + + "github.com/basecamp/basecamp-sdk/go/pkg/basecamp/eventfeed" + "github.com/basecamp/basecamp-sdk/go/pkg/basecamp/eventfeed/feedtest" +) + +// pollEvent builds one poll-lane row: the eight always-present keys, and no +// visible_to_clients (poll rows omit it — the presence asymmetry). +func pollEvent(id int64) eventfeed.Event { + return eventfeed.Event{ + ID: id, + Kind: "message", + EventType: "message.created", + Action: "created", + CreatedAt: time.Date(2026, 8, 1, 12, 0, 0, 0, time.UTC), + BucketID: 2, + CreatorID: 3, + RecordingID: 900, + } +} + +// storedHarness wires a harness to a scripted checkpoint store recording its +// saves onto the shared ledger, so "delivery strictly precedes this page's +// save" is a statement about one ordered list. +func storedHarness(t *testing.T, store *feedtest.Store, opts ...eventfeed.Option) *harness { + t.Helper() + base := make([]eventfeed.Option, 0, 2+len(opts)) + base = append(base, + eventfeed.WithCheckpointStore(store), + eventfeed.WithConsumerNamespace("agent")) + h := newHarness(t, append(base, opts...)...) + store.OnSave(func(position string) { h.record("save " + position) }) + return h +} + +func assertLedger(t *testing.T, got, want []string) { + t.Helper() + if len(got) != len(want) { + t.Fatalf("ledger = %v, want %v", got, want) + } + for i := range want { + if got[i] != want[i] { + t.Fatalf("ledger = %v, want %v", got, want) + } + } +} + +func assertIDs(t *testing.T, got []int64, want ...int64) { + t.Helper() + if len(got) != len(want) { + t.Fatalf("delivered ids = %v, want %v", got, want) + } + for i := range want { + if got[i] != want[i] { + t.Fatalf("delivered ids = %v, want %v", got, want) + } + } +} + +func assertPositions(t *testing.T, got []string, want ...string) { + t.Helper() + if len(got) != len(want) { + t.Fatalf("checkpoint ledger = %v, want %v", got, want) + } + for i := range want { + if got[i] != want[i] { + t.Fatalf("checkpoint ledger = %v, want %v", got, want) + } + } +} + +// TestCatchUpWalkCheckpointsAfterEachPage is fixtures 12 + 01: a +// position-resume entry walks the pages, delivering each page's events BEFORE +// that page's position is saved, follows `next` verbatim, drains the live +// buffer after the walk, and then streams — with live ids never advancing the +// durable position and a re-pushed poll id suppressed by the dedupe LRU. +func TestCatchUpWalkCheckpointsAfterEachPage(t *testing.T) { + store := feedtest.NewStore() + store.Stored("pos-0") + h := storedHarness(t, store) + h.minter.ScriptTicket(ticket(1)) + next := testOrigin + "/999/events.json?after=102" + h.polls.ScriptPage(eventfeed.PollPage{ + Events: []eventfeed.Event{pollEvent(101), pollEvent(102)}, + Position: "pos-1", + Next: next, + }) + h.polls.ScriptPage(eventfeed.PollPage{Events: []eventfeed.Event{pollEvent(103)}, Position: "pos-2"}) + base := runtime.NumGoroutine() + h.start() + + conn := h.driveToSubscribed() + h.serveSettled(conn, frameMessage(noFilterIdentifier, 104)) + conn.Serve(frameConfirm(noFilterIdentifier)) + h.awaitStreaming() + + assertLedger(t, h.ledger(), []string{ + "event 101", "event 102", "save pos-1", + "event 103", "save pos-2", + "event 104", + }) + assertPositions(t, store.Saves(), "pos-1", "pos-2") + + calls := h.polls.Calls() + if len(calls) != 2 { + t.Fatalf("poll seam calls = %d, want 2", len(calls)) + } + if calls[0].Cursor != (eventfeed.Cursor{Position: "pos-0"}) { + t.Fatalf("entry cursor = %+v, want the loaded position", calls[0].Cursor) + } + if calls[1].Cursor != (eventfeed.Cursor{PageURL: next}) { + t.Fatalf("continuation cursor = %+v, want the `next` URL verbatim", calls[1].Cursor) + } + assertTimers(t, h.clock, map[string]int{timerStaleness: 1, timerRepairPoll: 1}) + + // Streaming: a fresh live id is delivered; a re-push of a poll-served id + // is suppressed by delivered-id dedupe; neither saves. + conn.Serve(frameMessage(noFilterIdentifier, 105)) + h.waitUntil("live delivery", func() bool { return len(h.deliveredIDs()) == 5 }) + h.serveSettled(conn, frameMessage(noFilterIdentifier, 103)) + conn.Serve(frameMessage(noFilterIdentifier, 106)) + h.waitUntil("post-duplicate delivery", func() bool { return len(h.deliveredIDs()) == 6 }) + assertIDs(t, h.deliveredIDs(), 101, 102, 103, 104, 105, 106) + assertPositions(t, store.Saves(), "pos-1", "pos-2") + + h.conn.Close() + h.join() + assertTimers(t, h.clock, map[string]int{}) + assertGoroutinesSettle(t, base) +} + +// TestPresentEntryHoldsPositionUntilDrained is fixture 19: on a present-class +// entry the entry poll's position is HELD, the admitted straggler is +// delivered, and only then does the position save — the conjunctive +// save-ordering invariant. A save landing first would fail the ledger. +func TestPresentEntryHoldsPositionUntilDrained(t *testing.T) { + store := feedtest.NewStore() // Missing → present-class entry + h := storedHarness(t, store) + h.minter.ScriptTicket(ticket(1)) + h.polls.ScriptPage(eventfeed.PollPage{Position: "pos-1"}) + h.start() + + conn := h.driveToSubscribed() + h.serveSettled(conn, frameMessage(noFilterIdentifier, 41)) + conn.Serve(frameConfirm(noFilterIdentifier)) + h.awaitStreaming() + + assertLedger(t, h.ledger(), []string{"event 41", "save pos-1"}) + assertIDs(t, h.deliveredIDs(), 41) + assertPositions(t, store.Saves(), "pos-1") + if calls := h.polls.Calls(); len(calls) != 1 || calls[0].Cursor != (eventfeed.Cursor{}) { + t.Fatalf("poll calls = %+v, want one bare present entry", calls) + } + assertTimers(t, h.clock, map[string]int{timerStaleness: 1, timerRepairPoll: 1}) +} + +// TestOwnershipCutAdmitsFramesArrivingDuringTheEntryPoll pins the cut's +// observation rule: a straggler that arrives AFTER confirmation and while the +// entry poll is in flight counts as observed — admitted into the +// state-machine-owned buffer at or before the cut, whether the state machine +// admits it while the seam call is outstanding or in the bounded pass taken +// once the response is accepted — so it is delivered before the held position +// saves. +func TestOwnershipCutAdmitsFramesArrivingDuringTheEntryPoll(t *testing.T) { + store := feedtest.NewStore() + h := storedHarness(t, store) + h.minter.ScriptTicket(ticket(1)) + h.polls.ScriptPage(eventfeed.PollPage{Position: "pos-1"}) + + var conn *feedtest.Conn + handed := make(chan struct{}) + h.polls.OnCall(func(feedtest.PollCall) { + // Inside the entry poll seam call: serve the straggler plus a + // trailing ping and wait for the pump to take both, so the frame is + // queued for the state machine before the response is accepted. + conn.Serve(frameMessage(noFilterIdentifier, 41)) + conn.Serve(framePing()) + deadline := time.Now().Add(watchdog) + for conn.Pending() > 0 && time.Now().Before(deadline) { + time.Sleep(time.Millisecond) + } + close(handed) + }) + h.start() + + conn = h.driveToSubscribed() + conn.Serve(frameConfirm(noFilterIdentifier)) + select { + case <-handed: + case <-time.After(watchdog): + t.Fatal("the entry poll was never reached") + } + h.awaitStreaming() + + assertLedger(t, h.ledger(), []string{"event 41", "save pos-1"}) + assertPositions(t, store.Saves(), "pos-1") +} + +// TestPostSnapshotStragglerDeliveredAndDeduped is fixture 20: a lower-id +// straggler arriving after the snapshot and its save is still delivered — the +// LRU tracks delivered ids, never position ordering — a repeat is suppressed, +// and the position never regresses. +func TestPostSnapshotStragglerDeliveredAndDeduped(t *testing.T) { + store := feedtest.NewStore() + h := storedHarness(t, store) + h.minter.ScriptTicket(ticket(1)) + h.polls.ScriptPage(eventfeed.PollPage{Position: "pos-1"}) + h.start() + + conn := h.driveToSubscribed() + conn.Serve(frameConfirm(noFilterIdentifier)) + h.awaitStreaming() + assertPositions(t, store.Saves(), "pos-1") + + conn.Serve(frameMessage(noFilterIdentifier, 41)) + h.waitUntil("straggler delivered", func() bool { return len(h.deliveredIDs()) == 1 }) + h.serveSettled(conn, frameMessage(noFilterIdentifier, 41)) + conn.Serve(frameMessage(noFilterIdentifier, 42)) + h.waitUntil("second delivery", func() bool { return len(h.deliveredIDs()) == 2 }) + + assertIDs(t, h.deliveredIDs(), 41, 42) + assertPositions(t, store.Saves(), "pos-1") +} + +// TestOverflowAcceptedStillDeliversRetained is fixture 22: accepting a +// BufferOverflow is not license to skip retained deliveries — the retained +// pre-cut events complete BEFORE the held entry position saves. +func TestOverflowAcceptedStillDeliversRetained(t *testing.T) { + var invocations int + handler := func(s eventfeed.Signal) eventfeed.Disposition { + ov, ok := s.(eventfeed.BufferOverflow) + if !ok || ov.DroppedCount != 1 || len(ov.DroppedIDs) != 1 || ov.DroppedIDs[0] != 51 { + t.Errorf("signal = %+v, want BufferOverflow{DroppedIDs:[51], DroppedCount:1}", s) + } + invocations++ + return eventfeed.Accept + } + store := feedtest.NewStore() + h := storedHarness(t, store, + eventfeed.WithLiveBufferCapacity(2), + eventfeed.WithSignalHandler(handler)) + h.minter.ScriptTicket(ticket(1)) + h.polls.ScriptPage(eventfeed.PollPage{Position: "pos-1"}) + h.start() + + conn := h.driveToSubscribed() + h.serveSettled(conn, + frameMessage(noFilterIdentifier, 51), + frameMessage(noFilterIdentifier, 52), + frameMessage(noFilterIdentifier, 53)) + conn.Serve(frameConfirm(noFilterIdentifier)) + h.awaitStreaming() + + assertLedger(t, h.ledger(), []string{"event 52", "event 53", "save pos-1"}) + assertPositions(t, store.Saves(), "pos-1") + if invocations != 1 { + t.Fatalf("handler invocations = %d, want exactly 1", invocations) + } +} + +// TestOverflowDefaultTerminalNeverSaves is fixture 21: with no handler the +// overflow is terminal — and structurally there is no entry poll and no save. +func TestOverflowDefaultTerminalNeverSaves(t *testing.T) { + store := feedtest.NewStore() + h := storedHarness(t, store, eventfeed.WithLiveBufferCapacity(2)) + h.minter.ScriptTicket(ticket(1)) + h.start() + + conn := h.driveToSubscribed() + conn.Serve(frameMessage(noFilterIdentifier, 51)) + conn.Serve(frameMessage(noFilterIdentifier, 52)) + conn.Serve(frameMessage(noFilterIdentifier, 53)) + h.join() + + _, terminal, _ := h.snapshot() + if terminal == nil || terminal.Reason != eventfeed.ReasonBufferOverflow { + t.Fatalf("terminal = %v, want reason %q", terminal, eventfeed.ReasonBufferOverflow) + } + if got := h.polls.CallCount(); got != 0 { + t.Fatalf("poll seam calls = %d, want 0", got) + } + assertPositions(t, store.Saves()) + if !conn.Closed() { + t.Fatal("teardown must close the socket") + } + assertTimers(t, h.clock, map[string]int{}) +} + +// TestCheckpointLoadFailureIsTerminal is fixture 28: Failed(load) is +// Terminal(checkpoint_load) with ZERO wire attempts, and is deliberately not +// collapsible to Missing (which would silently start at the present). +func TestCheckpointLoadFailureIsTerminal(t *testing.T) { + sentinel := errors.New("store unreadable") + store := feedtest.NewStore() + store.FailLoad(sentinel) + h := storedHarness(t, store) + h.minter.ScriptTicket(ticket(1)) + h.start() + h.join() + + _, terminal, elements := h.snapshot() + if terminal == nil || terminal.Reason != eventfeed.ReasonCheckpointLoad { + t.Fatalf("terminal = %v, want reason %q", terminal, eventfeed.ReasonCheckpointLoad) + } + if !errors.Is(terminal, sentinel) { + t.Fatalf("terminal must wrap the store error; got %v", terminal) + } + if elements != 1 { + t.Fatalf("iteration elements = %d, want exactly the one error element", elements) + } + if got := h.minter.Calls(); got != 0 { + t.Fatalf("mint seam calls = %d, want 0 (load precedes the first mint)", got) + } + if got := len(h.tr.Dials()); got != 0 { + t.Fatalf("connect seam calls = %d, want 0", got) + } + if got := h.polls.CallCount(); got != 0 { + t.Fatalf("poll seam calls = %d, want 0", got) + } + assertTimers(t, h.clock, map[string]int{}) +} + +// TestCheckpointIdentityAndEntrySelection: load happens exactly once under +// the full four-part identity, Missing enters present-class, Loaded enters at +// the stored position, and saves carry the same key. +func TestCheckpointIdentityAndEntrySelection(t *testing.T) { + filters := eventfeed.Filters{Types: []string{"message.created"}} + wantKey := eventfeed.CheckpointKey{ + Origin: testOrigin, + AccountID: "5951425", + ConsumerNamespace: "agent", + FilterKey: filters.FilterKey(), + } + + t.Run("missing enters present-class", func(t *testing.T) { + store := feedtest.NewStore() + h := storedHarness(t, store, eventfeed.WithFilters(filters)) + h.minter.ScriptTicket(ticket(1)) + h.polls.ScriptPage(eventfeed.PollPage{Position: "pos-1"}) + h.start() + + conn := h.driveToSubscribed() + conn.Serve(frameConfirm(eventfeed.ExportSubscribeIdentifier(filters))) + b := h.awaitBoundary() + if b.Entry != (eventfeed.Cursor{}) || !b.PresentClass { + t.Fatalf("entry = %+v present=%v, want the bare present entry", b.Entry, b.PresentClass) + } + h.awaitStreaming() + loads := store.Loads() + if len(loads) != 1 || loads[0] != wantKey { + t.Fatalf("load keys = %+v, want exactly one %+v", loads, wantKey) + } + keys := store.SaveKeys() + if len(keys) != 1 || keys[0] != wantKey { + t.Fatalf("save keys = %+v, want exactly one %+v", keys, wantKey) + } + }) + + t.Run("loaded enters at the stored position", func(t *testing.T) { + store := feedtest.NewStore() + store.Stored("pos-9") + h := storedHarness(t, store, eventfeed.WithFilters(filters)) + h.minter.ScriptTicket(ticket(1)) + h.polls.ScriptPage(eventfeed.PollPage{Position: "pos-10"}) + h.start() + + conn := h.driveToSubscribed() + conn.Serve(frameConfirm(eventfeed.ExportSubscribeIdentifier(filters))) + b := h.awaitBoundary() + if b.Entry != (eventfeed.Cursor{Position: "pos-9"}) || b.PresentClass { + t.Fatalf("entry = %+v present=%v, want the stored position, position-resume class", b.Entry, b.PresentClass) + } + }) +} + +// TestCheckpointSaveFailureContinues is fixture 29's store half: a failed +// save is an observer notification, the feed continues, and the NEXT save is +// still attempted — there is no save circuit breaker. +func TestCheckpointSaveFailureContinues(t *testing.T) { + sentinel := errors.New("disk full") + store := feedtest.NewStore() + store.Stored("pos-0") + store.FailNextSave(sentinel) + + var saveFailures []error + var checkpoints []string + obs := eventfeed.Observer{ + CheckpointSaveFailed: func(err error) { saveFailures = append(saveFailures, err) }, + Checkpoint: func(position string) { checkpoints = append(checkpoints, position) }, + } + h := storedHarness(t, store, eventfeed.WithObserver(obs)) + h.minter.ScriptTicket(ticket(1)) + next := testOrigin + "/999/events.json?after=101" + h.polls.ScriptPage(eventfeed.PollPage{Events: []eventfeed.Event{pollEvent(101)}, Position: "pos-1", Next: next}) + h.polls.ScriptPage(eventfeed.PollPage{Events: []eventfeed.Event{pollEvent(102)}, Position: "pos-2"}) + h.start() + + conn := h.driveToSubscribed() + conn.Serve(frameConfirm(noFilterIdentifier)) + h.awaitStreaming() + + assertIDs(t, h.deliveredIDs(), 101, 102) + assertPositions(t, store.Saves(), "pos-1", "pos-2") + if len(saveFailures) != 1 || !errors.Is(saveFailures[0], sentinel) { + t.Fatalf("CheckpointSaveFailed = %v, want exactly the one store error", saveFailures) + } + if len(checkpoints) != 1 || checkpoints[0] != "pos-2" { + t.Fatalf("Checkpoint = %v, want only the position that durably saved", checkpoints) + } +} + +// TestHostileContinuationIsTerminal is fixture 26: a cross-origin `next` +// terminates with invalid_continuation, no request is issued to the foreign +// origin, and the error carries the rejected URL redacted to its origin. +func TestHostileContinuationIsTerminal(t *testing.T) { + store := feedtest.NewStore() + store.Stored("pos-0") + h := storedHarness(t, store) + h.minter.ScriptTicket(ticket(1)) + h.polls.ScriptPage(eventfeed.PollPage{ + Events: []eventfeed.Event{pollEvent(101)}, + Position: "pos-1", + Next: "https://attacker.example.com/999/events.json?after=101&token=secret", + }) + h.start() + + conn := h.driveToSubscribed() + conn.Serve(frameConfirm(noFilterIdentifier)) + h.join() + + _, terminal, _ := h.snapshot() + if terminal == nil || terminal.Reason != eventfeed.ReasonInvalidContinuation { + t.Fatalf("terminal = %v, want reason %q", terminal, eventfeed.ReasonInvalidContinuation) + } + if !strings.Contains(terminal.Msg, "https://attacker.example.com") { + t.Fatalf("terminal message %q should name the rejected origin", terminal.Msg) + } + if strings.Contains(terminal.Msg, "token=secret") || strings.Contains(terminal.Msg, "/999/") { + t.Fatalf("terminal message %q must be redacted to the origin", terminal.Msg) + } + if got := h.polls.CallCount(); got != 1 { + t.Fatalf("poll seam calls = %d, want 1 (zero requests to the foreign origin)", got) + } + assertIDs(t, h.deliveredIDs(), 101) + assertPositions(t, store.Saves(), "pos-1") + if !conn.Closed() { + t.Fatal("teardown must close the socket") + } + assertTimers(t, h.clock, map[string]int{}) +} + +// TestContinuationValidation covers the rejected shapes (cross-origin, +// protocol downgrade, non-http(s) scheme, relative) and the accepted one (a +// same-origin absolute URL with a different path and query). +func TestContinuationValidation(t *testing.T) { + rejected := []struct { + name string + next string + }{ + {"cross-origin host", "https://attacker.example.com/events.json"}, + {"protocol downgrade", "http://3.basecampapi.com/999/events.json"}, + {"foreign port", "https://3.basecampapi.com:8443/999/events.json"}, + {"non-http scheme", "file:///etc/passwd"}, + {"relative", "/999/events.json?after=101"}, + // Authority is not hostname: ":443" is a nonempty url.Host with an + // empty hostname. CanonicalOrigin already reads Hostname(), so this + // takes the terminal path; the case is here so it stays that way. + {"port-only authority", "https://:443/999/events.json"}, + } + for _, tc := range rejected { + t.Run(tc.name, func(t *testing.T) { + h := newHarness(t) + h.minter.ScriptTicket(ticket(1)) + h.polls.ScriptPage(eventfeed.PollPage{Position: "pos-1", Next: tc.next}) + h.start() + + conn := h.driveToSubscribed() + conn.Serve(frameConfirm(noFilterIdentifier)) + h.join() + + _, terminal, _ := h.snapshot() + if terminal == nil || terminal.Reason != eventfeed.ReasonInvalidContinuation { + t.Fatalf("terminal = %v, want reason %q", terminal, eventfeed.ReasonInvalidContinuation) + } + if got := h.polls.CallCount(); got != 1 { + t.Fatalf("poll seam calls = %d, want 1 (the failing URL is never requested)", got) + } + }) + } + + t.Run("same-origin continuation is followed", func(t *testing.T) { + h := newHarness(t) + h.minter.ScriptTicket(ticket(1)) + next := testOrigin + "/999/events.json?after=101&filter=x" + h.polls.ScriptPage(eventfeed.PollPage{Events: []eventfeed.Event{pollEvent(101)}, Position: "pos-1", Next: next}) + h.polls.ScriptPage(eventfeed.PollPage{Events: []eventfeed.Event{pollEvent(102)}, Position: "pos-2"}) + h.start() + + conn := h.driveToSubscribed() + conn.Serve(frameConfirm(noFilterIdentifier)) + h.awaitStreaming() + assertIDs(t, h.deliveredIDs(), 101, 102) + if calls := h.polls.Calls(); len(calls) != 2 || calls[1].Cursor.PageURL != next { + t.Fatalf("poll calls = %+v, want the continuation followed verbatim", calls) + } + }) +} + +// TestPollRetryTiming: transient and throttled poll failures retry inside +// CatchingUp on the `poll-retry` timer at the SAME cursor — never terminal — +// with a server-directed Retry-After waited exactly and the local draw +// bounded by the consecutive-poll-failure index. +func TestPollRetryTiming(t *testing.T) { + t.Run("transient retries the same cursor", func(t *testing.T) { + store := feedtest.NewStore() + store.Stored("pos-0") + h := storedHarness(t, store) + h.minter.ScriptTicket(ticket(1)) + h.polls.ScriptError(&eventfeed.PollError{Kind: eventfeed.PollTransient, Err: errors.New("502")}) + h.polls.ScriptPage(eventfeed.PollPage{Events: []eventfeed.Event{pollEvent(101)}, Position: "pos-1"}) + h.start() + + conn := h.driveToSubscribed() + conn.Serve(frameConfirm(noFilterIdentifier)) + h.awaitTimer(timerPollRetry) + assertTimers(t, h.clock, map[string]int{timerStaleness: 1, timerPollRetry: 1}) + if d := h.fireTimer(timerPollRetry); d < 0 || d >= time.Second { + t.Fatalf("first poll-retry delay = %s, want within [0, 1s) (full jitter, k=1)", d) + } + h.awaitStreaming() + + calls := h.polls.Calls() + if len(calls) != 2 || calls[0].Cursor != calls[1].Cursor { + t.Fatalf("poll calls = %+v, want the retry at the same cursor", calls) + } + assertIDs(t, h.deliveredIDs(), 101) + if _, terminal, _ := h.snapshot(); terminal != nil { + t.Fatalf("a transient poll is never terminal; got %v", terminal) + } + }) + + t.Run("throttled waits the Retry-After exactly", func(t *testing.T) { + h := newHarness(t) + h.minter.ScriptTicket(ticket(1)) + h.polls.ScriptError(&eventfeed.PollError{Kind: eventfeed.PollThrottled, RetryAfter: 90 * time.Second}) + h.polls.ScriptPage(eventfeed.PollPage{Position: "pos-1"}) + h.start() + + conn := h.driveToSubscribed() + conn.Serve(frameConfirm(noFilterIdentifier)) + if d := h.fireTimer(timerPollRetry); d != 90*time.Second { + t.Fatalf("poll-retry delay = %s, want the 90s Retry-After exactly (cap-exempt)", d) + } + h.awaitStreaming() + }) +} + +// TestPollFailureClassification: the terminal poll dispositions this slice +// owns — filter_invalid with the server's message preserved, poll_failed for +// unrecoverable and unclassified outcomes, and invalid_continuation for a +// refused redirect (never poll_failed). +func TestPollFailureClassification(t *testing.T) { + cases := []struct { + name string + err error + reason eventfeed.TerminalReason + wantMsg string + }{ + { + "filter invalid preserves the server message", + &eventfeed.PollError{Kind: eventfeed.PollFilterInvalid, Msg: "unknown event types: nope.created"}, + eventfeed.ReasonFilterInvalid, + "unknown event types: nope.created", + }, + { + "unrecoverable is poll_failed", + &eventfeed.PollError{Kind: eventfeed.PollUnrecoverable, Err: errors.New("404 not found")}, + eventfeed.ReasonPollFailed, + "", + }, + { + "unclassified is poll_failed", + errors.New("adapter bug: unclassified"), + eventfeed.ReasonPollFailed, + "", + }, + { + "an unknown kind is poll_failed, never a guessed re-entry", + &eventfeed.PollError{Err: errors.New("adapter bug: unset kind")}, + eventfeed.ReasonPollFailed, + "unknown poll error kind 0", + }, + { + "redirect refused is invalid_continuation", + &eventfeed.PollError{Kind: eventfeed.PollRedirectRefused, LocationOrigin: "https://attacker.example.com"}, + eventfeed.ReasonInvalidContinuation, + "https://attacker.example.com", + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + h := newHarness(t) + h.minter.ScriptTicket(ticket(1)) + h.polls.ScriptError(tc.err) + h.start() + + conn := h.driveToSubscribed() + conn.Serve(frameConfirm(noFilterIdentifier)) + h.join() + + _, terminal, _ := h.snapshot() + if terminal == nil || terminal.Reason != tc.reason { + t.Fatalf("terminal = %v, want reason %q", terminal, tc.reason) + } + if tc.wantMsg != "" && !strings.Contains(terminal.Msg, tc.wantMsg) { + t.Fatalf("terminal message %q should carry %q", terminal.Msg, tc.wantMsg) + } + if got := h.minter.Calls(); got != 1 { + t.Fatalf("mint seam calls = %d, want 1 (terminal, never retried into)", got) + } + if !conn.Closed() { + t.Fatal("teardown must close the socket") + } + assertTimers(t, h.clock, map[string]int{}) + }) + } +} + +// TestUnauthorizedPollRidesReconnectCycle: an unauthorized poll tears the +// attempt down to Backoff (the fresh mint/token cycle is its recovery path) +// and increments the SHARED authorization counter — while a successful poll +// page is the one thing that resets it. +func TestUnauthorizedPollRidesReconnectCycle(t *testing.T) { + unauthorizedPoll := func() error { + return &eventfeed.PollError{Kind: eventfeed.PollUnauthorized, Err: errors.New("401")} + } + + t.Run("first failure reconnects with a fresh ticket", func(t *testing.T) { + h := newHarness(t) + h.minter.ScriptTicket(ticket(1)) + h.minter.ScriptTicket(ticket(2)) + h.polls.ScriptError(unauthorizedPoll()) + h.polls.ScriptPage(eventfeed.PollPage{Position: "pos-1"}) + h.start() + + conn1 := h.driveToSubscribed() + conn1.Serve(frameConfirm(noFilterIdentifier)) + h.awaitTimer(timerBackoff) + if !conn1.Closed() { + t.Fatal("an unauthorized poll must tear the attempt down") + } + assertTimers(t, h.clock, map[string]int{timerBackoff: 1}) + h.fireTimer(timerBackoff) + conn2 := h.driveToSubscribed() + conn2.Serve(frameConfirm(noFilterIdentifier)) + h.awaitStreaming() + if got := h.minter.Calls(); got != 2 { + t.Fatalf("mint seam calls = %d, want 2", got) + } + if _, terminal, _ := h.snapshot(); terminal != nil { + t.Fatalf("one unauthorized poll must not be terminal; got %v", terminal) + } + }) + + t.Run("three consecutive are terminal", func(t *testing.T) { + h := newHarness(t) + for i := 1; i <= 3; i++ { + h.minter.ScriptTicket(ticket(i)) + h.polls.ScriptError(unauthorizedPoll()) + } + h.start() + + for i := 1; i <= 2; i++ { + conn := h.driveToSubscribed() + conn.Serve(frameConfirm(noFilterIdentifier)) + h.fireTimer(timerBackoff) + } + conn := h.driveToSubscribed() + conn.Serve(frameConfirm(noFilterIdentifier)) + h.join() + + _, terminal, _ := h.snapshot() + if terminal == nil || terminal.Reason != eventfeed.ReasonAuthorizationFailed { + t.Fatalf("terminal = %v, want reason %q", terminal, eventfeed.ReasonAuthorizationFailed) + } + assertTimers(t, h.clock, map[string]int{}) + }) + + t.Run("a successful page resets the shared counter", func(t *testing.T) { + h := newHarness(t) + h.minter.ScriptTicket(ticket(1)) + h.polls.ScriptError(unauthorizedPoll()) // failure 1 + h.minter.ScriptTicket(ticket(2)) + h.polls.ScriptPage(eventfeed.PollPage{Position: "pos-1"}) // resets to 0 + h.minter.ScriptError(&eventfeed.MintError{Kind: eventfeed.MintUnauthorized}) + h.minter.ScriptError(&eventfeed.MintError{Kind: eventfeed.MintUnauthorized}) + h.start() + + conn1 := h.driveToSubscribed() + conn1.Serve(frameConfirm(noFilterIdentifier)) + h.fireTimer(timerBackoff) + conn2 := h.driveToSubscribed() + conn2.Serve(frameConfirm(noFilterIdentifier)) + h.awaitStreaming() + + // Two more unauthorized failures: without the reset this would be the + // third consecutive and terminal. + conn2.Serve(frameDisconnect("unauthorized", false)) + h.fireTimer(timerBackoff) + h.fireTimer(timerBackoff) + h.waitUntil("both unauthorized mints attempted", func() bool { return h.minter.Calls() == 4 }) + if _, terminal, _ := h.snapshot(); terminal != nil { + t.Fatalf("the successful page must reset the counter; got %v", terminal) + } + }) +} + +// TestSocketFailureMidWalk is transition 21: a socket failure observed while +// the walk waits on `poll-retry` tears the attempt down to Backoff, keeping +// the per-page checkpoints already saved. +func TestSocketFailureMidWalk(t *testing.T) { + store := feedtest.NewStore() + store.Stored("pos-0") + h := storedHarness(t, store) + h.minter.ScriptTicket(ticket(1)) + h.minter.ScriptTicket(ticket(2)) + next := testOrigin + "/999/events.json?after=101" + h.polls.ScriptPage(eventfeed.PollPage{Events: []eventfeed.Event{pollEvent(101)}, Position: "pos-1", Next: next}) + h.polls.ScriptError(&eventfeed.PollError{Kind: eventfeed.PollTransient, Err: errors.New("502")}) + h.start() + + conn := h.driveToSubscribed() + conn.Serve(frameConfirm(noFilterIdentifier)) + h.awaitTimer(timerPollRetry) + conn.FailReads(errors.New("connection reset")) + h.awaitTimer(timerBackoff) + + if !conn.Closed() { + t.Fatal("the socket failure must dispose the attempt") + } + assertTimers(t, h.clock, map[string]int{timerBackoff: 1}) + assertPositions(t, store.Saves(), "pos-1") + if _, terminal, _ := h.snapshot(); terminal != nil { + t.Fatalf("a mid-walk socket failure is never terminal; got %v", terminal) + } +} + +// TestStreamingFailureEdges: transition 26 (a raw protocol-fatal disconnect +// read while streaming is terminal from every socket-open state) and +// transition 25 (staleness expiry tears the socket down to Backoff). +func TestStreamingFailureEdges(t *testing.T) { + stream := func(t *testing.T, h *harness) *feedtest.Conn { + t.Helper() + h.minter.ScriptTicket(ticket(1)) + h.minter.ScriptTicket(ticket(2)) + h.polls.ScriptPage(eventfeed.PollPage{Position: "pos-1"}) + h.start() + conn := h.driveToSubscribed() + conn.Serve(frameConfirm(noFilterIdentifier)) + h.awaitStreaming() + return conn + } + + t.Run("protocol fatal is terminal", func(t *testing.T) { + h := newHarness(t) + conn := stream(t, h) + conn.Serve(frameDisconnect("invalid_event_stream_command", false)) + h.join() + _, terminal, _ := h.snapshot() + if terminal == nil || terminal.Reason != eventfeed.ReasonProtocolFatal { + t.Fatalf("terminal = %v, want reason %q", terminal, eventfeed.ReasonProtocolFatal) + } + if got := h.minter.Calls(); got != 1 { + t.Fatalf("mint seam calls = %d, want 1 (never retried into)", got) + } + assertTimers(t, h.clock, map[string]int{}) + }) + + t.Run("staleness reconnects", func(t *testing.T) { + h := newHarness(t) + conn := stream(t, h) + h.clock.Advance(7500 * time.Millisecond) + h.awaitTimer(timerBackoff) + if !conn.Closed() { + t.Fatal("staleness must tear the socket down") + } + assertTimers(t, h.clock, map[string]int{timerBackoff: 1}) + if _, terminal, _ := h.snapshot(); terminal != nil { + t.Fatalf("staleness is never terminal; got %v", terminal) + } + }) +} + +// TestRepairPollTimerArmedOnStreaming: entry to Streaming arms the jittered +// `repair-poll` timer (60s ± 20%), and every cycle re-arms it once the walk +// that firing drove has returned through Draining. +func TestRepairPollTimerArmedOnStreaming(t *testing.T) { + h := newHarness(t) + h.minter.ScriptTicket(ticket(1)) + for _, position := range []string{"pos-1", "pos-2", "pos-3"} { + h.polls.ScriptPage(eventfeed.PollPage{Position: position}) + } + h.start() + + conn := h.driveToSubscribed() + conn.Serve(frameConfirm(noFilterIdentifier)) + h.awaitStreaming() + + for cycle := 1; cycle <= 2; cycle++ { + d := h.fireTimer(timerRepairPoll) + if d < 48*time.Second || d > 72*time.Second { + t.Fatalf("repair-poll delay (cycle %d) = %s, want within 60s ± 20%%", cycle, d) + } + h.awaitStreaming() + } + assertTimers(t, h.clock, map[string]int{timerStaleness: 1, timerRepairPoll: 1}) +} + +// TestConsumerBreakSkipsThePagesCheckpoint: a consumer that breaks mid-page +// ends iteration cleanly and that page's position is never saved — +// checkpoint-after-processing, structurally. +func TestConsumerBreakSkipsThePagesCheckpoint(t *testing.T) { + store := feedtest.NewStore() + store.Stored("pos-0") + h := storedHarness(t, store) + h.breakAfter = 1 + h.minter.ScriptTicket(ticket(1)) + h.polls.ScriptPage(eventfeed.PollPage{ + Events: []eventfeed.Event{pollEvent(101), pollEvent(102)}, + Position: "pos-1", + }) + base := runtime.NumGoroutine() + h.start() + + conn := h.driveToSubscribed() + conn.Serve(frameConfirm(noFilterIdentifier)) + h.join() + + assertIDs(t, h.deliveredIDs(), 101) + assertPositions(t, store.Saves()) + if _, terminal, _ := h.snapshot(); terminal != nil { + t.Fatalf("a consumer break ends iteration with no error element; got %v", terminal) + } + if !conn.Closed() { + t.Fatal("a consumer break must dispose the attempt") + } + assertTimers(t, h.clock, map[string]int{}) + assertGoroutinesSettle(t, base) +} + +// TestNoStoreNeverSaves: with no checkpoint store the walk still delivers and +// advances its in-memory position — the store is write-through durability, +// not the position's authority. +func TestNoStoreNeverSaves(t *testing.T) { + h := newHarness(t) + h.minter.ScriptTicket(ticket(1)) + h.polls.ScriptPage(eventfeed.PollPage{Events: []eventfeed.Event{pollEvent(101)}, Position: "pos-1"}) + h.start() + + conn := h.driveToSubscribed() + conn.Serve(frameConfirm(noFilterIdentifier)) + h.awaitStreaming() + assertIDs(t, h.deliveredIDs(), 101) + if got := fmt.Sprint(h.ledger()); got != "[event 101]" { + t.Fatalf("ledger = %s, want only the delivery", got) + } +} + +// startOverflowUnderTheEntryPoll drives a position-resume walk to the moment +// the live buffer fills UNDER an in-flight entry poll: 41 and 42 are admitted +// by the in-flight servicing, 43 cannot be, so its drop is dispatched right +// there — in the goroutine that received it, with the seam call still +// outstanding. +// +// The rendezvous on the raised signal is what makes the ordering deterministic +// rather than sampled: the scripted page is produced only after the poll's +// OnCall returns, so waiting there for the drop removes the state machine's +// race between the queued frames and the finished call. onSignal, when set, +// observes each signal ahead of that rendezvous. +func startOverflowUnderTheEntryPoll(t *testing.T, store *feedtest.Store, onSignal func(eventfeed.Signal), wire func(*harness), opts ...eventfeed.Option) *harness { + t.Helper() + store.Stored("pos-0") + h := storedHarness(t, store, append([]eventfeed.Option{eventfeed.WithLiveBufferCapacity(2)}, opts...)...) + h.minter.ScriptTicket(ticket(1)) + h.polls.ScriptPage(eventfeed.PollPage{Position: "pos-1"}) + dropped := make(chan eventfeed.Signal, 4) + h.conn.OnSignal(func(s eventfeed.Signal) { + if onSignal != nil { + onSignal(s) + } + dropped <- s + }) + if wire != nil { + wire(h) + } + var conn *feedtest.Conn + h.polls.OnCall(func(feedtest.PollCall) { + conn.Serve(frameMessage(noFilterIdentifier, 41)) + conn.Serve(frameMessage(noFilterIdentifier, 42)) + conn.Serve(frameMessage(noFilterIdentifier, 43)) + select { + case s := <-dropped: + if _, ok := s.(eventfeed.BufferOverflow); !ok { + t.Errorf("signal = %+v, want the overflowing admission's BufferOverflow", s) + } + case <-time.After(watchdog): + t.Error("the overflowing admission was never dispatched") + } + }) + h.start() + conn = h.driveToSubscribed() + conn.Serve(frameConfirm(noFilterIdentifier)) + return h +} + +// TestOverflowUnderTheEntryPollNeverSavesUnderTheDefaultTerminal is the +// conjunctive save-ordering invariant against the one drop the walk used to +// slip past it. An admission that overflows while the page's poll is in flight +// is a pre-cut loss condition, so its disposition must run BEFORE the page's +// position moves anything durable: with no handler registered that is +// Terminal(buffer_overflow) with the checkpoint exactly where it was, or a +// restart resumes past the dropped event and never serves it. +// +// Dispatching it at a later cut is not merely late, it is unreliable: parked +// until the page boundary the drain would already have emptied the buffer, so +// the admission drops nothing and the signal never fires at all — and parked +// until the call returns, a call that stalls or fails never dispatches it. +// Dispatching where the drop is observed is what makes this invariant +// structural: a disposition that ran strictly earlier cannot run after a save. +func TestOverflowUnderTheEntryPollNeverSavesUnderTheDefaultTerminal(t *testing.T) { + store := feedtest.NewStore() + var signals []eventfeed.Signal + h := startOverflowUnderTheEntryPoll(t, store, + func(s eventfeed.Signal) { signals = append(signals, s) }, nil) + h.join() + + _, terminal, _ := h.snapshot() + if terminal == nil || terminal.Reason != eventfeed.ReasonBufferOverflow { + t.Fatalf("terminal = %v, want reason %q", terminal, eventfeed.ReasonBufferOverflow) + } + if len(signals) != 1 { + t.Fatalf("signals = %v, want exactly one BufferOverflow raised at drop time", signals) + } + ov, ok := signals[0].(eventfeed.BufferOverflow) + if !ok || ov.DroppedCount != 1 || len(ov.DroppedIDs) != 1 || ov.DroppedIDs[0] != 41 { + t.Fatalf("signal = %+v, want BufferOverflow{DroppedIDs:[41], DroppedCount:1}", signals[0]) + } + assertPositions(t, store.Saves()) + if got := fmt.Sprint(h.ledger()); got != "[]" { + t.Fatalf("ledger = %s, want nothing delivered and nothing saved", got) + } +} + +// TestOverflowUnderTheEntryPollSavesAfterTheDisposition is the same ordering +// from the accepting side: the disposition is taken first, the page's position +// saves second, and acceptance is still not license to skip the retained +// events — 42 and 43 are both delivered by the drain that follows. +func TestOverflowUnderTheEntryPollSavesAfterTheDisposition(t *testing.T) { + store := feedtest.NewStore() + // The handler runs on the consumer's goroutine, so it reaches the ledger + // through a target published BEFORE that goroutine exists — the wire hook + // runs ahead of the iteration, which is the happens-before the plain + // assign-after-construct spelling does not have. + var target *harness + h := startOverflowUnderTheEntryPoll(t, store, nil, + func(h *harness) { target = h }, + eventfeed.WithSignalHandler(func(eventfeed.Signal) eventfeed.Disposition { + target.record("overflow") + return eventfeed.Accept + })) + h.awaitStreaming() + + assertLedger(t, h.ledger(), []string{"overflow", "save pos-1", "event 42", "event 43"}) + assertPositions(t, store.Saves(), "pos-1") +} + +// TestSocketOutcomeDuringAStalledPollIsBounded is transition 21 against a +// seam call that returns only on cancellation. The socket dies while the entry +// poll is outstanding, so the outcome is deferred and the call is awaited — +// the in-flight page is still the page boundary's to accept — but the wait is +// bounded by the same staleness window every other socket-open wait is. +// Awaiting it unconditionally parks the consumer's goroutine forever behind a +// socket that has already spoken, and the staleness expiry that would tear +// that socket down is unobservable for as long as the call is in flight. +func TestSocketOutcomeDuringAStalledPollIsBounded(t *testing.T) { + store := feedtest.NewStore() + store.Stored("pos-0") + h := storedHarness(t, store) + h.minter.ScriptTicket(ticket(1)) + h.polls.StallNext() + + deferred := make(chan struct{}, 4) + h.conn.OnFrameDeferred(func() { deferred <- struct{}{} }) + errQueued := make(chan struct{}, 1) + h.conn.OnPumpHandedOff(func(isErr bool) { + if !isErr { + return + } + select { + case errQueued <- struct{}{}: + default: + } + }) + var conn *feedtest.Conn + h.polls.OnCall(func(feedtest.PollCall) { + conn.FailReads(errors.New("connection reset by peer")) + select { + case <-errQueued: + case <-time.After(watchdog): + t.Error("the pump never handed the socket failure to the state machine") + } + select { + case <-deferred: + case <-time.After(watchdog): + t.Error("the socket failure was never deferred") + } + // The window now lapses with the seam call still outstanding. + h.clock.Advance(staleAfter) + }) + h.start() + conn = h.driveToSubscribed() + conn.Serve(frameConfirm(noFilterIdentifier)) + + h.awaitTimer(timerBackoff) + if !conn.Closed() { + t.Fatal("abandoning the superseded call must dispose the attempt") + } + if _, terminal, _ := h.snapshot(); terminal != nil { + t.Fatalf("a socket failure mid-walk is never terminal; got %v", terminal) + } + assertPositions(t, store.Saves()) + assertTimers(t, h.clock, map[string]int{timerBackoff: 1}) +} + +// TestProtocolFatalDuringDrainingIsImmediate is SPEC.md §23's carve-out from +// Draining's deferred consumption: "a raw `invalid_event_stream_command` +// observed during Draining is Terminal(`protocol_fatal`) immediately — the +// drain is not completed, the held entry position is NOT saved, and no +// `caught_up` is announced; only recoverable failures defer." +// +// Both subtests are the same frame reaching Draining by the two routes it +// can: parked in the deferral slot by the entry poll's own servicing, and +// queued by the pump while the drain is mid-delivery. A drain that replays +// the buffer without ever looking at the frame queue takes the recoverable +// path in both — held save, `caught_up`, and only then the terminal. +func TestProtocolFatalDuringDrainingIsImmediate(t *testing.T) { + // A store with nothing in it makes the entry present-class, which is what + // gives the drain a HELD position to (wrongly) save. + newDrainingHarness := func(t *testing.T, caughtUp *int, opts ...eventfeed.Option) (*harness, *feedtest.Store) { + t.Helper() + store := feedtest.NewStore() + h := storedHarness(t, store, append([]eventfeed.Option{eventfeed.WithObserver(eventfeed.Observer{ + CaughtUp: func() { *caughtUp++ }, + })}, opts...)...) + h.minter.ScriptTicket(ticket(1)) + h.polls.ScriptPage(eventfeed.PollPage{Position: "pos-1"}) + return h, store + } + + t.Run("deferred by the entry poll's servicing", func(t *testing.T) { + var caughtUp int + h, store := newDrainingHarness(t, &caughtUp) + deferred := make(chan struct{}, 4) + h.conn.OnFrameDeferred(func() { deferred <- struct{}{} }) + var conn *feedtest.Conn + h.polls.OnCall(func(feedtest.PollCall) { + conn.Serve(frameDisconnect("invalid_event_stream_command", false)) + select { + case <-deferred: + case <-time.After(watchdog): + t.Error("the disconnect frame was never deferred") + } + }) + h.start() + conn = h.driveToSubscribed() + h.serveSettled(conn, frameMessage(noFilterIdentifier, 41)) + conn.Serve(frameConfirm(noFilterIdentifier)) + h.join() + + assertProtocolFatalDrain(t, h, store, caughtUp) + assertIDs(t, h.deliveredIDs()) + }) + + t.Run("queued while the drain is delivering", func(t *testing.T) { + var caughtUp int + h, store := newDrainingHarness(t, &caughtUp) + h.pauseAfter = 1 + h.start() + conn := h.driveToSubscribed() + h.serveSettled(conn, frameMessage(noFilterIdentifier, 41)) + conn.Serve(frameConfirm(noFilterIdentifier)) + + // The consumer parks inside the drain's delivery of the retained + // event; the fatal frame is queued while it is parked. + h.waitUntil("the drain parked mid-delivery", func() bool { return len(h.deliveredIDs()) == 1 }) + h.serveSettled(conn, frameDisconnect("invalid_event_stream_command", false)) + h.resume() + h.join() + + assertProtocolFatalDrain(t, h, store, caughtUp) + assertIDs(t, h.deliveredIDs(), 41) + }) + + // The carve-out is about the FRAME, not about how many ordinary frames sit + // in front of it. The scan's reach is the pump's hand-off queue depth — + // the only bound that guarantees every frame the pump had queued when the + // drain began is examined — and deliberately NOT the caller-configurable + // live-buffer capacity, which measures a different thing entirely (how + // much the state machine will hold) and can be set as low as 1. Bounding + // the scan by it lets a single queued ping spend the whole budget, and the + // fatal frame behind it goes unseen until Streaming — after the held save + // and the caught_up announcement the carve-out exists to prevent. + t.Run("behind an ordinary frame under a one-event live buffer", func(t *testing.T) { + var caughtUp int + h, store := newDrainingHarness(t, &caughtUp, eventfeed.WithLiveBufferCapacity(1)) + h.pauseAfter = 1 + h.start() + conn := h.driveToSubscribed() + h.serveSettled(conn, frameMessage(noFilterIdentifier, 41)) + conn.Serve(frameConfirm(noFilterIdentifier)) + + h.waitUntil("the drain parked mid-delivery", func() bool { return len(h.deliveredIDs()) == 1 }) + // A ping is queued AHEAD of the fatal frame: both are in the pump's + // queue before the drain's next scan, so both were "observed during + // Draining" by every reading of the carve-out. + h.serveSettled(conn, framePing(), frameDisconnect("invalid_event_stream_command", false)) + h.resume() + h.join() + + assertProtocolFatalDrain(t, h, store, caughtUp) + assertIDs(t, h.deliveredIDs(), 41) + }) +} + +// TestOverflowUnderAStalledPollDispatchesWithoutWaiting is SPEC.md §23's +// dispatch-timing rule against the one thing that can postpone a page boundary +// indefinitely: "a semantic signal is dispatched at the first consumer-context +// opportunity after its condition arises, with 'before the next save' as the +// outer bound... an implementation must not defer the signal to a later cut +// that may never come." +// +// The poll here returns only on cancellation, so there is no later cut. The +// goroutine that received the dropping frame IS the consumer's, so drop time is +// the first opportunity — and parking the disposition until the call returns +// postpones the handler forever. Worse, a call that then fails (unauthorized, +// unrecoverable, or a socket death during its retry wait) disposes the attempt +// and clears the parked deferral, so the signal is lost outright. +func TestOverflowUnderAStalledPollDispatchesWithoutWaiting(t *testing.T) { + // fill drives a position-resume walk into a stalled entry poll and then + // overflows the live buffer under it: 41 and 42 are admitted by the + // in-flight servicing, 43 cannot be, and the drop is dispatched where it is + // observed. + fill := func(t *testing.T, store *feedtest.Store, opts ...eventfeed.Option) (*harness, *feedtest.Conn) { + t.Helper() + store.Stored("pos-0") + h := storedHarness(t, store, append([]eventfeed.Option{eventfeed.WithLiveBufferCapacity(2)}, opts...)...) + h.minter.ScriptTicket(ticket(1)) + h.polls.StallNext() + h.start() + conn := h.driveToSubscribed() + conn.Serve(frameConfirm(noFilterIdentifier)) + h.waitUntil("the entry poll is outstanding", func() bool { return h.polls.CallCount() == 1 }) + conn.Serve(frameMessage(noFilterIdentifier, 41)) + conn.Serve(frameMessage(noFilterIdentifier, 42)) + conn.Serve(frameMessage(noFilterIdentifier, 43)) + return h, conn + } + + t.Run("an accepting handler is invoked while the call is outstanding", func(t *testing.T) { + store := feedtest.NewStore() + raised := make(chan eventfeed.Signal, 4) + h, _ := fill(t, store, eventfeed.WithSignalHandler(func(s eventfeed.Signal) eventfeed.Disposition { + raised <- s + return eventfeed.Accept + })) + + select { + case s := <-raised: + ov, ok := s.(eventfeed.BufferOverflow) + if !ok || ov.DroppedCount != 1 || len(ov.DroppedIDs) != 1 || ov.DroppedIDs[0] != 41 { + t.Fatalf("signal = %+v, want BufferOverflow{DroppedIDs:[41], DroppedCount:1}", s) + } + case <-time.After(watchdog): + t.Fatal("the drop never reached the handler: the stalled poll postponed the dispatch") + } + // The seam call never returned, so nothing durable can have moved — and + // an Accept keeps awaiting it, exactly as before. + if got := h.polls.CallCount(); got != 1 { + t.Fatalf("poll seam calls = %d, want the one stalled call", got) + } + assertPositions(t, store.Saves()) + }) + + t.Run("a terminating disposition does not wait for the call either", func(t *testing.T) { + store := feedtest.NewStore() + h, conn := fill(t, store, eventfeed.WithSignalHandler( + func(eventfeed.Signal) eventfeed.Disposition { return eventfeed.Terminate })) + h.join() + + _, terminal, _ := h.snapshot() + if terminal == nil || terminal.Reason != eventfeed.ReasonBufferOverflow { + t.Fatalf("terminal = %v, want reason %q", terminal, eventfeed.ReasonBufferOverflow) + } + assertPositions(t, store.Saves()) + if got := fmt.Sprint(h.ledger()); got != "[]" { + t.Fatalf("ledger = %s, want nothing delivered and nothing saved", got) + } + if !conn.Closed() { + t.Fatal("terminating on the drop must dispose the attempt, which is what returns the abandoned call") + } + assertTimers(t, h.clock, map[string]int{}) + }) +} + +// TestOverflowIsNotSwallowedByAFailingPoll is the other half of the same +// defect, and the reason the dispatch point had to move rather than gain a +// second site. A drop observed while the call is in flight, parked for a cut +// taken only after a SUCCESSFUL page, is lost outright when the call fails: +// every failure edge disposes the attempt, and disposal drops the deferral +// with the socket it belonged to. §23 is unambiguous that it cannot vanish — +// "an unhandled semantic signal cannot disappear" — so with no handler +// registered this is Terminal(buffer_overflow), not a quiet reconnect. +func TestOverflowIsNotSwallowedByAFailingPoll(t *testing.T) { + store := feedtest.NewStore() + store.Stored("pos-0") + h := storedHarness(t, store, eventfeed.WithLiveBufferCapacity(2)) + h.minter.ScriptTicket(ticket(1)) + // The call the drop is observed under fails on the edge that rides the + // reconnect cycle — the quietest of the failure edges, and the one that + // would otherwise carry on as if nothing had been dropped. + h.polls.ScriptError(&eventfeed.PollError{Kind: eventfeed.PollUnauthorized}) + var signals []eventfeed.Signal + dropped := make(chan eventfeed.Signal, 4) + h.conn.OnSignal(func(s eventfeed.Signal) { + signals = append(signals, s) + dropped <- s + }) + var conn *feedtest.Conn + h.polls.OnCall(func(feedtest.PollCall) { + conn.Serve(frameMessage(noFilterIdentifier, 41)) + conn.Serve(frameMessage(noFilterIdentifier, 42)) + conn.Serve(frameMessage(noFilterIdentifier, 43)) + select { + case <-dropped: + case <-time.After(watchdog): + t.Error("the drop was never dispatched: the failing call swallowed it") + } + }) + h.start() + conn = h.driveToSubscribed() + conn.Serve(frameConfirm(noFilterIdentifier)) + h.join() + + _, terminal, _ := h.snapshot() + if terminal == nil || terminal.Reason != eventfeed.ReasonBufferOverflow { + t.Fatalf("terminal = %v, want reason %q", terminal, eventfeed.ReasonBufferOverflow) + } + if len(signals) != 1 { + t.Fatalf("signals = %v, want exactly one BufferOverflow", signals) + } + assertPositions(t, store.Saves()) + // The drop ended the cycle before the poll's own failure could route it + // into the reconnect lane, so the feed never re-mints. + if got := h.minter.Calls(); got != 1 { + t.Fatalf("mint seam calls = %d, want 1 — the terminal is never retried into", got) + } +} + +// TestDrainScanAdmissionIsNotStranded: the drain's protocol-fatal scan admits +// correlated events as it goes, and an admission landing in an iteration that +// took the buffer EMPTY must not end the drain. Streaming never drains the +// buffer, so a stranded event waits for a later repair walk — behind +// `caught_up` and behind the held entry save this drain is what gates. +func TestDrainScanAdmissionIsNotStranded(t *testing.T) { + store := feedtest.NewStore() // Missing → present-class entry, so the save is HELD + h := storedHarness(t, store) + h.minter.ScriptTicket(ticket(1)) + h.polls.ScriptPage(eventfeed.PollPage{Position: "pos-1"}) + // Park the consumer inside the drain's delivery of the one retained event, + // so the next iteration takes an empty buffer with a frame queued behind it. + h.pauseAfter = 1 + h.start() + + conn := h.driveToSubscribed() + h.serveSettled(conn, frameMessage(noFilterIdentifier, 41)) + conn.Serve(frameConfirm(noFilterIdentifier)) + h.waitUntil("the drain parked mid-delivery", func() bool { return len(h.deliveredIDs()) == 1 }) + h.serveSettled(conn, frameMessage(noFilterIdentifier, 42)) + h.resume() + h.awaitStreaming() + + assertLedger(t, h.ledger(), []string{"event 41", "event 42", "save pos-1"}) + assertIDs(t, h.deliveredIDs(), 41, 42) + assertPositions(t, store.Saves(), "pos-1") +} + +// TestDrainHoldsNoMoreThanTheLiveBufferCapacity: the live buffer's capacity +// is a bound on events HELD AT ONCE — SPEC §23 sizes the connector's whole +// memory ceiling off it, "(pump depth + EVENT_FEED_LIVE_BUFFER_CAPACITY) × +// EVENT_FEED_MAX_FRAME_BYTES" — so a drain must not be able to hold a batch +// outside the buffer while the buffer refills to capacity behind it. +// +// The scenario pins it exactly: the buffer is FULL at capacity when Draining +// begins, and 2× capacity more live frames are already queued in the pump, with +// no consumer progress possible in between. Every one of them passes through a +// capacity-sized buffer, so at most `capacity` may survive to delivery and the +// rest must be reported dropped, oldest-id first. +func TestDrainHoldsNoMoreThanTheLiveBufferCapacity(t *testing.T) { + const capacity = 4 + var dropped []int64 + handler := func(s eventfeed.Signal) eventfeed.Disposition { + ov, ok := s.(eventfeed.BufferOverflow) + if !ok { + t.Errorf("signal = %+v, want a BufferOverflow", s) + return eventfeed.Terminate + } + if ov.DroppedCount != len(ov.DroppedIDs) { + t.Errorf("signal = %+v: DroppedCount disagrees with DroppedIDs", ov) + } + dropped = append(dropped, ov.DroppedIDs...) + return eventfeed.Accept + } + // A position-resume entry: no ownership cut, so the ONLY pass that can + // dequeue the queued frames is the drain's own. + h := newHarness(t, + eventfeed.WithLiveBufferCapacity(capacity), + eventfeed.WithStart(eventfeed.StartAtPosition("pos-0")), + eventfeed.WithSignalHandler(handler)) + h.pauseAfter = 1 + h.minter.ScriptTicket(ticket(1)) + h.polls.ScriptPage(eventfeed.PollPage{Events: []eventfeed.Event{pollEvent(1)}, Position: "pos-1"}) + h.start() + + conn := h.driveToSubscribed() + // Fill the buffer to capacity before confirmation. + for id := int64(11); id <= 14; id++ { + conn.Serve(frameMessage(noFilterIdentifier, id)) + } + h.serveSettled(conn) + conn.Serve(frameConfirm(noFilterIdentifier)) + + // Park inside the entry page's delivery: the poll has returned and the + // drain has not begun, so the frames queued now are exactly what the + // drain's scan will find. + h.waitUntil("the entry page parked mid-delivery", func() bool { return len(h.deliveredIDs()) == 1 }) + for id := int64(21); id <= 28; id++ { + conn.Serve(frameMessage(noFilterIdentifier, id)) + } + h.serveSettled(conn) + h.resume() + h.awaitStreaming() + h.conn.Close() + h.join() + + var live []int64 + for _, id := range h.deliveredIDs() { + if id != 1 { + live = append(live, id) + } + } + if len(live) > capacity { + t.Fatalf("the drain delivered %d live-buffered events (%v) under a capacity of %d: "+ + "the taken batch escaped the buffer's accounting", len(live), live, capacity) + } + assertIDs(t, live, 25, 26, 27, 28) + assertIDs(t, dropped, 11, 12, 13, 14, 21, 22, 23, 24) +} + +// TestRedirectRefusalExposesOnlyTheLocationOrigin: the refused-redirect edge +// promises the rejected Location redacted to its ORIGIN — a hostile +// continuation's path and query are exactly what must not be echoed — so the +// terminal cannot retain the seam error as its cause. PollError.Error and +// PollError.Unwrap both reach the underlying generated error, whose text +// routinely carries the request URL in full. +func TestRedirectRefusalExposesOnlyTheLocationOrigin(t *testing.T) { + const secret = "/steal?ticket=abc123&next=%2Fadmin" + h := newHarness(t) + h.minter.ScriptTicket(ticket(1)) + h.polls.ScriptError(&eventfeed.PollError{ + Kind: eventfeed.PollRedirectRefused, + LocationOrigin: "https://attacker.example.com", + Err: errors.New("302 Location: https://attacker.example.com" + secret), + }) + h.start() + + conn := h.driveToSubscribed() + conn.Serve(frameConfirm(noFilterIdentifier)) + h.join() + + _, terminal, _ := h.snapshot() + if terminal == nil || terminal.Reason != eventfeed.ReasonInvalidContinuation { + t.Fatalf("terminal = %v, want reason %q", terminal, eventfeed.ReasonInvalidContinuation) + } + if !strings.Contains(terminal.Msg, "https://attacker.example.com") { + t.Fatalf("terminal message %q should name the refused origin", terminal.Msg) + } + // The whole rendering, not just Msg: Error walks the cause chain. + if rendered := terminal.Error(); strings.Contains(rendered, secret) { + t.Fatalf("terminal rendering %q leaks the rejected Location's path and query", rendered) + } + for err := error(terminal); err != nil; err = errors.Unwrap(err) { + if strings.Contains(err.Error(), secret) { + t.Fatalf("the terminal's cause chain leaks the rejected Location: %v", err) + } + } + if !conn.Closed() { + t.Fatal("teardown must close the socket") + } +} + +// assertProtocolFatalDrain asserts the carve-out's three consequences: the +// typed terminal, no held save, and no caught_up announcement. +func assertProtocolFatalDrain(t *testing.T, h *harness, store *feedtest.Store, caughtUp int) { + t.Helper() + _, terminal, _ := h.snapshot() + if terminal == nil || terminal.Reason != eventfeed.ReasonProtocolFatal { + t.Fatalf("terminal = %v, want reason %q", terminal, eventfeed.ReasonProtocolFatal) + } + assertPositions(t, store.Saves()) + if caughtUp != 0 { + t.Fatalf("caught_up announcements = %d, want 0 — the drain never completed", caughtUp) + } +} + +// TestSupersededPollBoundSurvivesPumpBackPressure is the second half of +// transition 21's bound, and the reason the wait no longer borrows the +// staleness verdict. +// +// Every OTHER socket-open wait in the connector keeps DRAINING the hand-off +// queue while it waits, which is what makes §23's suspension rule sound +// there: a full queue really does prove the peer is outrunning a connector +// that is still consuming. This wait is the one that deliberately stops +// consuming — a frame is already parked in the deferral slot. So the queue it +// stopped draining fills, the pump blocks in hand-off, and the suspension is +// granted against a premise that is false by construction: every firing gets +// disregarded and re-armed, forever, while a compliant PollSource that +// returns only on cancellation holds the consumer's goroutine. A peer that +// keeps sending is then enough to keep an already-dead socket's verdict from +// ever landing. +func TestSupersededPollBoundSurvivesPumpBackPressure(t *testing.T) { + store := feedtest.NewStore() + store.Stored("pos-0") + h := storedHarness(t, store) + h.minter.ScriptTicket(ticket(1)) + h.polls.StallNext() + + deferred := make(chan struct{}, 4) + h.conn.OnFrameDeferred(func() { deferred <- struct{}{} }) + blocked := make(chan struct{}, 1) + h.conn.OnPumpBlocked(func() { + select { + case blocked <- struct{}{}: + default: + } + }) + var conn *feedtest.Conn + h.polls.OnCall(func(feedtest.PollCall) { + // The socket speaks its last word while the entry poll is + // outstanding: the outcome is deferred, and the walk stops draining. + conn.Serve(frameDisconnect("remote", true)) + select { + case <-deferred: + case <-time.After(watchdog): + t.Error("the disconnect frame was never deferred") + } + // A peer that keeps sending now fills the queue nothing is draining. + for i := 0; i <= eventfeed.ExportPumpDepth; i++ { + conn.Serve(framePing()) + } + select { + case <-blocked: + case <-time.After(watchdog): + t.Error("the pump never blocked on a full hand-off queue") + } + // One window from the deferral, with the suspension in force. + h.clock.Advance(staleAfter) + }) + h.start() + conn = h.driveToSubscribed() + conn.Serve(frameConfirm(noFilterIdentifier)) + + h.awaitTimer(timerBackoff) + if !conn.Closed() { + t.Fatal("abandoning the superseded call must dispose the attempt") + } + if _, terminal, _ := h.snapshot(); terminal != nil { + t.Fatalf("a socket drop mid-walk is never terminal; got %v", terminal) + } + assertPositions(t, store.Saves()) +} + +// TestDeferredProtocolFatalOutranksAFailedPoll dispatches a deferred socket +// outcome before a poll error is classified. Transition 21's +// finish-the-page ordering exists so an accepted page's deliveries and save +// are not stranded by the socket's death — it is about a page that SUCCEEDED. +// A failed poll has no such page, and recoverPoll's terminal and re-entry +// branches dispose the attempt, which discards the deferral: the server's own +// protocol-fatal verdict is replaced by whatever the poll happened to fail +// with. +func TestDeferredProtocolFatalOutranksAFailedPoll(t *testing.T) { + store := feedtest.NewStore() + store.Stored("pos-0") + h := storedHarness(t, store) + h.minter.ScriptTicket(ticket(1)) + h.minter.ScriptTicket(ticket(2)) + h.polls.ScriptError(&eventfeed.PollError{Kind: eventfeed.PollUnauthorized}) + + deferred := make(chan struct{}, 4) + h.conn.OnFrameDeferred(func() { deferred <- struct{}{} }) + var conn *feedtest.Conn + h.polls.OnCall(func(feedtest.PollCall) { + conn.Serve(frameDisconnect("invalid_event_stream_command", false)) + select { + case <-deferred: + case <-time.After(watchdog): + t.Error("the disconnect frame was never deferred") + } + }) + h.start() + conn = h.driveToSubscribed() + conn.Serve(frameConfirm(noFilterIdentifier)) + h.join() + + _, terminal, _ := h.snapshot() + if terminal == nil || terminal.Reason != eventfeed.ReasonProtocolFatal { + t.Fatalf("terminal = %v, want reason %q — the server's verdict was already observed", terminal, eventfeed.ReasonProtocolFatal) + } + assertPositions(t, store.Saves()) +} + +// TestProtocolFatalBehindTheBlockedHandOffIsImmediate extends the carve-out's +// scan bound to the frame the pump has already READ but not yet handed off. +// +// The drain-start boundary the guarantee is stated over is "every frame the +// pump had already read", and that differs from the queue's contents by +// exactly one: the pump is a single goroutine, so at most one hand-off is in +// flight. With the queue full of ordinary frames and the fatal one held in +// that hand-off, a budget of pumpDepth spends itself on the fatal frame's +// predecessors — Go hands a blocked sender's value into the buffer during the +// very first receive, so the fatal enters at the TAIL and sits exactly one +// dequeue past the budget. The drain then completes, the held entry position +// saves and `caught_up` announces: precisely what the carve-out forbids. +func TestProtocolFatalBehindTheBlockedHandOffIsImmediate(t *testing.T) { + var caughtUp int + store := feedtest.NewStore() + h := storedHarness(t, store, eventfeed.WithObserver(eventfeed.Observer{ + CaughtUp: func() { caughtUp++ }, + })) + h.minter.ScriptTicket(ticket(1)) + h.polls.ScriptPage(eventfeed.PollPage{Position: "pos-1"}) + blocked := make(chan struct{}, 1) + h.conn.OnPumpBlocked(func() { + select { + case blocked <- struct{}{}: + default: + } + }) + h.pauseAfter = 1 + h.start() + + conn := h.driveToSubscribed() + h.serveSettled(conn, frameMessage(noFilterIdentifier, 41)) + conn.Serve(frameConfirm(noFilterIdentifier)) + + // The consumer parks inside the drain's delivery of the retained event, + // so nothing dequeues while the queue fills. + h.waitUntil("the drain parked mid-delivery", func() bool { return len(h.deliveredIDs()) == 1 }) + // The scan handles a whole queue's worth of frames in one pass, which is + // more than the harness's frame-handled channel holds; nothing waits on + // it from here, so it is drained for the rest of the test. + drained := make(chan struct{}) + go func() { + for { + select { + case <-h.handled: + case <-drained: + return + } + } + }() + t.Cleanup(func() { close(drained) }) + for i := 0; i < eventfeed.ExportPumpDepth; i++ { + conn.Serve(framePing()) + } + conn.Serve(frameDisconnect("invalid_event_stream_command", false)) + select { + case <-blocked: + case <-time.After(watchdog): + t.Fatal("the pump never blocked handing off the fatal frame") + } + h.resume() + h.join() + + assertProtocolFatalDrain(t, h, store, caughtUp) + assertIDs(t, h.deliveredIDs(), 41) +} diff --git a/go/pkg/basecamp/eventfeed/connector.go b/go/pkg/basecamp/eventfeed/connector.go new file mode 100644 index 0000000000..ee0e66ac5e --- /dev/null +++ b/go/pkg/basecamp/eventfeed/connector.go @@ -0,0 +1,478 @@ +package eventfeed + +import ( + "context" + "fmt" + "iter" + "math/rand/v2" + "net/url" + "sync" + "sync/atomic" + "time" +) + +// Connector-owned constants (SPEC.md §23 "Constants" / Appendix A's +// EVENT_FEED_* rows). Exported where they back a configurable option; +// unexported where §23 pins them. +const ( + // DefaultConfirmationDeadline is the default confirmation deadline + // (EVENT_FEED_CONFIRMATION_DEADLINE, 10s; configurable via + // WithConfirmationDeadline). + DefaultConfirmationDeadline = 10 * time.Second + // DefaultLiveBufferCapacity is the default live-buffer capacity + // (EVENT_FEED_LIVE_BUFFER_CAPACITY, 10,000 events; configurable via + // WithLiveBufferCapacity; deliberately decoupled from the dedupe + // capacity — only event-bearing frames are buffered). + DefaultLiveBufferCapacity = 10_000 + + // handshakeDeadline (EVENT_FEED_HANDSHAKE_DEADLINE, 10s) spans + // dial-to-welcome: it is armed on entry to Connecting, before dial, so a + // stalled TCP connect or HTTP upgrade cannot hang the connector. + handshakeDeadline = 10 * time.Second + // defaultStaleAfter (EVENT_FEED_STALE_AFTER, 7500ms) is the SDK-pinned + // staleness detection policy: two missed 3-second server heartbeats plus + // 25% grace. Not a public option; the conformance driver overrides it + // in-package. + defaultStaleAfter = 7500 * time.Millisecond + // authFailureThreshold (EVENT_FEED_AUTH_FAILURE_THRESHOLD, 3) is the + // shared connection-level authorization counter's terminal threshold. + authFailureThreshold = 3 + // maxFrameBytes (EVENT_FEED_MAX_FRAME_BYTES, 1 MiB) bounds every inbound + // frame; the transport enforces it while reading. + maxFrameBytes = 1 << 20 + // closeCodeNormal is the WebSocket normal-closure status the connector + // sends on every client-initiated close. + closeCodeNormal = 1000 +) + +// Start selects the feed's entry mode (SPEC.md §23 "Options and Per-Language +// Naming"). The zero Start is StartResume. +type Start struct { + kind startKind + eventID int64 + position string +} + +type startKind int + +const ( + startResume startKind = iota + startPresent + startBeginning + startAfter + startAtPosition +) + +// StartResume is the default entry mode: the stored position if any, else +// the present. It is the ONLY mode a configured checkpoint store positions — +// the three explicit modes below enter where they say they do, however full +// the store is, which is what makes a checkpointed feed replayable and +// resettable. A store still loads under every mode, and the run's first +// accepted page saves under the same key, so an explicit mode repoints the +// lineage rather than forking one. +func StartResume() Start { return Start{kind: startResume} } + +// StartPresent enters at the present (since=now), whatever the store holds. +func StartPresent() Start { return Start{kind: startPresent} } + +// StartBeginning enters at the beginning of served history (since=0), +// whatever the store holds. +func StartBeginning() Start { return Start{kind: startBeginning} } + +// StartAfter enters just after the given event id (since=), whatever the +// store holds. +func StartAfter(eventID int64) Start { return Start{kind: startAfter, eventID: eventID} } + +// StartAtPosition enters at an explicit position token. Mutually exclusive +// with a checkpoint store: an explicit position and a stored lineage would +// contradict each other at entry. +func StartAtPosition(position string) Start { + return Start{kind: startAtPosition, position: position} +} + +// config is the connector's validated construction-time configuration. +type config struct { + // origin is the canonicalized API base origin (CanonicalOrigin). One + // input, two consumers: it is CheckpointKey.Origin, and it is the + // same-origin reference every continuation and resume URL is validated + // against before an authenticated follow. + origin string + accountID string + minter TicketMinter + polls PollSource + filters Filters + start Start + transport CableTransport + clock Clock + store CheckpointStore + consumerNamespace string + confirmationDeadline time.Duration + repairInterval time.Duration + dedupeCapacity int + liveBufferCapacity int + handler SignalHandler + observer Observer + + // staleAfter is the staleness window — SPEC-pinned at 7500ms, so not a + // public option; the tier-2 conformance driver overrides it in-package + // (its scenarios' stalenessMs config). + staleAfter time.Duration + // rand is the uniform [0, 1) source behind the jitter draws — white-box + // overridable in tests, no public option. + rand func() float64 +} + +// Option configures a Connector, functional-options house style (SPEC.md §23 +// "Options and Per-Language Naming" is the cross-language mapping table). +type Option func(*config) + +// WithFilters narrows the feed to the given filters. Positions are +// filter-bound: changing filters starts a new checkpoint lineage. The three +// slices are snapshotted into connector-owned storage, so a caller may reuse +// or mutate its own arrays afterwards without reaching the subscription, the +// polls, or the checkpoint key (Filters.clone). +func WithFilters(f Filters) Option { return func(c *config) { c.filters = f.clone() } } + +// WithStart selects the entry mode (default StartResume). +func WithStart(s Start) Option { return func(c *config) { c.start = s } } + +// WithTransport replaces the default WebSocket transport — product seam, the +// documented extension point for custom WebSocket stacks. +func WithTransport(t CableTransport) Option { return func(c *config) { c.transport = t } } + +// WithClock replaces the system clock — product seam, the extension point +// for embedded runtimes and deterministic tests. +func WithClock(cl Clock) Option { return func(c *config) { c.clock = cl } } + +// WithCheckpointStore configures durable position persistence (default: +// none). A configured store requires WithConsumerNamespace. +func WithCheckpointStore(s CheckpointStore) Option { return func(c *config) { c.store = s } } + +// WithConsumerNamespace names this consumer's checkpoint lineage — required +// whenever a store is configured (two independent consumers in one account +// must not share a lineage). +func WithConsumerNamespace(ns string) Option { return func(c *config) { c.consumerNamespace = ns } } + +// WithConfirmationDeadline overrides the confirmation deadline (default 10s). +func WithConfirmationDeadline(d time.Duration) Option { + return func(c *config) { c.confirmationDeadline = d } +} + +// WithRepairInterval overrides the repair-poll cadence (default 60s; ±20% +// jitter is applied per cycle either way). +func WithRepairInterval(d time.Duration) Option { return func(c *config) { c.repairInterval = d } } + +// WithDedupeCapacity overrides the delivered-id LRU capacity (default +// 10,000; must be positive — there is no dedupe-disabled mode). +func WithDedupeCapacity(n int) Option { return func(c *config) { c.dedupeCapacity = n } } + +// WithLiveBufferCapacity overrides the live-buffer capacity (default 10,000 +// events; must be positive). +func WithLiveBufferCapacity(n int) Option { return func(c *config) { c.liveBufferCapacity = n } } + +// WithSignalHandler registers the semantic-signal handler. With none +// registered, every semantic signal is terminal (SPEC.md §23 "Semantic +// Signals"). +func WithSignalHandler(h SignalHandler) Option { return func(c *config) { c.handler = h } } + +// WithObserver registers lifecycle observability callbacks. +func WithObserver(o Observer) Option { return func(c *config) { c.observer = o } } + +// Connector is the SPEC.md §23 Event Feed connector. Construct with New, +// consume with Events (single-shot), stop with Close. +type Connector struct { + cfg config + consumed atomic.Bool + hooks testHooks + + // mu guards the close latch and the active run's cancellation, which have + // to move together: close() is a universal edge from every non-absorbing + // state, so cancellation must be VISIBLE to the run before Close returns + // (see Close). The lock is held only across those two field accesses — + // never across a seam call, a yield, or an observer callback — so a Close + // taken from inside any of them cannot contend with the run goroutine. + mu sync.Mutex + // isClosed latches Close, so a run that starts afterwards cancels itself + // immediately rather than making one wire attempt first. + isClosed bool + // cancelRun cancels the active run's context. Registered by Events for + // exactly the span of one iteration, nil otherwise. + cancelRun context.CancelFunc +} + +// testHooks are unexported in-package observation points for the tier-2/3 +// harnesses (rendezvous without wall-clock sleeps). Production code never +// sets them. +type testHooks struct { + stateChanged func(connState) + frameHandled func(frameKind) + catchUpEntered func(catchUpHandoff) + // bufferOccupancy fires after every change to the state-machine-owned + // live buffer's occupancy (events admitted minus events dropped) — the + // tier-2 `expectBuffered` rendezvous. + bufferOccupancy func(int) + // signalRaised fires when a semantic signal is raised, before its + // disposition is taken. It is the tier-2 `expectSignal` rendezvous, and it + // is deliberately independent of handler registration: the default-terminal + // fixtures assert the signal's exact payload with no handler installed, + // which Observer.BufferOverflow (a count only) cannot carry. + signalRaised func(Signal) + // pumpBlocked fires when a frame pump hand-off finds the queue full — the + // rendezvous for the staleness suspension rule's precondition. + pumpBlocked func() + // pumpReleased fires once a blocked hand-off completes AND its release + // has re-armed the staleness window unsuspended. It is the rendezvous for + // "the suspension has lifted": the state machine dequeuing the frame is + // observable well before the pump's release runs, and only the release + // settles which window a subsequent expiry is evaluated against. + pumpReleased func() + // pumpHandedOff fires once an item reaches the hand-off queue, reporting + // whether it is the pump's terminating error. + pumpHandedOff func(isErr bool) + // frameDeferred fires when the in-flight-poll servicing parks one receive — + // always a socket outcome — for the walk's next dispatch point. It is the + // rendezvous for "the seam call is now the only thing this walk is waiting + // on", which is what makes transition 21's deferral assertable without + // racing the seam call's own completion. + frameDeferred func() + // subscribeWritten fires once the subscribe command has been written and + // BEFORE the phase deadline is stopped. It exists for the one ordering + // this package cannot otherwise assert: the deadline expiring in the + // window between a successful write and the Stop that cancels it — + // externally a coin flip between two ready select cases, and the reason + // Stop's result must be read rather than discarded. + subscribeWritten func() + // runContext fires with the active run's context as soon as it exists. + // Close's guarantee is a statement ABOUT that context, so a test needs the + // context itself; every in-struct proxy for it (a cleared cancel func, the + // close latch) is set by a broken Close too, and asserting on one would be + // vacuous. + runContext func(context.Context) +} + +// New validates configuration and returns a Connector. New does no I/O — +// first I/O happens on the first Events iteration — and every validation +// failure is a usage-coded *TerminalError construction error with zero wire +// attempts (SPEC.md §23 "Consumer Surface"). +// +// origin is the API base origin this feed is bound to (e.g. +// "https://3.basecampapi.com"), required and canonicalized with +// CanonicalOrigin. §23 gives it two consumers that must agree: it is +// CheckpointKey.Origin — the SDK supports configurable base URLs, so a +// position's lineage is origin-scoped — and it is the same-origin reference +// for the §8 validation every `next` continuation and 410 resume URL passes +// before an authenticated follow. It is a constructor input rather than an +// option precisely because both of those are unconditional; the Layer-1 +// adapter that builds the minter and poll source over the generated +// operations knows it already. +func New(origin, accountID string, minter TicketMinter, polls PollSource, opts ...Option) (*Connector, error) { + cfg := config{ + origin: origin, + accountID: accountID, + minter: minter, + polls: polls, + confirmationDeadline: DefaultConfirmationDeadline, + repairInterval: DefaultRepairInterval, + dedupeCapacity: DefaultDedupeCapacity, + liveBufferCapacity: DefaultLiveBufferCapacity, + staleAfter: defaultStaleAfter, + rand: rand.Float64, + } + for _, opt := range opts { + opt(&cfg) + } + if cfg.transport == nil { + cfg.transport = &WebSocketTransport{} + } + if cfg.clock == nil { + cfg.clock = SystemClock() + } + if err := validateConfig(&cfg); err != nil { + return nil, err + } + return &Connector{cfg: cfg}, nil +} + +// validateConfig applies §23's construction-time validation, fail-closed. It +// also canonicalizes the base origin in place: the canonical form is what +// both the checkpoint key and continuation validation compare against. +func validateConfig(cfg *config) error { + if cfg.origin == "" { + return usageError("an API base origin is required") + } + canonical, err := CanonicalOrigin(cfg.origin) + if err != nil { + return usageError(err.Error()) + } + if err := checkOriginScheme(canonical); err != nil { + return err + } + if err := checkIdentityText("the API base origin", canonical); err != nil { + return err + } + cfg.origin = canonical + if cfg.accountID == "" { + return usageError("accountID must be non-empty") + } + if err := checkIdentityText("accountID", cfg.accountID); err != nil { + return err + } + if err := checkIdentityText("the consumer namespace", cfg.consumerNamespace); err != nil { + return err + } + if cfg.minter == nil { + return usageError("a TicketMinter is required") + } + if cfg.polls == nil { + return usageError("a PollSource is required") + } + if err := cfg.filters.Validate(); err != nil { + return err + } + if cfg.dedupeCapacity <= 0 { + return usageError(fmt.Sprintf("dedupe capacity must be positive, got %d", cfg.dedupeCapacity)) + } + if cfg.liveBufferCapacity <= 0 { + return usageError(fmt.Sprintf("live buffer capacity must be positive, got %d", cfg.liveBufferCapacity)) + } + if cfg.confirmationDeadline <= 0 { + return usageError("confirmation deadline must be positive") + } + if cfg.repairInterval <= 0 { + return usageError("repair interval must be positive") + } + switch cfg.start.kind { + case startResume, startPresent, startBeginning: + // No mode-specific validation. + case startAfter: + if cfg.start.eventID <= 0 { + return usageError(fmt.Sprintf("StartAfter requires a positive event id, got %d", cfg.start.eventID)) + } + case startAtPosition: + if cfg.start.position == "" { + return usageError("StartAtPosition requires a non-empty position") + } + if cfg.store != nil { + return usageError("an explicit start position and a checkpoint store are mutually exclusive") + } + } + if cfg.store != nil && cfg.consumerNamespace == "" { + return usageError("a checkpoint store requires a consumer namespace") + } + return nil +} + +// checkOriginScheme applies §23's Security Invariants to the configured base +// origin: https:// everywhere, http:// only for the §9 localhost/loopback +// carve-out — the same line checkCableURL draws for the cable URL (wss:// +// with the same carve-out, via the same isLoopbackHost test) and the main Go +// client draws for its base URL. The origin is not merely where requests go: +// it is the trust anchor §8's same-origin algorithm validates every `next` +// continuation and 410 resume URL against, so a cleartext origin would make a +// cleartext continuation same-origin-valid and hand it to the authenticated +// poll seam. Its input is the canonical form, already lowercased with the +// default port stripped. +func checkOriginScheme(canonical string) error { + u, err := url.Parse(canonical) + if err != nil { + return usageError(fmt.Sprintf("unparseable origin %q", canonical)) + } + switch u.Scheme { + case "https": + return nil + case "http": + if isLoopbackHost(u.Hostname()) { + return nil + } + return usageError("the API base origin must use https (http is allowed only for localhost), got " + canonical) + default: + // The scheme is structural, never secret — safe to name. + return usageError("the API base origin scheme " + `"` + u.Scheme + `"` + " is not http(s)") + } +} + +// Events returns the feed as a serial, deduplicated event sequence. It is +// single-shot: a second consumption yields exactly one usage-terminal error +// element. A terminal condition yields exactly one final (Event{}, error) +// element; cancellation, Close, and a consumer break end iteration with no +// error element — a clean stop, the feed is resumable by design. +func (c *Connector) Events(ctx context.Context) iter.Seq2[Event, error] { + return func(yield func(Event, error) bool) { + if !c.consumed.CompareAndSwap(false, true) { + yield(Event{}, &TerminalError{ + Reason: ReasonUsage, + Msg: "Events is single-shot: this connector was already consumed", + }) + return + } + runCtx, cancel := context.WithCancel(ctx) + defer cancel() + // Register this run's cancellation under the same lock Close latches + // under, so the two cannot straddle each other: a Close that already + // latched cancels here, and one that latches later finds the func and + // cancels it itself. Either way runCtx is done before Close returns — + // there is no window in which the run proceeds past a returned Close. + c.mu.Lock() + if c.isClosed { + // Closed before the first iteration: end cleanly with zero wire + // attempts, deterministically. + cancel() + } else { + c.cancelRun = cancel + } + c.mu.Unlock() + // Fired only once the cancellation is REGISTERED: before that point + // the run is protected by the isClosed latch checked just above, not + // by the cancel func, so a context handed out earlier would invite an + // assertion about a window the latch already covers. + if c.hooks.runContext != nil { + c.hooks.runContext(runCtx) + } + defer func() { + c.mu.Lock() + c.cancelRun = nil + c.mu.Unlock() + }() + newLoop(runCtx, &c.cfg, c.hooks).run(yield) + } +} + +// Close stops the feed: it abandons, never drains (undelivered buffered +// events are re-served from the last usable checkpoint on the next run). +// Idempotent and safe from any goroutine, including the iteration goroutine +// itself. The iterator ends with no error element. +// +// Cancellation is SYNCHRONOUS with the return: an active run's context is +// cancelled here, on the caller's goroutine, so no seam call (mint, dial, +// poll) and no delivery can BEGIN after Close has returned. §23 makes +// close() a universal edge from every non-absorbing state, and delegating the +// cancel to an independently scheduled goroutine would leave the edge merely +// eventual — a Close taken inside Observer.Connecting would return, and the +// mint one statement later would still go out. +// +// What Close deliberately does NOT do is wait for the run goroutine to exit. +// Close is callable from inside a consumer callback — an observer, a signal +// handler, the iteration's own loop body — every one of which runs ON that +// goroutine, so waiting for it would deadlock on the caller. Cancellation +// visible before the return is the guarantee; the run then unwinds through +// its own dispatch points, closing the socket and joining the pump. +// The cancellation fires while callers are still SERIALIZED, which is what +// makes the guarantee hold for every caller rather than only the first. +// Clearing the field and cancelling outside the lock leaves a window: the +// first Close can be descheduled after unlocking and before invoking cancel, +// and a concurrent second Close then observes nil, takes no action and +// returns — with the run context still live. Its caller has been told the feed +// is closed while a delivery or a seam call can still begin, which is the one +// thing Close promises does not happen. Holding the mutex across the cancel is +// safe: a CancelFunc runs no consumer code, and any context.AfterFunc it +// triggers runs on its own goroutine, so it cannot re-enter Close. +func (c *Connector) Close() error { + c.mu.Lock() + defer c.mu.Unlock() + c.isClosed = true + if c.cancelRun != nil { + c.cancelRun() + c.cancelRun = nil + } + return nil +} diff --git a/go/pkg/basecamp/eventfeed/connector_test.go b/go/pkg/basecamp/eventfeed/connector_test.go new file mode 100644 index 0000000000..49677c8d08 --- /dev/null +++ b/go/pkg/basecamp/eventfeed/connector_test.go @@ -0,0 +1,622 @@ +// Construction-time validation and consumer-surface tests for the connector +// (SPEC.md §23 "Consumer Surface"): usage-coded construction errors with +// zero wire attempts, no I/O in New, single-shot Events, idempotent Close. +package eventfeed_test + +import ( + "context" + "errors" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/basecamp/basecamp-sdk/go/pkg/basecamp/eventfeed" + "github.com/basecamp/basecamp-sdk/go/pkg/basecamp/eventfeed/feedtest" +) + +// nopStore is a CheckpointStore stub for construction tests. +type nopStore struct{} + +func (nopStore) Load(context.Context, eventfeed.CheckpointKey) (string, bool, error) { + return "", false, nil +} + +func (nopStore) Save(context.Context, eventfeed.CheckpointKey, string) error { return nil } + +func TestNewValidation(t *testing.T) { + minter := feedtest.NewMinter() + polls := feedtest.NewPolls() + longIDs := make([]int64, 101) + for i := range longIDs { + longIDs[i] = int64(i + 1) + } + + cases := []struct { + name string + origin string + accountID string + minter eventfeed.TicketMinter + polls eventfeed.PollSource + opts []eventfeed.Option + wantMsg string + }{ + {"empty origin", "", "1", minter, polls, nil, "base origin"}, + {"unparseable origin", "://nope", "1", minter, polls, nil, "origin"}, + {"origin without a host", "https://", "1", minter, polls, nil, "scheme and host"}, + {"empty account id", testOrigin, "", minter, polls, nil, "accountID"}, + {"nil minter", testOrigin, "1", nil, polls, nil, "TicketMinter"}, + {"nil poll source", testOrigin, "1", minter, nil, nil, "PollSource"}, + {"filter type with whitespace", testOrigin, "1", minter, polls, + []eventfeed.Option{eventfeed.WithFilters(eventfeed.Filters{Types: []string{"a b"}})}, + "whitespace"}, + {"filter list over the cap", testOrigin, "1", minter, polls, + []eventfeed.Option{eventfeed.WithFilters(eventfeed.Filters{Buckets: longIDs})}, + "at most 100"}, + {"non-positive filter id", testOrigin, "1", minter, polls, + []eventfeed.Option{eventfeed.WithFilters(eventfeed.Filters{Creators: []int64{0}})}, + "positive"}, + {"zero dedupe capacity", testOrigin, "1", minter, polls, + []eventfeed.Option{eventfeed.WithDedupeCapacity(0)}, + "dedupe capacity"}, + {"negative live buffer capacity", testOrigin, "1", minter, polls, + []eventfeed.Option{eventfeed.WithLiveBufferCapacity(-1)}, + "live buffer capacity"}, + {"non-positive confirmation deadline", testOrigin, "1", minter, polls, + []eventfeed.Option{eventfeed.WithConfirmationDeadline(0)}, + "confirmation deadline"}, + {"non-positive repair interval", testOrigin, "1", minter, polls, + []eventfeed.Option{eventfeed.WithRepairInterval(-time.Second)}, + "repair interval"}, + {"store without a consumer namespace", testOrigin, "1", minter, polls, + []eventfeed.Option{eventfeed.WithCheckpointStore(nopStore{})}, + "consumer namespace"}, + {"start after a non-positive id", testOrigin, "1", minter, polls, + []eventfeed.Option{eventfeed.WithStart(eventfeed.StartAfter(0))}, + "positive event id"}, + {"start at an empty position", testOrigin, "1", minter, polls, + []eventfeed.Option{eventfeed.WithStart(eventfeed.StartAtPosition(""))}, + "non-empty position"}, + {"explicit position with a store", testOrigin, "1", minter, polls, + []eventfeed.Option{ + eventfeed.WithStart(eventfeed.StartAtPosition("pos-1")), + eventfeed.WithCheckpointStore(nopStore{}), + eventfeed.WithConsumerNamespace("agent"), + }, + "mutually exclusive"}, + // SPEC §9's transport-security rule (and §23's "the configured base + // origin is the trust anchor every continuation is validated + // against"): cleartext is the localhost carve-out only. + {"cleartext non-loopback origin", "http://api.example", "1", minter, polls, nil, "https"}, + {"cleartext non-loopback origin with a port", "http://api.example:8080", "1", minter, polls, nil, "https"}, + {"non-http origin scheme", "wss://cable.example.com", "1", minter, polls, nil, "http(s)"}, + // Checkpoint-identity components must be valid UTF-8: the identity + // encoding is one-to-one only over valid UTF-8 (checkpoint.go). + {"invalid utf-8 account id", testOrigin, "\xff", minter, polls, nil, "UTF-8"}, + {"invalid utf-8 consumer namespace", testOrigin, "1", minter, polls, + []eventfeed.Option{ + eventfeed.WithCheckpointStore(nopStore{}), + eventfeed.WithConsumerNamespace("\xff"), + }, + "UTF-8"}, + {"invalid utf-8 filter type", testOrigin, "1", minter, polls, + []eventfeed.Option{eventfeed.WithFilters(eventfeed.Filters{Types: []string{"message.\xff"}})}, + "UTF-8"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + c, err := eventfeed.New(tc.origin, tc.accountID, tc.minter, tc.polls, tc.opts...) + if err == nil { + t.Fatalf("New succeeded (%v), want a usage-coded construction error", c) + } + var te *eventfeed.TerminalError + if !errors.As(err, &te) || te.Reason != eventfeed.ReasonUsage { + t.Fatalf("error = %v, want *TerminalError with reason %q", err, eventfeed.ReasonUsage) + } + if !strings.Contains(err.Error(), tc.wantMsg) { + t.Fatalf("error %q does not mention %q", err, tc.wantMsg) + } + }) + } +} + +func TestNewValidConfigurations(t *testing.T) { + minter := feedtest.NewMinter() + polls := feedtest.NewPolls() + + if _, err := eventfeed.New(testOrigin, "1", minter, polls); err != nil { + t.Fatalf("minimal New: %v", err) + } + _, err := eventfeed.New(testOrigin, "1", minter, polls, + eventfeed.WithFilters(eventfeed.Filters{Types: []string{"message.created"}}), + eventfeed.WithStart(eventfeed.StartPresent()), + eventfeed.WithTransport(feedtest.NewTransport()), + eventfeed.WithClock(feedtest.NewClock()), + eventfeed.WithCheckpointStore(nopStore{}), + eventfeed.WithConsumerNamespace("agent"), + eventfeed.WithConfirmationDeadline(5*time.Second), + eventfeed.WithRepairInterval(30*time.Second), + eventfeed.WithDedupeCapacity(100), + eventfeed.WithLiveBufferCapacity(100), + eventfeed.WithSignalHandler(func(eventfeed.Signal) eventfeed.Disposition { return eventfeed.Accept }), + eventfeed.WithObserver(eventfeed.Observer{}), + ) + if err != nil { + t.Fatalf("fully-optioned New: %v", err) + } +} + +// TestNewAcceptsTheLoopbackCarveOut: cleartext is permitted for exactly the +// hosts checkCableURL permits ws:// for — SPEC §9's localhost/loopback +// carve-out, same helper, same rule. +func TestNewAcceptsTheLoopbackCarveOut(t *testing.T) { + for _, origin := range []string{ + "https://api.example", + "http://localhost:3000", + "http://127.0.0.1:9292", + "http://[::1]:3000", + "http://app.localhost", + "HTTP://LOCALHOST:3000", + } { + if _, err := eventfeed.New(origin, "1", feedtest.NewMinter(), feedtest.NewPolls()); err != nil { + t.Errorf("New(%q): %v", origin, err) + } + } +} + +// TestNewRejectsIndistinguishableConsumerNamespaces: 0xff and 0xfe are +// distinct namespaces that the identity encoding renders identically (its +// rune-wise JSON escaping maps every invalid byte to U+FFFD), so two +// independent consumers would otherwise share — and overwrite — one +// checkpoint lineage, each skipping the other's events. Construction refuses +// them rather than silently mangling. +func TestNewRejectsIndistinguishableConsumerNamespaces(t *testing.T) { + for _, ns := range []string{"\xff", "\xfe", "agent\xc3(1"} { + _, err := eventfeed.New(testOrigin, "1", feedtest.NewMinter(), feedtest.NewPolls(), + eventfeed.WithCheckpointStore(nopStore{}), eventfeed.WithConsumerNamespace(ns)) + if err == nil { + t.Fatalf("New(namespace %q) succeeded, want a usage-coded construction error", ns) + } + var te *eventfeed.TerminalError + if !errors.As(err, &te) || te.Reason != eventfeed.ReasonUsage { + t.Fatalf("New(namespace %q) error = %v, want *TerminalError with reason %q", ns, err, eventfeed.ReasonUsage) + } + } +} + +// TestNewDoesNoIO: construction is validation only — the first wire attempt +// happens on the first iteration. +func TestNewDoesNoIO(t *testing.T) { + minter := feedtest.NewMinter() + tr := feedtest.NewTransport() + if _, err := eventfeed.New(testOrigin, "1", minter, feedtest.NewPolls(), + eventfeed.WithTransport(tr), eventfeed.WithClock(feedtest.NewClock())); err != nil { + t.Fatal(err) + } + if minter.Calls() != 0 || len(tr.Dials()) != 0 { + t.Fatalf("New performed I/O: %d mints, %d dials", minter.Calls(), len(tr.Dials())) + } +} + +// TestEventsSingleShot: a second consumption yields exactly one +// usage-terminal error element. +func TestEventsSingleShot(t *testing.T) { + h := newHarness(t) + h.minter.StallNext() + h.start() + h.waitUntil("mint in flight", func() bool { return h.minter.Calls() == 1 }) + h.conn.Close() + h.join() + if _, terminal, elements := h.snapshot(); terminal != nil || elements != 0 { + t.Fatalf("first consumption should end cleanly; got %d elements, terminal %v", elements, terminal) + } + + var second []error + for _, err := range h.conn.Events(context.Background()) { + second = append(second, err) + } + if len(second) != 1 { + t.Fatalf("second consumption yielded %d elements, want exactly 1", len(second)) + } + var te *eventfeed.TerminalError + if !errors.As(second[0], &te) || te.Reason != eventfeed.ReasonUsage { + t.Fatalf("second consumption error = %v, want reason %q", second[0], eventfeed.ReasonUsage) + } +} + +// TestCloseIdempotent: Close is idempotent from any goroutine, and closing +// before the first iteration ends it immediately with no wire attempts. +func TestCloseIdempotent(t *testing.T) { + h := newHarness(t) + if err := h.conn.Close(); err != nil { + t.Fatalf("Close: %v", err) + } + if err := h.conn.Close(); err != nil { + t.Fatalf("second Close: %v", err) + } + h.start() + h.join() + if _, terminal, elements := h.snapshot(); terminal != nil || elements != 0 { + t.Fatalf("pre-closed iteration should end cleanly; got %d elements, terminal %v", elements, terminal) + } +} + +// TestCloseFromAnObserverCancelsBeforeItReturns: SPEC §23 makes close() a +// universal edge from EVERY non-absorbing state, so the active run must +// observe cancellation before Close returns — not when some independently +// scheduled goroutine happens to run. Observer.Connecting is the tightest +// statement of the rule: it fires on the iteration goroutine itself, one +// statement before the mint seam call, so a Close taken inside it must stop +// the mint that was about to happen. It must also not deadlock — Close is +// waiting on nothing, least of all on the goroutine that called it. +func TestCloseFromAnObserverCancelsBeforeItReturns(t *testing.T) { + const runs = 200 + for i := range runs { + minter := feedtest.NewMinter() + tr := feedtest.NewTransport() + var c *eventfeed.Connector + var err error + c, err = eventfeed.New(testOrigin, "1", minter, feedtest.NewPolls(), + eventfeed.WithTransport(tr), + eventfeed.WithClock(feedtest.NewClock()), + eventfeed.WithObserver(eventfeed.Observer{ + Connecting: func(int, time.Duration) { + if cerr := c.Close(); cerr != nil { + t.Fatalf("run %d: Close: %v", i, cerr) + } + }, + })) + if err != nil { + t.Fatalf("run %d: New: %v", i, err) + } + // Scripted so an un-cancelled mint SUCCEEDS: the assertion must be + // that the seam was never called, not that it happened to fail. + minter.ScriptTicket(ticket(1)) + elements := 0 + for range c.Events(context.Background()) { + elements++ + } + if minter.Calls() != 0 { + t.Fatalf("run %d: %d mint seam call(s) after Close returned, want 0", i, minter.Calls()) + } + if got := len(tr.Dials()); got != 0 { + t.Fatalf("run %d: %d dial(s) after Close returned, want 0", i, got) + } + if elements != 0 { + t.Fatalf("run %d: iteration yielded %d element(s), want none (the Closed edge)", i, elements) + } + } +} + +// TestCloseStopsDeliveryAlreadyUnderway: the same universal edge, stated over +// deliveries rather than seam calls. The consumer is parked inside the loop +// body — the state machine is mid-page, with two more rows to yield — when +// Close is called from another goroutine. Nothing may be delivered after it +// returns; the run abandons the rest of the page and ends with no error +// element (the page is re-served from the last usable checkpoint next run). +func TestCloseStopsDeliveryAlreadyUnderway(t *testing.T) { + h := newHarness(t) + h.pauseAfter = 1 + h.minter.ScriptTicket(ticket(1)) + h.polls.ScriptPage(eventfeed.PollPage{ + Events: []eventfeed.Event{pollEvent(101), pollEvent(102), pollEvent(103)}, + Position: "pos-1", + }) + h.start() + + conn := h.driveToSubscribed() + conn.Serve(frameConfirm(noFilterIdentifier)) + h.waitUntil("the page's first row was delivered", func() bool { return len(h.deliveredIDs()) == 1 }) + if err := h.conn.Close(); err != nil { + t.Fatalf("Close: %v", err) + } + h.resume() + h.join() + + assertIDs(t, h.deliveredIDs(), 101) + if _, terminal, elements := h.snapshot(); terminal != nil || elements != 1 { + t.Fatalf("iteration yielded %d element(s), terminal %v; want exactly the one pre-Close delivery", + elements, terminal) + } +} + +// filterMutationHarness drives one full run under caller-held filter slices, +// mutating them at the moment mutate says, and returns everything the run +// derived the filters from: the subscribe frame written to the socket, the +// filters each poll carried, and the checkpoint keys the store saw. +func filterMutationHarness(t *testing.T, original eventfeed.Filters, live eventfeed.Filters, mutate func(), fromConnected bool) (writes [][]byte, polled []eventfeed.Filters, keys []eventfeed.CheckpointKey) { + t.Helper() + store := feedtest.NewStore() + store.Stored("pos-0") + opts := []eventfeed.Option{eventfeed.WithFilters(live)} + if fromConnected { + // The review's exact scenario: the mutation lands from a lifecycle + // callback, after newLoop has already frozen the subscription + // identifier — so a retained backing array splits the subscribed + // filters from the polled and checkpointed ones. + opts = append(opts, eventfeed.WithObserver(eventfeed.Observer{Connected: mutate})) + } + h := storedHarness(t, store, opts...) + if !fromConnected { + mutate() + } + h.minter.ScriptTicket(ticket(1)) + h.polls.ScriptPage(eventfeed.PollPage{Events: []eventfeed.Event{pollEvent(101)}, Position: "pos-1"}) + h.start() + + conn := h.driveToSubscribed() + // Asserted here rather than with the rest: the identifier the connector + // subscribed under is what the confirm frame below has to correlate + // against, so a mutated subscription would otherwise surface as a + // confirmation that never arrives. + writes = conn.Writes() + if len(writes) != 1 || string(writes[0]) != string(eventfeed.ExportSubscribeFrame(original)) { + t.Fatalf("subscribe writes = %q, want exactly %q", writes, eventfeed.ExportSubscribeFrame(original)) + } + conn.Serve(frameConfirm(eventfeed.ExportSubscribeIdentifier(original))) + h.awaitStreaming() + h.conn.Close() + h.join() + + calls := h.polls.Calls() + polled = make([]eventfeed.Filters, len(calls)) + for i, c := range calls { + polled[i] = c.Filters + } + return writes, polled, append(store.Loads(), store.SaveKeys()...) +} + +// assertFiltersUnaffected asserts every filter-derived artifact of the run is +// the constructed filter set: the subscribe bytes, every poll's filters, and +// every checkpoint key's lineage. +func assertFiltersUnaffected(t *testing.T, original eventfeed.Filters, writes [][]byte, polled []eventfeed.Filters, keys []eventfeed.CheckpointKey) { + t.Helper() + want := eventfeed.ExportSubscribeIdentifier(original) + if len(writes) != 1 || string(writes[0]) != string(eventfeed.ExportSubscribeFrame(original)) { + t.Fatalf("subscribe writes = %q, want exactly %q", writes, eventfeed.ExportSubscribeFrame(original)) + } + if len(polled) == 0 { + t.Fatal("no poll was made; the run did not reach the poll lane") + } + for i, f := range polled { + if got := eventfeed.ExportSubscribeIdentifier(f); got != want { + t.Errorf("poll %d filters = %s, want %s", i, got, want) + } + } + if len(keys) == 0 { + t.Fatal("the store saw no key; the run did not reach the checkpoint lane") + } + for i, k := range keys { + if k.FilterKey != original.FilterKey() { + t.Errorf("checkpoint key %d lineage = %s, want %s", i, k.FilterKey, original.FilterKey()) + } + } +} + +// TestFiltersMutatedBeforeIterationDoNotReachTheWire: Filters carries slices, +// so a caller that keeps its backing arrays could otherwise change the +// subscription, the poll parameters and the checkpoint lineage AFTER the +// constructor validated them — including into a filter set Validate rejects. +// New snapshots them, so the run is unaffected. +func TestFiltersMutatedBeforeIterationDoNotReachTheWire(t *testing.T) { + types := []string{"message.created"} + buckets := []int64{11} + creators := []int64{22} + original := eventfeed.Filters{ + Types: []string{"message.created"}, + Buckets: []int64{11}, + Creators: []int64{22}, + } + writes, polled, keys := filterMutationHarness(t, original, + eventfeed.Filters{Types: types, Buckets: buckets, Creators: creators}, + func() { + // "a b" would not survive Filters.Validate — the fail-closed + // guarantee is only worth what the connector actually holds. + types[0] = "a b" + buckets[0] = 99 + creators[0] = 98 + }, false) + assertFiltersUnaffected(t, original, writes, polled, keys) +} + +// TestFiltersMutatedFromConnectedDoNotSplitTheLineage: the same hazard from +// Observer.Connected, after the subscription identifier is frozen. Without a +// snapshot the cable stays subscribed under the original filters while the +// polls and the checkpoint key move to the mutated ones — a position saved +// under a lineage the feed never subscribed to. +func TestFiltersMutatedFromConnectedDoNotSplitTheLineage(t *testing.T) { + types := []string{"message.created"} + buckets := []int64{11} + creators := []int64{22} + original := eventfeed.Filters{ + Types: []string{"message.created"}, + Buckets: []int64{11}, + Creators: []int64{22}, + } + writes, polled, keys := filterMutationHarness(t, original, + eventfeed.Filters{Types: types, Buckets: buckets, Creators: creators}, + func() { + types[0] = "todo.completed" + buckets[0] = 99 + creators[0] = 98 + }, true) + assertFiltersUnaffected(t, original, writes, polled, keys) +} + +// TestPollSeamCannotRepointTheLineage: the filters handed to the poll seam +// are a copy too. The seam is host code — an adapter that sorts or dedupes in +// place while building its query would otherwise mutate the connector's own +// storage mid-run, splitting the subscribed filters from the polled and +// checkpointed ones exactly as a retained caller array does. +func TestPollSeamCannotRepointTheLineage(t *testing.T) { + original := eventfeed.Filters{ + Types: []string{"message.created"}, + Buckets: []int64{11}, + Creators: []int64{22}, + } + store := feedtest.NewStore() + store.Stored("pos-0") + h := storedHarness(t, store, eventfeed.WithFilters(eventfeed.Filters{ + Types: []string{"message.created"}, + Buckets: []int64{11}, + Creators: []int64{22}, + })) + // The ENTRY poll's seam call mutates what it was handed, in place. Only + // the first: the assertion below reads the second call's filters, and the + // seam's own copy is what the fake records, so a mutation on that call + // would prove nothing either way. + var polled atomic.Int32 + h.polls.OnCall(func(c feedtest.PollCall) { + if polled.Add(1) != 1 { + return + } + c.Filters.Types[0] = "todo.completed" + c.Filters.Buckets[0] = 99 + c.Filters.Creators[0] = 98 + }) + h.minter.ScriptTicket(ticket(1)) + next := testOrigin + "/999/events.json?after=101" + h.polls.ScriptPage(eventfeed.PollPage{ + Events: []eventfeed.Event{pollEvent(101)}, + Position: "pos-1", + Next: next, + }) + h.polls.ScriptPage(eventfeed.PollPage{Events: []eventfeed.Event{pollEvent(102)}, Position: "pos-2"}) + h.start() + + conn := h.driveToSubscribed() + conn.Serve(frameConfirm(eventfeed.ExportSubscribeIdentifier(original))) + h.awaitStreaming() + h.conn.Close() + h.join() + + calls := h.polls.Calls() + if len(calls) != 2 { + t.Fatalf("poll seam calls = %d, want 2 (the entry page and its continuation)", len(calls)) + } + // The first call's record aliases what the seam mutated; the SECOND call + // is the one that proves the connector's storage survived it. + want := eventfeed.ExportSubscribeIdentifier(original) + if got := eventfeed.ExportSubscribeIdentifier(calls[1].Filters); got != want { + t.Errorf("continuation poll filters = %s, want %s", got, want) + } + for i, k := range append(store.Loads(), store.SaveKeys()...) { + if k.FilterKey != original.FilterKey() { + t.Errorf("checkpoint key %d lineage = %s, want %s", i, k.FilterKey, original.FilterKey()) + } + } +} + +// TestConcurrentCloseAllCallersSeeCancellation pins Close's guarantee for +// EVERY caller, not just the first. Events' own comment states it — "there is +// no window in which the run proceeds past a returned Close" — and clearing +// the cancellation field and invoking it outside the lock left one: the first +// caller descheduled between the unlock and the cancel, a concurrent second +// caller observing nil, taking no action, and returning while the run context +// is still live. Its caller has been told the feed is closed while a delivery +// or a seam call can still begin. +// +// It asserts on the run CONTEXT, deliberately. Every in-struct proxy for it — +// a cleared cancel func, the close latch — is set by the broken Close too, so +// asserting on one would pass against un-fixed code. +// +// The interleaving is real rather than designed, so the proof is +// probabilistic: the assertion is exact and the iteration count is sized off +// MEASURED hits rather than guessed. Against un-fixed code the window was hit +// at iterations 92, 142, 169 and 362 across four runs, so the budget is set +// several times the worst observed — a count near the worst hit is how a +// regression test quietly becomes flaky-green. +func TestConcurrentCloseAllCallersSeeCancellation(t *testing.T) { + for i := 0; i < 1500; i++ { + h := newHarness(t) + h.minter.ScriptTicket(ticket(1)) + got := make(chan context.Context, 1) + h.conn.OnRunContext(func(ctx context.Context) { + select { + case got <- ctx: + default: + } + }) + h.start() + var runCtx context.Context + select { + case runCtx = <-got: + case <-time.After(watchdog): + t.Fatal("the run context was never registered") + } + + var wg sync.WaitGroup + bad := make(chan struct{}, 4) + for g := 0; g < 4; g++ { + wg.Add(1) + go func() { + defer wg.Done() + _ = h.conn.Close() + if runCtx.Err() == nil { + select { + case bad <- struct{}{}: + default: + } + } + }() + } + wg.Wait() + select { + case <-bad: + t.Fatalf("iteration %d: Close returned with the run context still live", i) + default: + } + h.cancel() + } +} + +// emptyPositionStore is a custom CheckpointStore that reports FOUND with an +// empty position — the state the built-in FileStore classifies as a store +// failure, and which a custom store is under no obligation to. +type emptyPositionStore struct{ loads int } + +func (s *emptyPositionStore) Load(context.Context, eventfeed.CheckpointKey) (string, bool, error) { + s.loads++ + return "", true, nil +} + +func (s *emptyPositionStore) Save(context.Context, eventfeed.CheckpointKey, string) error { + return nil +} + +// TestFoundButEmptyPositionIsALoadFailure closes a SILENT history skip at the +// seam. entryCursor selects on a non-empty position, so an empty one loaded +// under StartResume falls through to the mode's default — a bare present +// entry, which is present-class — and the feed resumes at the server's head +// having skipped everything between the stored position and now, with no +// signal at all. +// +// The built-in store already refuses to return this state ("an empty position +// cannot be told apart from having none"), but that is one implementation of a +// public seam; the invariant has to hold for every store, so it is enforced +// where the load is consumed. +func TestFoundButEmptyPositionIsALoadFailure(t *testing.T) { + store := &emptyPositionStore{} + h := newHarness(t, + eventfeed.WithCheckpointStore(store), + eventfeed.WithConsumerNamespace("agent")) + h.minter.ScriptTicket(ticket(1)) + h.start() + h.join() + + _, terminal, _ := h.snapshot() + if terminal == nil || terminal.Reason != eventfeed.ReasonCheckpointLoad { + t.Fatalf("terminal = %v, want reason %q — a found-but-empty position is a store failure", terminal, eventfeed.ReasonCheckpointLoad) + } + if store.loads != 1 { + t.Errorf("store loads = %d, want 1", store.loads) + } + // Terminal BEFORE any wire attempt, like every other checkpoint_load. + if got := h.minter.Calls(); got != 0 { + t.Errorf("mint seam calls = %d, want 0 — the load fails before any wire attempt", got) + } + if got := len(h.tr.Dials()); got != 0 { + t.Errorf("dials = %d, want 0", got) + } +} diff --git a/go/pkg/basecamp/eventfeed/export_test.go b/go/pkg/basecamp/eventfeed/export_test.go new file mode 100644 index 0000000000..dda9c9619a --- /dev/null +++ b/go/pkg/basecamp/eventfeed/export_test.go @@ -0,0 +1,131 @@ +package eventfeed + +import ( + "context" + "errors" + "slices" + "time" +) + +// export_test.go bridges the external eventfeed_test package (which uses the +// feedtest fakes and therefore cannot live in-package — import cycle) into +// the connector's unexported observation points. Everything here is +// test-binary-only; nothing leaks into the shipped surface. + +// CatchUpBoundary is the external-test projection of the internal +// catchUpHandoff — the slice-4b seam a confirmed subscription hands off at. +type CatchUpBoundary struct { + // Entry is the selected entry cursor. + Entry Cursor + // PresentClass reports whether the entry resolves at the present head. + PresentClass bool + // Buffered is a snapshot of the live buffer's contents at hand-off. + Buffered []Event +} + +// OnCatchUpEntered registers a hook fired when a confirmed subscription +// reaches the catch-up boundary. +func (c *Connector) OnCatchUpEntered(f func(CatchUpBoundary)) { + c.hooks.catchUpEntered = func(h catchUpHandoff) { + f(CatchUpBoundary{ + Entry: h.entry, + PresentClass: h.presentClass, + Buffered: slices.Clone(h.buffer.events), + }) + } +} + +// OnStateChanged registers a hook receiving each state transition's fixture +// spelling. +func (c *Connector) OnStateChanged(f func(string)) { + c.hooks.stateChanged = func(s connState) { f(s.String()) } +} + +// OnFrameHandled registers a hook receiving each handled inbound frame's +// kind name, fired after the state machine dequeued and classified it. +func (c *Connector) OnFrameHandled(f func(string)) { + c.hooks.frameHandled = func(k frameKind) { f(k.String()) } +} + +// OnPumpBlocked registers a hook fired when a frame pump hand-off finds the +// bounded queue full — the moment the staleness evaluation suspends. +func (c *Connector) OnPumpBlocked(f func()) { c.hooks.pumpBlocked = f } + +// OnPumpReleased registers a hook fired once a blocked hand-off completes and +// its release has re-armed the staleness window unsuspended. It is the +// rendezvous for "the suspension has lifted" — the counterpart to +// OnPumpBlocked, and the only observation point that proves the release's +// re-arm has landed rather than still being in flight behind the dequeue. +func (c *Connector) OnPumpReleased(f func()) { c.hooks.pumpReleased = f } + +// OnPumpHandedOff registers a hook fired once an item reaches the hand-off +// queue, reporting whether it is the pump's terminating error. It is the +// rendezvous for "the socket's death is queued for the state machine". +func (c *Connector) OnPumpHandedOff(f func(isErr bool)) { c.hooks.pumpHandedOff = f } + +// OnFrameDeferred registers a hook fired when the in-flight-poll servicing +// parks one receive — always a socket outcome — for the walk's next dispatch +// point. It is the rendezvous for "the walk is now waiting on nothing but the +// seam call": without it, a test that serves the deferring frame and then lets +// the poll return races the state machine's own select between the queued frame +// and the completed call. +func (c *Connector) OnFrameDeferred(f func()) { c.hooks.frameDeferred = f } + +// ExportPumpDepth is the frame pump's bounded hand-off queue depth: the +// number of frames a test must serve to make the pump block. +const ExportPumpDepth = pumpDepth + +// SetStaleAfter overrides the staleness window — the tier-2 driver's +// stalenessMs scenario config. Deliberately not a public option: SPEC §23 +// pins 7500ms. +func (c *Connector) SetStaleAfter(d time.Duration) { c.cfg.staleAfter = d } + +// SetRand overrides the uniform [0, 1) jitter source — white-box, no public +// option. +func (c *Connector) SetRand(r func() float64) { c.cfg.rand = r } + +// OnBufferOccupancy registers a hook receiving the state-machine-owned live +// buffer's occupancy after every change (events admitted minus events +// dropped) — the tier-2 `expectBuffered` rendezvous. +func (c *Connector) OnBufferOccupancy(f func(int)) { c.hooks.bufferOccupancy = f } + +// OnSignal registers a hook receiving every semantic signal as it is raised, +// before its disposition is taken. Unlike the Observer callbacks it carries +// the signal's full payload and fires whether or not a handler is registered — +// which is what makes the default-terminal fixtures' exact `expectSignal` +// assertions (dropped ids included) observable. +func (c *Connector) OnSignal(f func(Signal)) { c.hooks.signalRaised = f } + +// ExportIsInvalidFrameError reports whether err is a §23 invalid-frame-class +// violation — the indication Observer.Disconnected carries for the tier-2 +// `expectDisconnectedInvalidFrame` rendezvous. +func ExportIsInvalidFrameError(err error) bool { + var ife *invalidFrameError + return errors.As(err, &ife) +} + +// ExportSubscribeIdentifier exposes the exact EventsChannel subscription +// identifier for frame construction in tests. +func ExportSubscribeIdentifier(f Filters) string { return subscribeIdentifier(f) } + +// ExportSubscribeFrame exposes the exact subscribe command bytes the connector +// writes. +func ExportSubscribeFrame(f Filters) []byte { return subscribeCommand(subscribeIdentifier(f)) } + +// OnSubscribeWritten registers a hook fired once the subscribe command has +// been written and BEFORE the phase deadline is stopped. It is the only +// rendezvous for the deadline-vs-welcome ordering: from outside, a deadline +// that fires while the welcome frame is already queued leaves the state +// machine's select with two ready cases and no way for a test to say which +// one it must take. +func (c *Connector) OnSubscribeWritten(f func()) { c.hooks.subscribeWritten = f } + +// OnRunContext registers a hook receiving the active run's context once its +// cancellation has been REGISTERED — the point from which Close's guarantee is +// a statement about that context. Before it, the run is held by the isClosed +// latch instead, so a context handed out earlier would invite an assertion +// about a window the latch already covers. It is the only way to observe Close's actual guarantee — +// "cancellation is visible before the return" is a statement about that +// context, and any proxy for it (a nil cancel func, a closed latch) is exactly +// the state a broken Close sets too early. +func (c *Connector) OnRunContext(f func(context.Context)) { c.hooks.runContext = f } diff --git a/go/pkg/basecamp/eventfeed/loop.go b/go/pkg/basecamp/eventfeed/loop.go new file mode 100644 index 0000000000..e4ffd9beeb --- /dev/null +++ b/go/pkg/basecamp/eventfeed/loop.go @@ -0,0 +1,1306 @@ +package eventfeed + +import ( + "context" + "errors" + "fmt" + "strconv" + "sync" + "time" +) + +// The connector run loop (SPEC.md §23 "State Machine"). The whole protocol +// executes on the consumer's goroutine — ranging Events IS the state machine +// (design decision 3) — with one auxiliary reader pump per live socket, one +// transient goroutine per in-flight poll seam call (catchup.go's pollPage, +// which carries no state: the state machine stays the only mutator), and no +// mutex-guarded connector state. This file carries Idle through +// the confirm/reject/deadline paths (§23 transitions 1–15, the universal +// Closed edge, and the out-of-inventory usage / mint_failed / +// invalid_cable_url / buffer_overflow edges), the reconnect cycle every +// continuable failure returns to, and the frame pump with the staleness +// policy it feeds; a confirmed subscription hands off at the typed +// catchUpHandoff boundary below to the catch-up walk, the entry boundary, and +// streaming, which live in catchup.go. +// +// What a reconnect preserves and what it rebuilds is a §23 distinction, not +// an implementation detail. Preserved on the loop, across every attempt: the +// in-memory position (authoritative for resume within the run — the store is +// never re-read after its one load), the highest id the poll lane has served +// (the reset cursor a 400-position or 409 re-enters at), the delivered-id +// LRU, the live buffer of admitted-but-undelivered events, and the shared +// authorization counter, which only a successful poll page resets. Rebuilt +// per attempt: a freshly minted ticket (the connector never stores a mint URL +// across attempts), the socket, the pump, the staleness holder, every timer, +// and the consecutive-poll-failure index. The failed-cycle count grows per +// failed cycle and resets on confirmation. + +// connState enumerates SPEC.md §23's 11 states. String renders the tier-2 +// fixture spelling (snake_case). +type connState int + +const ( + stateIdle connState = iota + stateBackoff + stateMinting + stateConnecting + stateAwaitingWelcome + stateAwaitingConfirmation + stateCatchingUp + stateDraining + stateStreaming + stateTerminal + stateClosed +) + +// String renders the fixture spelling of the state. +func (s connState) String() string { + switch s { + case stateIdle: + return "idle" + case stateBackoff: + return "backoff" + case stateMinting: + return "minting" + case stateConnecting: + return "connecting" + case stateAwaitingWelcome: + return "awaiting_welcome" + case stateAwaitingConfirmation: + return "awaiting_confirmation" + case stateCatchingUp: + return "catching_up" + case stateDraining: + return "draining" + case stateStreaming: + return "streaming" + case stateTerminal: + return "terminal" + case stateClosed: + return "closed" + default: + return fmt.Sprintf("connState(%d)", int(s)) + } +} + +// Timer kinds (SPEC.md §23 "Clock, Timers, and Virtual Time": six kinds, +// kebab-case; the four armed by the lifecycle are here, and `repair-poll` +// and `poll-retry` are armed by the walk, in catchup.go). +const ( + timerHandshakeDeadline = "handshake-deadline" + timerConfirmationDeadline = "confirmation-deadline" + timerBackoff = "backoff" + timerStaleness = "staleness" +) + +// Disconnect reason literals (class-1 wire literals, SPEC.md §23 "Disconnect +// Dispatch"). Dispatch is on the reason string, never the reconnect flag +// alone; `remote` and unrecognized reasons need no literal here — they take +// the socket-drop default. +const ( + disconnectReasonUnauthorized = "unauthorized" + disconnectReasonProtocolFatal = "invalid_event_stream_command" +) + +// sincePresent is the `since` value that enters at the server's present head +// (the bare entry is equivalent). Every cursor spelled with it is +// present-class (SPEC.md §23 "Entry Boundary"). +const sincePresent = "now" + +// errStaleConnection reports a staleness teardown. +var errStaleConnection = errors.New("event feed connection stale: no inbound frames within the staleness window") + +// attempt is one connect cycle's mutable ownership: its cancellation scope +// (cancelled on teardown, so an in-flight seam call or dial belonging to the +// attempt returns promptly) and, once dialed, the live connection. +type attempt struct { + ctx context.Context + cancel context.CancelFunc + lc *liveConn +} + +// pumpDepth is the frame pump's bounded hand-off queue depth (SPEC.md §23 +// "Cable Protocol Details": bounded, blocking, never dropping — the +// state-machine-owned live buffer is the only place a frame can ever be +// dropped). +const pumpDepth = 256 + +// pumpItem is one hand-off from the reader pump: a raw frame, or the read +// error that ended the pump. +type pumpItem struct { + data []byte + err error +} + +// liveConn is one live socket with its reader pump and staleness holder. +type liveConn struct { + conn CableConn + frames chan pumpItem + stale *staleHolder + hooks testHooks +} + +// newLiveConn arms staleness (socket open) and then starts the pump, in that +// order, so the timer exists before the first frame can reset it. +func newLiveConn(ctx context.Context, conn CableConn, clock Clock, staleAfter time.Duration, hooks testHooks) *liveConn { + lc := &liveConn{ + conn: conn, + frames: make(chan pumpItem, pumpDepth), + stale: newStaleHolder(clock, staleAfter), + hooks: hooks, + } + go lc.pump(ctx) + return lc +} + +// pump reads frames and hands them to the state machine over the bounded +// queue. The staleness reset happens HERE, at frame receipt — pump-side, so +// frame-vs-deadline ordering is defined at the transport boundary regardless +// of queue depth or consumer latency (SPEC.md §23 "Cable Protocol Details"). +// The pump exits by sending the terminating read error (unless the attempt +// was already cancelled) and closing the channel; the closed channel is the +// join signal disposal drains to. +func (lc *liveConn) pump(ctx context.Context) { + defer close(lc.frames) + for { + data, err := lc.conn.ReadFrame(ctx) + if err != nil { + lc.handOff(ctx, pumpItem{err: err}) + return + } + lc.stale.reset() + if !lc.handOff(ctx, pumpItem{data: data}) { + return + } + } +} + +// handOff passes one item to the state machine over the bounded queue. The +// queue never drops: at capacity the pump BLOCKS, propagating back-pressure +// through the socket to TCP (SPEC.md §23 "Cable Protocol Details" — the +// state-machine-owned live buffer is the only place a frame can be dropped). +// A blocked hand-off suspends the staleness EVALUATION for as long as it +// lasts: a full queue proves the peer is sending faster than the connector +// consumes — the opposite of a dead peer — and a pump that is not reading +// cannot observe the resets that would prove liveness, so the absence of one +// is not evidence. It reports whether the item was handed off. +func (lc *liveConn) handOff(ctx context.Context, item pumpItem) bool { + select { + case lc.frames <- item: + lc.handedOff(item) + return true + default: + } + lc.stale.suspend() + defer func() { + lc.stale.resume() + if lc.hooks.pumpReleased != nil { + lc.hooks.pumpReleased() + } + }() + if lc.hooks.pumpBlocked != nil { + lc.hooks.pumpBlocked() + } + select { + case lc.frames <- item: + lc.handedOff(item) + return true + case <-ctx.Done(): + return false + } +} + +func (lc *liveConn) handedOff(item pumpItem) { + if lc.hooks.pumpHandedOff != nil { + lc.hooks.pumpHandedOff(item.err != nil) + } +} + +// dispose tears the live connection down: closes the socket, cancels the +// attempt (its in-flight seam calls and the pump — same context), joins the +// pump, and stops the staleness timer. +// +// The CLOSE COMES FIRST, and the order is load-bearing rather than +// stylistic. §23 requires the connector to close the still-open socket +// explicitly — the rejected subscription Action Cable leaves open, and every +// terminal — and a close is only observable to the peer as a close frame. +// Cancellation is allowed to kill the connection outright (the seam contract +// says a cancelled read returns promptly, and the default transport's library +// aborts the socket to do it), so cancelling first races the close handshake +// and the peer sees an abrupt teardown instead. Close is documented to unblock +// ReadFrame, so the pump still exits; the cancel that follows is what returns +// any in-flight seam call promptly. +func (lc *liveConn) dispose(cancel context.CancelFunc) { + _ = lc.conn.Close(closeCodeNormal, "") + cancel() + for range lc.frames { //nolint:revive // draining to the pump's close is the join + } + lc.stale.stop() +} + +// staleHolder owns the per-socket staleness timer: armed at socket open, +// re-armed by the pump on every inbound frame of any kind. Because the pump +// swaps timers concurrently with the state machine's select, firings carry a +// generation: a firing whose generation is no longer current was superseded +// by a frame the pump received first, and is disregarded. +// +// Suspension while the pump is blocked on a full hand-off queue is realized +// AT EVALUATION, not at arming (SPEC.md §23 "Cable Protocol Details"): the +// timer stays armed throughout — `staleness` remains in every socket-open +// state's exact timer set — and a firing whose window overlapped a +// pump-blocked interval is disregarded and re-armed rather than dispatched. A +// firing whose window the pump spent reading is authoritative. +type staleHolder struct { + mu sync.Mutex + clock Clock + d time.Duration + timer Timer + gen int + last time.Time // last frame receipt (or arm); Now readings, deltas only + stopped bool + // blocked counts hand-offs currently blocked on the full queue, and + // suspended latches whether the ARMED window has overlapped one: a firing + // is evidence of a dead peer only if the pump spent the whole window able + // to observe frames. + blocked int + suspended bool + // expired latches an AUTHORITATIVE firing that the state machine has not + // observed yet — a window that closed while the consumer was inside a + // delivery or a callback, with the pump reading throughout. Once latched + // nothing re-arms over it: the fired timer stays put so the next select + // the state machine runs still picks the firing up, and evaluate reports + // it whatever else has happened since. + expired bool + // rearm wakes the state machine when the window is swapped from the pump + // goroutine: a select pass holds ONE timer, so a swap is invisible to a + // parked select until something else wakes it. + rearm chan struct{} +} + +func newStaleHolder(clock Clock, d time.Duration) *staleHolder { + // The window's origin is read BEFORE the timer is armed, here and in arm. + // Arming publishes the timer — a virtual clock's advance, or a real + // clock's scheduler, can run between the two — and a `last` taken after + // that lands INSIDE the window, understating the silence the expiry + // reports (0s for a full expiry, once the advance is the one that fires + // it). Taken before, it can only sit at or fractionally ahead of the + // window's start, so the reported age is never short. + last := clock.Now() + return &staleHolder{ + clock: clock, + d: d, + timer: clock.NewTimer(d, timerStaleness), + last: last, + rearm: make(chan struct{}, 1), + } +} + +// rearmed is the wake channel for window swaps. +func (h *staleHolder) rearmed() <-chan struct{} { return h.rearm } + +// current returns the armed timer and its generation for one select pass. +func (h *staleHolder) current() (Timer, int) { + h.mu.Lock() + defer h.mu.Unlock() + return h.timer, h.gen +} + +// reset re-arms the timer — pump-side, per inbound frame. The frame's receipt +// instant becomes the new window's origin only if the window was actually +// re-armed; a frame that arrived after the deadline had already fired does +// not move it (see arm). +func (h *staleHolder) reset() { + h.mu.Lock() + defer h.mu.Unlock() + if h.stopped { + return + } + // Read before arming, for newStaleHolder's reason. + now := h.clock.Now() + if h.arm() { + h.last = now + } +} + +// arm replaces the armed timer with a fresh window, reporting whether it did. +// Callers hold h.mu. +// +// Stop's result is the ORDERING DISCRIMINATOR, not a formality: it reports +// whether the timer was still pending, so a false return means the deadline +// had already fired before this call. The generation guard's own premise is +// that a superseding frame is one "the pump received first" — but a bare +// gen++ supersedes unconditionally, including for a frame that arrived AFTER +// the window closed. SPEC.md §23 pins that ordering the other way: the reset +// happens pump-side at frame receipt precisely "so frame-vs-deadline ordering +// is well-defined at the transport boundary regardless of queue depth or +// consumer latency: a fired staleness deadline observed on return from a slow +// delivery is authoritative". Re-arming over such a firing erases it — and +// the state machine, which may be inside a slow delivery or an Observer +// callback and holding no timer at all, would never see the expired window, +// nor the timer that carried it once it was swapped out. +// +// So an already-fired window is LATCHED rather than superseded, and nothing +// re-arms afterwards. The one exemption is a window that overlapped a blocked +// hand-off: a full queue is a fast peer, not a dead one, so that firing is +// not evidence and re-arms as before. +func (h *staleHolder) arm() bool { + if h.expired { + return false + } + if !h.timer.Stop() && !h.suspended { + h.expired = true + return false + } + h.timer = h.clock.NewTimer(h.d, timerStaleness) + h.gen++ + select { + case h.rearm <- struct{}{}: + default: + } + // The new window starts suspended only if the pump is blocked right now: + // it is already unable to observe a reset. + h.suspended = h.blocked > 0 + return true +} + +// suspend marks the armed window as overlapping a blocked hand-off. A full +// queue means the peer is sending faster than the connector consumes — the +// opposite of a dead peer — and a pump that is not reading cannot observe the +// resets that would prove liveness. +func (h *staleHolder) suspend() { + h.mu.Lock() + defer h.mu.Unlock() + h.blocked++ + h.suspended = true +} + +// resume releases one blocked hand-off. When the last one clears, the window +// is re-armed fresh: the hand-off completing is itself proof the peer was +// sending, and the only window that can testify to a dead peer is one the pump +// spent reading from end to end. +func (h *staleHolder) resume() { + h.mu.Lock() + defer h.mu.Unlock() + h.blocked-- + if h.blocked == 0 && !h.stopped { + // suspended is true here by construction (suspend set it, and only arm + // clears it — which cannot have run with blocked > 0), so this never + // latches: a firing over a blocked window is not evidence. It is still + // a no-op if an expiry was latched BEFORE the block, which is right — + // that window closed while the pump was reading. + h.arm() + } +} + +// evaluate decides one observed firing, returning the age of the silence when +// the firing is authoritative. It is disregarded when a frame the pump +// received first already superseded it (a stale generation), and when its +// window overlapped a pump-blocked interval — in which case the window is +// re-armed here, so the state's exact timer set is unchanged. +// +// A LATCHED expiry outranks both tests: it was already decided authoritative +// at the moment the deadline beat the frame that followed it (arm), and by +// then the window is over — a hand-off that blocks afterwards says nothing +// about it, and no re-arm can have moved the generation, since the latch +// stops them. +func (h *staleHolder) evaluate(gen int) (time.Duration, bool) { + h.mu.Lock() + defer h.mu.Unlock() + if h.stopped { + return 0, false + } + if h.expired { + return h.clock.Now().Sub(h.last), true + } + if gen != h.gen { + return 0, false + } + if h.suspended { + h.arm() + return 0, false + } + return h.clock.Now().Sub(h.last), true +} + +// stop cancels the timer permanently. +func (h *staleHolder) stop() { + h.mu.Lock() + defer h.mu.Unlock() + h.stopped = true + h.timer.Stop() +} + +// liveBuffer is the state-machine-owned live-event buffer (SPEC.md §23 +// "Semantic Signals"): only event-bearing frames are admitted, so it is +// denominated in events and every dropped entry has an id. +type liveBuffer struct { + capacity int + events []Event + // onChange reports the buffer's occupancy after every change. Test-only + // (testHooks.bufferOccupancy); nil in production. + onChange func(int) +} + +func newLiveBuffer(capacity int, onChange func(int)) *liveBuffer { + return &liveBuffer{capacity: capacity, onChange: onChange} +} + +// add admits ev, returning the ids of any events dropped to make room — +// oldest first. +func (b *liveBuffer) add(ev Event) []int64 { + var dropped []int64 + for len(b.events) >= b.capacity { + dropped = append(dropped, b.events[0].ID) + // Zeroed before the reslice, for shift's reason: the reslice removes + // the evicted event logically, but the slice that results still + // points into the same backing array, whose prefix keeps that + // event's strings reachable until a later reallocation. Under + // sustained overflow that is a second buffer's worth of payload held + // beyond the ceiling §23 publishes — retained by events that no + // longer count toward occupancy. + b.events[0] = Event{} + b.events = b.events[1:] + } + b.events = append(b.events, ev) + b.changed() + return dropped +} + +// shift removes and returns the oldest buffered event, reporting false when +// the buffer is empty. It is the drain's ONLY read, one event at a time: an +// event leaves the buffer exactly when it is about to be delivered, so +// occupancy always accounts for everything still pending and the capacity +// stays a bound on events held at once. +// +// Taking the whole buffer in one batch instead — the shape this replaced — +// let a drain hold `capacity` events outside the buffer while the buffer +// refilled to `capacity` behind them, so a slow drain retained twice the +// configured number of events and twice SPEC.md §23's published memory +// ceiling. Counting that batch against admission instead of removing it would +// need the batch to be evictable too (an overflow drops the OLDEST, which +// during a drain are precisely the taken ones), which is this method with a +// second container in front of it. +func (b *liveBuffer) shift() (Event, bool) { + if len(b.events) == 0 { + return Event{}, false + } + ev := b.events[0] + // Re-slicing alone would pin the whole backing array through the + // drain; zeroing the vacated slot lets it go. + b.events[0] = Event{} + b.events = b.events[1:] + b.changed() + return ev, true +} + +func (b *liveBuffer) changed() { + if b.onChange != nil { + b.onChange(len(b.events)) + } +} + +// catchUpHandoff is the typed boundary between the subscription lifecycle +// (this file) and the catch-up walk (SPEC.md §23 transitions 16–26): it +// carries everything a freshly confirmed subscription owns. The walk consumes +// it in catchup.go through loop.catchUp; the lifecycle side does not change +// shape for what the walk does with it. +type catchUpHandoff struct { + // at is the live attempt: its cancellation scope, the open socket, the + // running frame pump, and the armed staleness holder. + at *attempt + // entry is the selected entry cursor for the catch-up walk. + entry Cursor + // presentClass reports whether entry resolves at the server's present + // head — the Entry Boundary's hold-then-save discipline applies — rather + // than in served history (per-page saves). + presentClass bool + // buffer holds live events admitted before confirmation; its pre-cut + // contents become the entry snapshot. + buffer *liveBuffer +} + +// cycleOutcome is what one connect cycle (or the catch-up continuation) +// reports back to the outer run loop. +type cycleOutcome struct { + kind outcomeKind + // retryAfter floors the next Backoff delay (server-directed; cap-exempt). + retryAfter time.Duration + // term is the terminal error element (outcomeTerminal only). + term *TerminalError +} + +type outcomeKind int + +const ( + // outcomeFailed — a continuable failure: the attempt is already disposed + // and the loop enters Backoff (transitions 4/7/9/14/15). + outcomeFailed outcomeKind = iota + 1 + // outcomeTerminal — the feed ends with exactly one typed error element. + outcomeTerminal + // outcomeClosed — close()/cancellation: the universal edge to Closed, no + // error element. + outcomeClosed +) + +// loop is one Events consumption's state machine. It runs entirely on the +// consumer's goroutine. +type loop struct { + cfg *config + hooks testHooks + runCtx context.Context + yield func(Event, error) bool + + state connState + // failedCycles is the reconnect-cycle failure count n — incremented per + // failed cycle, reset on confirmation (transition 11 resets only this). + failedCycles int + // authFailures is the shared connection-level authorization counter: + // unauthorized mints, `unauthorized` disconnects, and unauthorized polls + // increment it; it resets ONLY on a successful poll page (catchup.go) — + // never on confirmation. + authFailures int + + // pollFailures is the consecutive-poll-failure index k behind the + // `poll-retry` timer — separate from failedCycles, reset by any + // successful poll page and by socket teardown. + pollFailures int + // position is the connector's in-memory position: authoritative for + // resume and repair within the run, seeded by the checkpoint load under + // the resume entry mode and advanced only by accepted poll pages. The + // store is write-through durability; this field is never re-read from it + // mid-run. + position string + // reentry latches the cursor a re-entry selected (transitions 17/18/19) + // as RECONNECT state, held until a page replaces it. A re-entry's cursor + // is the connector's whole answer to a position the server refused, went + // gone on, or whose lineage a filter change ended — and it lives only in + // the walk that chose it. Without the latch, a socket torn down before + // the re-entry's first page lands drops back to a position the server + // already rejected, or to the configured start mode, which for a + // poll-served reset cursor means silently jumping to the present. + reentry *reentryCursor + // lastPollServedID is the highest event id the POLL lane has served on + // this run — the reset cursor transitions 18/19 re-enter at, tracked + // independently of delivery, dedupe, and the live lane (a live-delivered + // id is never a reset cursor). Empty pages advance nothing; with no + // poll-served id the re-entry is present-class. + lastPollServedID int64 + // stopped latches a consumer break (a false yield): nothing is ever + // yielded after it. + stopped bool + + identifier string + subscribeFrame []byte + buffer *liveBuffer + dedupe *dedupe + + // deferred is the one pump receive the in-flight-poll servicing took out + // of band and left for the walk's ordinary dispatch point (catchup.go's + // pollPage). It belongs to the attempt that produced it: disposal clears + // it, so a dead attempt's last frame can never leak into the next one. + deferred *deferredFrame + + // catchUp is the post-confirmation continuation: the catch-up walk, the + // entry boundary, the drain, and streaming (catchup.go). + catchUp func(catchUpHandoff) cycleOutcome +} + +func newLoop(runCtx context.Context, cfg *config, hooks testHooks) *loop { + l := &loop{ + cfg: cfg, + hooks: hooks, + runCtx: runCtx, + state: stateIdle, + buffer: newLiveBuffer(cfg.liveBufferCapacity, hooks.bufferOccupancy), + dedupe: newDedupe(cfg.dedupeCapacity), + } + // Built once; identical bytes on every (re)connection and retransmit. + l.identifier = subscribeIdentifier(cfg.filters) + l.subscribeFrame = subscribeCommand(l.identifier) + l.catchUp = l.runCatchUp + return l +} + +func (l *loop) setState(s connState) { + l.state = s + if l.hooks.stateChanged != nil { + l.hooks.stateChanged(s) + } +} + +// run executes the state machine. The checkpoint load comes first — exactly +// once, before the first mint, so its failure is terminal with zero wire +// attempts. Transition 1: the first cycle is then immediate — no backoff. +// Terminal outcomes yield exactly one error element; the universal Closed +// edge yields none. +func (l *loop) run(yield func(Event, error) bool) { + l.yield = yield + if l.runCtx.Err() != nil { + l.setState(stateClosed) + return + } + if terr := l.loadCheckpoint(); terr != nil { + l.setState(stateTerminal) + l.yield(Event{}, terr) + return + } + var delay time.Duration + for { + out := l.runCycle(delay) + switch out.kind { + case outcomeClosed: + l.setState(stateClosed) + return + case outcomeTerminal: + l.setState(stateTerminal) + l.yield(Event{}, out.term) + return + case outcomeFailed: + // The failed attempt is fully disposed, so the exact + // outstanding-timer set on entry to Backoff is {backoff}. + l.failedCycles++ + l.setState(stateBackoff) + d := reconnectDelay(out.retryAfter, l.failedCycles, l.cfg.rand) + t := l.cfg.clock.NewTimer(d, timerBackoff) + select { + case <-t.C(): + // Transition 2: Backoff → Minting; a fresh ticket is ALWAYS + // minted next. + delay = d + case <-l.runCtx.Done(): + t.Stop() + l.setState(stateClosed) + return + } + } + } +} + +// runCycle runs one pass Minting → Connecting → AwaitingWelcome → +// AwaitingConfirmation, continuing into the catch-up handoff on +// confirmation. Every exit path has disposed the attempt (socket closed, +// pump joined, timers stopped) before returning — except the confirmed +// handoff, which transfers ownership to the catch-up continuation. +func (l *loop) runCycle(delay time.Duration) cycleOutcome { + at := &attempt{} + at.ctx, at.cancel = context.WithCancel(l.runCtx) + defer at.cancel() + + // Transitions 1/2 → Minting. No timers are armed here (per-state + // invariant: Minting's set is {}); mint cancellation rides the attempt + // context. + l.setState(stateMinting) + if l.cfg.observer.Connecting != nil { + l.cfg.observer.Connecting(l.failedCycles+1, delay) + } + // The observer runs on this goroutine and may Close from inside it, which + // cancels synchronously (Connector.Close). Re-checking here is what turns + // that cancellation into the universal Closed edge BEFORE the run's first + // wire act, rather than one mint after it. + if l.runCtx.Err() != nil { + return cycleOutcome{kind: outcomeClosed} + } + ticket, err := l.cfg.minter.MintStreamTicket(at.ctx) + if l.runCtx.Err() != nil { + return cycleOutcome{kind: outcomeClosed} + } + if err != nil { + return l.classifyMintFailure(err) + } + + // Transition 3 → Connecting: dial the mint's url verbatim. The cable-URL + // policy pre-check runs before anything is armed — a policy violation + // recurs on every re-mint, so it is Terminal(invalid_cable_url), never + // Backoff. The error never carries the URL: the ticket rides in its + // query string. + l.setState(stateConnecting) + if derr := checkCableURL(ticket.URL); derr != nil { + return cycleOutcome{kind: outcomeTerminal, term: &TerminalError{ + Reason: ReasonInvalidCableURL, Msg: derr.Reason, Err: derr, + }} + } + // The handshake deadline arms on entry to Connecting, BEFORE dial — it + // spans dial-to-welcome, so a stalled dial expires it (transition 7). + hs := l.cfg.clock.NewTimer(handshakeDeadline, timerHandshakeDeadline) + type dialResult struct { + conn CableConn + err error + } + dialCh := make(chan dialResult, 1) + go func() { + conn, dialErr := l.cfg.transport.Dial(at.ctx, ticket.URL, maxFrameBytes) + dialCh <- dialResult{conn: conn, err: dialErr} + }() + var conn CableConn + select { + case r := <-dialCh: + if l.runCtx.Err() != nil { + hs.Stop() + if r.conn != nil { + _ = r.conn.Close(closeCodeNormal, "") + } + return cycleOutcome{kind: outcomeClosed} + } + if r.err != nil { + hs.Stop() + var derr *DialError + if errors.As(r.err, &derr) && derr.Kind == DialPolicy { + return cycleOutcome{kind: outcomeTerminal, term: &TerminalError{ + Reason: ReasonInvalidCableURL, Msg: derr.Reason, Err: derr, + }} + } + return cycleOutcome{kind: outcomeFailed} // transition 7 + } + conn = r.conn + case <-hs.C(): + // Transition 7: the deadline expired mid-dial — cancel the pending + // dial (the seam contract requires a prompt return) and dispose any + // connection it raced to open. + at.cancel() + if r := <-dialCh; r.conn != nil { + _ = r.conn.Close(closeCodeNormal, "") + } + return cycleOutcome{kind: outcomeFailed} + case <-l.runCtx.Done(): + at.cancel() + if r := <-dialCh; r.conn != nil { + _ = r.conn.Close(closeCodeNormal, "") + } + hs.Stop() + return cycleOutcome{kind: outcomeClosed} + } + + // Transition 6 → AwaitingWelcome: staleness arms at socket open, the + // pump starts, and the handshake deadline keeps running to `welcome`. + if l.cfg.observer.Connected != nil { + l.cfg.observer.Connected() + } + at.lc = newLiveConn(at.ctx, conn, l.cfg.clock, l.cfg.staleAfter, l.hooks) + return l.awaitConfirmation(at, hs) +} + +// classifyMintFailure maps a failed mint seam call onto transitions 4/5 or +// the out-of-inventory mint_failed edge. An error the adapter did not +// classify (not a *MintError) is treated as unrecoverable: the seam contract +// maps every generated outcome onto exactly one kind, and retrying an +// unclassifiable failure risks a tight loop on a permanent condition. +func (l *loop) classifyMintFailure(err error) cycleOutcome { + var me *MintError + if !errors.As(err, &me) { + return cycleOutcome{kind: outcomeTerminal, term: &TerminalError{ + Reason: ReasonMintFailed, Msg: "unclassified mint failure", Err: err, + }} + } + switch me.Kind { + case MintTransient: + return cycleOutcome{kind: outcomeFailed} // transition 4 + case MintThrottled: + // Transition 4; Retry-After floors the next Backoff delay. + return cycleOutcome{kind: outcomeFailed, retryAfter: me.RetryAfter} + case MintUnauthorized: + // Transition 4 below the threshold; transition 5 — the shared + // connection-level counter's terminal — at the 3rd consecutive. + l.authFailures++ + if l.authFailures >= authFailureThreshold { + return cycleOutcome{kind: outcomeTerminal, term: &TerminalError{ + Reason: ReasonAuthorizationFailed, + Msg: fmt.Sprintf("%d consecutive connection-level authorization failures", l.authFailures), + Err: err, + }} + } + return cycleOutcome{kind: outcomeFailed, retryAfter: me.RetryAfter} + case MintUnrecoverable: + return cycleOutcome{kind: outcomeTerminal, term: &TerminalError{Reason: ReasonMintFailed, Err: err}} + default: + return cycleOutcome{kind: outcomeTerminal, term: &TerminalError{ + Reason: ReasonMintFailed, Msg: fmt.Sprintf("unknown mint error kind %d", int(me.Kind)), Err: err, + }} + } +} + +// awaitConfirmation drives AwaitingWelcome and AwaitingConfirmation +// (transitions 8–15 plus the state-generic protocol-fatal and invalid-frame +// dispatch). deadline is the running handshake-deadline on entry; `welcome` +// re-arms it as the confirmation-deadline (transition 8). +func (l *loop) awaitConfirmation(at *attempt, deadline Timer) cycleOutcome { + l.setState(stateAwaitingWelcome) + for { + staleTimer, staleGen := at.lc.stale.current() + select { + case <-l.runCtx.Done(): + l.disposeAttempt(at, deadline) + return cycleOutcome{kind: outcomeClosed} + case <-deadline.C(): + // Transition 9 (handshake lapse) or 14 (confirmation lapse): + // full teardown — conn, pump, and ALL the attempt's timers — + // then a jittered fresh-ticket retry. + lapsed := errDeadlineLapsed(l.state) + l.disposeAttempt(at, nil) + l.observeDisconnected("", lapsed) + return cycleOutcome{kind: outcomeFailed} + case <-at.lc.stale.rearmed(): + continue + case <-staleTimer.C(): + age, ok := at.lc.stale.evaluate(staleGen) + if !ok { + // Superseded by a frame the pump received first, or suspended + // by a blocked hand-off and re-armed. + continue + } + // Staleness expiry — rows 9/15's staleness trigger. + l.disposeAttempt(at, deadline) + if l.cfg.observer.StaleConnection != nil { + l.cfg.observer.StaleConnection(age) + } + l.observeDisconnected("", errStaleConnection) + return cycleOutcome{kind: outcomeFailed} + case item, ok := <-at.lc.frames: + if !ok { + // The pump exited without a terminating error — only + // possible under a cancellation race; treat as a socket + // failure. + l.disposeAttempt(at, deadline) + l.observeDisconnected("", errors.New("event feed frame pump exited")) + return cycleOutcome{kind: outcomeFailed} + } + if out, done := l.handleFrame(at, &deadline, item); done { + return out + } + } + } +} + +// writeSubscribe sends the subscribe command, BOUNDED by the phase deadline +// that is running when it is sent. +// +// This is the one place the state machine hands control to the transport +// synchronously while a deadline governs the phase, and left synchronous it +// defeats that deadline outright. `CableConn.WriteFrame` may block — a peer +// whose receive window has closed, or any seam implementation that returns +// only when its context is cancelled — and the context in question is +// `at.ctx`, which nothing cancels but a teardown this very write is blocking. +// The state machine cannot return to its select, so neither the handshake nor +// the confirmation deadline can ever be observed, and the feed hangs after +// `welcome` until the consumer closes the connector. +// +// Running the write on its own goroutine and selecting the deadline against +// it restores the bound: a lapse takes transition 9/14 exactly as it would +// have had the write never blocked, and disposal's cancel is what returns the +// abandoned write (the seam contract requires a cancelled write to return +// promptly, and Close is documented to unblock it too). Staleness is +// deliberately not in this select — the phase deadline is the tighter bound +// on a handshake that is going nowhere, and adding a second timer here would +// duplicate the generation dance for no additional guarantee. +func (l *loop) writeSubscribe(at *attempt, deadline *Timer) (cycleOutcome, bool) { + written := make(chan error, 1) + go func() { written <- at.lc.conn.WriteFrame(at.ctx, l.subscribeFrame) }() + select { + case werr := <-written: + if werr != nil { + l.disposeAttempt(at, *deadline) + l.observeDisconnected("", werr) + return cycleOutcome{kind: outcomeFailed}, true + } + if l.hooks.subscribeWritten != nil { + l.hooks.subscribeWritten() + } + return cycleOutcome{}, false + case <-l.runCtx.Done(): + l.disposeAttempt(at, *deadline) + return cycleOutcome{kind: outcomeClosed}, true + case <-(*deadline).C(): + // The deadline is spent, so disposal is handed nil: the attempt's + // timer set is empty from here, as on every other lapse. + lapsed := errDeadlineLapsed(l.state) + l.disposeAttempt(at, nil) + l.observeDisconnected("", lapsed) + return cycleOutcome{kind: outcomeFailed}, true + } +} + +// errDeadlineLapsed names the deadline that lapsed in the given state. +func errDeadlineLapsed(s connState) error { + if s == stateAwaitingWelcome { + return errors.New("event feed handshake deadline lapsed before welcome") + } + return errors.New("event feed confirmation deadline lapsed before confirm_subscription") +} + +// handleFrame dispatches one pump item in AwaitingWelcome / +// AwaitingConfirmation. It returns done=true when the cycle is over — +// including the confirmed handoff's own outcome flowing back through it. +func (l *loop) handleFrame(at *attempt, deadline *Timer, item pumpItem) (cycleOutcome, bool) { + if item.err != nil { + // Socket failure: peer close, read error, or the transport's + // frame-size rejection (rows 9/15). + l.disposeAttempt(at, *deadline) + l.observeDisconnected("", item.err) + return cycleOutcome{kind: outcomeFailed}, true + } + f, err := parseFrame(item.data) + if err != nil { + // Invalid-frame class, parse shape: a peer protocol violation + // dispatched as a socket failure — never terminal, never a silent + // skip (SPEC.md §23 "Cable Protocol Details"). + l.disposeAttempt(at, *deadline) + l.observeDisconnected("", err) + return cycleOutcome{kind: outcomeFailed}, true + } + if l.hooks.frameHandled != nil { + l.hooks.frameHandled(f.kind) + } + switch f.kind { + case frameWelcome: + // Transition 8 on the first welcome; a duplicate welcome resends + // the identical subscribe bytes (stock behavior — the server + // absorbs identical retransmits) without touching the deadline. + if out, done := l.writeSubscribe(at, deadline); done { + return out, true + } + if l.state == stateAwaitingWelcome { + if !(*deadline).Stop() { + // The handshake deadline had ALREADY FIRED. Stop's result is + // the ordering discriminator here for the same reason it is + // in staleHolder.arm: the frame and the expiry were both + // ready and the select picked one at random. Re-arming over + // the firing would swap the expired timer out for a fresh + // confirmation window and accept a welcome that arrived past + // the deadline — transition 9's lapse, silently converted + // into a live handshake half the time it happens. + lapsed := errDeadlineLapsed(l.state) + l.disposeAttempt(at, nil) + l.observeDisconnected("", lapsed) + return cycleOutcome{kind: outcomeFailed}, true + } + *deadline = l.cfg.clock.NewTimer(l.cfg.confirmationDeadline, timerConfirmationDeadline) + l.setState(stateAwaitingConfirmation) + } + return cycleOutcome{}, false + case framePing, frameUnknown: + // Liveness only — the pump already reset staleness. + return cycleOutcome{}, false + case frameConfirm: + if l.state != stateAwaitingConfirmation || f.identifier != l.identifier { + // Pre-subscribe, or a foreign identifier: ignored. + return cycleOutcome{}, false + } + // Transition 11: cancel the deadline, reset the attempt counter + // (the authorization counter resets only on a successful poll + // page), select the entry cursor, hand off to catch-up. + if !(*deadline).Stop() { + // The confirmation deadline had already fired — the same select + // race the welcome branch above reads Stop for, and the same + // answer. Proceeding here is worse than proceeding there: the + // connector would announce Confirmed and enter catch-up on a + // subscription whose confirmation arrived past transition 14's + // deadline, so the lapse is not merely swallowed but overwritten + // by a successful handoff. + lapsed := errDeadlineLapsed(l.state) + l.disposeAttempt(at, nil) + l.observeDisconnected("", lapsed) + return cycleOutcome{kind: outcomeFailed}, true + } + l.failedCycles = 0 + if l.cfg.observer.Confirmed != nil { + l.cfg.observer.Confirmed() + } + entry, present := l.entryCursor() + return l.catchUp(catchUpHandoff{at: at, entry: entry, presentClass: present, buffer: l.buffer}), true + case frameReject: + if l.state != stateAwaitingConfirmation || f.identifier != l.identifier { + // A foreign identifier, or a rejection that arrived before the + // connector had sent any subscribe command at all. §23 draws + // transition 12 only from AwaitingConfirmation, and the confirm + // branch above already gates on exactly that. Ungated, this is + // the connector's single most severe verdict — Terminal with + // ZERO reconnects — produced against zero subscription attempts + // by one unsolicited frame from a peer that has only completed + // the WebSocket handshake. + return cycleOutcome{}, false + } + // Transition 12: always terminal — cancel the deadline, explicitly + // close the still-open socket (Action Cable leaves a rejected + // socket open), ZERO reconnects. + l.disposeAttempt(at, *deadline) + return cycleOutcome{kind: outcomeTerminal, term: &TerminalError{ + Reason: ReasonSubscriptionRejected, + Msg: "the server rejected the EventsChannel subscription", + }}, true + case frameDisconnect: + return l.dispatchDisconnect(at, *deadline, f), true + case frameMessage: + if f.identifier != l.identifier { + return cycleOutcome{}, false + } + ev, derr := decodeMessageEvent(f.message) + if derr != nil { + // Invalid-frame class, decode shape: same socket-failure + // disposition as the parse shape. + l.disposeAttempt(at, *deadline) + l.observeDisconnected("", derr) + return cycleOutcome{kind: outcomeFailed}, true + } + return l.admitLive(at, *deadline, ev) + } + return cycleOutcome{}, false +} + +// dispatchDisconnect applies §23's reason-string dispatch — never the +// reconnect flag alone — to a raw disconnect text frame read pre-confirm. +// The attempt is disposed first either way: a disconnect frame is the +// server's last word on this socket. +func (l *loop) dispatchDisconnect(at *attempt, deadline Timer, f frame) cycleOutcome { + preWelcome := l.state == stateAwaitingWelcome + l.disposeAttempt(at, deadline) + l.observeDisconnected(f.reason, nil) + switch f.reason { + case disconnectReasonProtocolFatal: + // Transition 13 — and the state-generic rule for AwaitingWelcome: + // the server's own protocol verdict is terminal from every + // socket-open state, never retried into. + return cycleOutcome{kind: outcomeTerminal, term: &TerminalError{ + Reason: ReasonProtocolFatal, + Msg: "server disconnected: " + f.reason, + }} + case disconnectReasonUnauthorized: + if preWelcome { + // Rows 9/10: `unauthorized` arrives only pre-welcome at the + // verified head — a fresh-ticket retry below the shared + // counter's threshold, terminal at the 3rd consecutive. + l.authFailures++ + if l.authFailures >= authFailureThreshold { + return cycleOutcome{kind: outcomeTerminal, term: &TerminalError{ + Reason: ReasonAuthorizationFailed, + Msg: fmt.Sprintf("%d consecutive connection-level authorization failures", l.authFailures), + }} + } + return cycleOutcome{kind: outcomeFailed} + } + // Wire-impossible post-welcome: a socket drop (row 15) with NO + // counter increment — the reconnect cycle re-mints, and a genuinely + // revoked user's mint then fails and increments the counter. + return cycleOutcome{kind: outcomeFailed} + default: + // `remote`, unrecognized reasons, and any reconnect flag: a socket + // drop (rows 9/15). Unknown reasons are never guessed into a + // terminal class and never increment the authorization counter. + return cycleOutcome{kind: outcomeFailed} + } +} + +// admitLive admits a live event to the buffer — the live buffer is the only +// carrier of an in-flight-at-entry straggler — and dispatches the +// BufferOverflow semantic signal at drop time, the first consumer-context +// opportunity (SPEC.md §23 "Semantic Signals"). +// +// It is the single place a drop is dispatched from, on whichever path observed +// it: the pre-confirmation and Streaming dispatches, the bounded admission +// passes, the drain's scan, and the servicing of frames arriving while a poll +// seam call is in flight. Every one of them runs on the consumer's goroutine, +// which is what makes drop time an opportunity to dispatch on rather than +// something to park until a later cut — a cut a stalled or failing call may +// never reach. +func (l *loop) admitLive(at *attempt, deadline Timer, ev Event) (cycleOutcome, bool) { + dropped := l.buffer.add(ev) + if len(dropped) == 0 { + return cycleOutcome{}, false + } + signal := BufferOverflow{DroppedIDs: dropped, DroppedCount: len(dropped)} + if l.hooks.signalRaised != nil { + l.hooks.signalRaised(signal) + } + if l.cfg.observer.BufferOverflow != nil { + l.cfg.observer.BufferOverflow(len(dropped)) + } + if l.cfg.handler != nil && l.cfg.handler(signal) == Accept { + // Accept: the consumer owns the acknowledged incompleteness; the + // feed continues (acceptance is not license to skip retained + // deliveries — the catch-up slice's save-ordering invariant). + return cycleOutcome{}, false + } + // No handler — or Terminate: Terminal(buffer_overflow), no save. + l.disposeAttempt(at, deadline) + return cycleOutcome{kind: outcomeTerminal, term: &TerminalError{ + Reason: ReasonBufferOverflow, + Msg: fmt.Sprintf("live buffer overflow: %d event(s) dropped", len(dropped)), + }}, true +} + +// reentryCursor is a latched re-entry: the cursor transitions 17/18/19 +// selected and the entry class it carries. +type reentryCursor struct { + cursor Cursor + presentClass bool +} + +// acceptPosition records a position an accepted poll page moved the connector +// to. It is also what releases a latched re-entry — "until a page replaces +// it" is exactly this assignment, so a present-class re-entry stays latched +// through its whole walk and drain (nothing durable has moved until the held +// save) while a position-resume one is released by its first saved page. +func (l *loop) acceptPosition(position string) { + l.position = position + l.reentry = nil +} + +// entryCursor selects the catch-up entry cursor and whether it is +// present-class (SPEC.md §23 "Entry Boundary"). Three sources, in priority +// order, and the distinction between them is the point: +// +// - A LATCHED re-entry wins outright. It is the connector's live answer to +// a 410/400-position/409, still unreplaced by a page, and every other +// source is a cursor the server has already refused or superseded. +// - The IN-MEMORY position — accepted from a page earlier in this run, or +// seeded by the checkpoint load under the resume mode — comes next: it is +// authoritative within a run, so a reconnect resumes where the feed +// actually got to rather than re-entering at the mode's original cursor. +// - Otherwise the configured Start mode, which is the ONLY source for the +// explicit modes on a fresh run. `StartPresent`, `StartBeginning` and +// `StartAfter` promise `since=now`, `since=0` and `since=`; only +// `StartResume` is defined as "the stored position if any", which is why +// the load seeds the in-memory position for that mode alone (see +// loadCheckpoint) rather than here. +func (l *loop) entryCursor() (Cursor, bool) { + if l.reentry != nil { + return l.reentry.cursor, l.reentry.presentClass + } + if l.position != "" { + return Cursor{Position: l.position}, false + } + switch l.cfg.start.kind { + case startPresent: + return Cursor{Since: sincePresent}, true + case startBeginning: + return Cursor{Since: "0"}, false + case startAfter: + return Cursor{Since: strconv.FormatInt(l.cfg.start.eventID, 10)}, false + case startAtPosition: + return Cursor{Position: l.cfg.start.position}, false + default: // startResume + return Cursor{}, true + } +} + +// disposeAttempt tears one attempt down: stops the phase deadline (nil when +// it already fired), cancels the attempt context — which also cancels any +// in-flight seam call or write belonging to it — closes the connection, +// joins the pump, and stops the staleness timer. The next state is entered +// with the attempt's timer set empty. +func (l *loop) disposeAttempt(at *attempt, deadline Timer) { + // The deferred frame belongs to the attempt that produced it: a disposed + // attempt's last out-of-band receive is gone with its socket. + l.deferred = nil + if deadline != nil { + deadline.Stop() + } + if at.lc != nil { + at.lc.dispose(at.cancel) + } else { + at.cancel() + } +} + +// observeDisconnected reports one socket teardown. The reason is frame-derived +// (a raw disconnect frame's reason string, bounded only by the 1 MiB frame +// limit) and observers log it, so it is bounded by the same §9 cap the +// invalid-frame rendering uses — §23's Security Invariants apply it to frame +// contents generally, not just to errors. Dispatch reads f.reason itself, so +// the cap can never widen an unrecognized reason into a recognized one. +func (l *loop) observeDisconnected(reason string, err error) { + if l.cfg.observer.Disconnected != nil { + l.cfg.observer.Disconnected(truncateErrorText(reason), err) + } +} + +// checkpointKey is this run's durable identity — all four parts, always. The +// identity type and the store seam it names are in checkpoint.go; only the +// run-scoped derivation and the two calls through the seam are here. +func (l *loop) checkpointKey() CheckpointKey { + return CheckpointKey{ + Origin: l.cfg.origin, + AccountID: l.cfg.accountID, + ConsumerNamespace: l.cfg.consumerNamespace, + FilterKey: l.cfg.filters.FilterKey(), + } +} + +// loadCheckpoint runs the store's load exactly once, on the first iteration +// and BEFORE the first mint. Loaded seeds the in-memory position UNDER THE +// RESUME MODE (which is authoritative from then on); Missing proceeds to a +// present-class entry — no stored cursor is not an error; Failed is +// Terminal(checkpoint_load) with zero wire attempts, because collapsing it to +// Missing would silently start at the present and skip history. It runs +// whenever a store is configured, including under an explicit entry mode: the +// lineage's identity is settled before anything durable can move. +// +// Only `StartResume` is defined as "the stored position if any" (SPEC.md §23 +// "Consumer Ergonomics"). `StartPresent`, `StartBeginning` and `StartAfter` +// promise `since=now`, `since=0` and `since=`, and a caller pairing one +// with a store means it: seeding the position from the load would make every +// explicit mode behave as resume the moment the store had anything in it, so +// a checkpointed feed could never be deliberately replayed or reset. The load +// still HAPPENS under those modes — its failure edge and its lineage identity +// are not mode-dependent — the value is simply not what the entry is taken +// from. The store is written under the same key either way, so the run's +// first accepted page repoints the lineage. +func (l *loop) loadCheckpoint() *TerminalError { + if l.cfg.store == nil { + return nil + } + position, ok, err := l.cfg.store.Load(l.runCtx, l.checkpointKey()) + if err != nil { + return &TerminalError{ + Reason: ReasonCheckpointLoad, + Msg: "the checkpoint store failed to load the stored position", + Err: err, + } + } + if ok && position == "" { + // A store reporting FOUND with an empty position is a store failure, + // not a missing checkpoint, and the seam is where that has to be + // enforced: the built-in FileStore already classifies it this way + // (filestore.go, "an empty position cannot be told apart from having + // none"), but a custom store is under no obligation to, and nothing + // downstream can tell the two apart afterwards. + // + // Left unenforced it is a SILENT history skip rather than a loud + // failure. entryCursor selects on `l.position != ""`, so an empty + // position falls through to the StartResume default — a bare present + // entry, which is present-class — and the feed resumes at the server's + // head having skipped everything between the stored position and now, + // with no signal of any kind. Terminal(checkpoint_load) before any + // wire attempt is the same edge a load error takes. + // + // Checked regardless of start mode, for this function's stated reason: + // the load happens under every mode, and its failure edge is not + // mode-dependent — only which value the entry is taken from is. + return &TerminalError{ + Reason: ReasonCheckpointLoad, + Msg: "the checkpoint store reported a stored position that is empty; an empty position cannot be told apart from having none", + } + } + if ok && l.cfg.start.kind == startResume { + l.position = position + } + return nil +} + +// saveCheckpoint write-throughs one accepted position. The connector's own +// position tracking is in-memory and authoritative for resume and repair +// within the run; the store is durability only, so a failed save is reported +// through the observer and the feed continues — subsequent saves are still +// attempted, and the live cursor is neither regressed nor blanked. +func (l *loop) saveCheckpoint(position string) { + if l.cfg.store == nil { + return + } + if err := l.cfg.store.Save(l.runCtx, l.checkpointKey(), position); err != nil { + if l.cfg.observer.CheckpointSaveFailed != nil { + l.cfg.observer.CheckpointSaveFailed(err) + } + return + } + if l.cfg.observer.Checkpoint != nil { + l.cfg.observer.Checkpoint(position) + } +} diff --git a/go/pkg/basecamp/eventfeed/loop_test.go b/go/pkg/basecamp/eventfeed/loop_test.go new file mode 100644 index 0000000000..31e7d654e4 --- /dev/null +++ b/go/pkg/basecamp/eventfeed/loop_test.go @@ -0,0 +1,1759 @@ +// Slice-4a acceptance tests for the connector run loop: mint → dial → +// welcome → subscribe → confirm/reject/deadline (SPEC.md §23 transitions +// 1–15 plus the out-of-inventory mint_failed / invalid_cable_url / +// buffer_overflow / usage edges). Structured to mirror the tier-2 scenario +// fixtures (01–07, 17) so the fixture driver can later subsume them: all +// time flows through the injected feedtest.Clock — no wall-clock sleeps +// except bounded watchdog joins — and mint/connect counts count seam calls. +package eventfeed_test + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "runtime" + "strings" + "sync" + "testing" + "time" + + "github.com/basecamp/basecamp-sdk/go/pkg/basecamp/eventfeed" + "github.com/basecamp/basecamp-sdk/go/pkg/basecamp/eventfeed/feedtest" +) + +// watchdog bounds every rendezvous wait; virtual time never depends on it. +const watchdog = 5 * time.Second + +// harness wires a Connector to the feedtest fakes and a collector goroutine +// consuming Events. +type harness struct { + t *testing.T + clock *feedtest.Clock + tr *feedtest.Transport + minter *feedtest.Minter + polls *feedtest.Polls + conn *eventfeed.Connector + + cancel context.CancelFunc + done chan struct{} + boundary chan eventfeed.CatchUpBoundary + handled chan string + states chan string + + // breakAfter, when positive, makes the collector break out of the range + // after that many delivered events — the consumer-break teardown path. + breakAfter int + // pauseAfter, when positive, parks the collector INSIDE the loop body once + // that many events have been delivered, until resume is called. The loop + // body is the consumer's processing, so parking there parks the state + // machine mid-delivery — the serial back-pressure the deferred-consumption + // and staleness-suspension rules are stated over. + pauseAfter int + pause chan struct{} + pauseOnce sync.Once + + mu sync.Mutex + events []eventfeed.Event + terminal *eventfeed.TerminalError + elements int + // log is the single ordered ledger of deliveries and checkpoint calls. + // Both are produced on the consumer's goroutine (the loop body IS the + // consumer), so their relative order is total, not sampled. + log []string +} + +// testOrigin is the harness's configured API base origin: the checkpoint +// key's origin and the same-origin reference for continuation validation. +const testOrigin = "https://3.basecampapi.com" + +func newHarness(t *testing.T, opts ...eventfeed.Option) *harness { + t.Helper() + h := &harness{ + t: t, + clock: feedtest.NewClock(), + tr: feedtest.NewTransport(), + minter: feedtest.NewMinter(), + polls: feedtest.NewPolls(), + done: make(chan struct{}), + pause: make(chan struct{}), + boundary: make(chan eventfeed.CatchUpBoundary, 8), + handled: make(chan string, 256), + states: make(chan string, 256), + } + base := []eventfeed.Option{ + eventfeed.WithTransport(h.tr), + eventfeed.WithClock(h.clock), + } + c, err := eventfeed.New(testOrigin, "5951425", h.minter, h.polls, append(base, opts...)...) + if err != nil { + t.Fatalf("New: %v", err) + } + h.conn = c + // A non-blocking send: a reconnect re-enters the boundary, and a test + // that never reads it must not wedge the loop. + c.OnCatchUpEntered(func(b eventfeed.CatchUpBoundary) { + select { + case h.boundary <- b: + default: + } + }) + c.OnFrameHandled(func(kind string) { h.handled <- kind }) + c.OnStateChanged(func(state string) { h.states <- state }) + return h +} + +// record appends one entry to the ordered delivery/checkpoint ledger. +func (h *harness) record(entry string) { + h.mu.Lock() + defer h.mu.Unlock() + h.log = append(h.log, entry) +} + +// ledger returns the ordered delivery/checkpoint log. +func (h *harness) ledger() []string { + h.mu.Lock() + defer h.mu.Unlock() + return append([]string(nil), h.log...) +} + +// start ranges Events on a collector goroutine. +func (h *harness) start() { + h.t.Helper() + ctx, cancel := context.WithCancel(context.Background()) + h.cancel = cancel + h.t.Cleanup(cancel) + h.t.Cleanup(func() { h.conn.Close() }) + // A parked collector must never outlive the test, however it ended. + h.t.Cleanup(h.resume) + go func() { + defer close(h.done) + for ev, err := range h.conn.Events(ctx) { + h.mu.Lock() + h.elements++ + if err != nil { + var te *eventfeed.TerminalError + if errors.As(err, &te) { + h.terminal = te + } else { + h.t.Errorf("iteration yielded a non-terminal error element: %v", err) + } + } else { + h.events = append(h.events, ev) + h.log = append(h.log, fmt.Sprintf("event %d", ev.ID)) + } + stop := h.breakAfter > 0 && len(h.events) >= h.breakAfter + park := err == nil && h.pauseAfter > 0 && len(h.events) == h.pauseAfter + h.mu.Unlock() + if park { + <-h.pause + } + if stop { + break + } + } + }() +} + +// resume releases a collector parked by pauseAfter. +func (h *harness) resume() { + h.pauseOnce.Do(func() { close(h.pause) }) +} + +// join waits for the iteration to end. +func (h *harness) join() { + h.t.Helper() + select { + case <-h.done: + case <-time.After(watchdog): + h.t.Fatalf("iteration did not end within the watchdog") + } +} + +// snapshot returns the collected elements so far. +func (h *harness) snapshot() (events []eventfeed.Event, terminal *eventfeed.TerminalError, elements int) { + h.mu.Lock() + defer h.mu.Unlock() + return append([]eventfeed.Event(nil), h.events...), h.terminal, h.elements +} + +// awaitTimer blocks until a timer of the given kind is outstanding. +func (h *harness) awaitTimer(kind string) { + h.t.Helper() + done := make(chan struct{}) + go func() { h.clock.AwaitTimer(kind); close(done) }() + select { + case <-done: + case <-time.After(watchdog): + h.t.Fatalf("timer %q was not armed within the watchdog", kind) + } +} + +// fireTimer awaits and fires the named timer without advancing the clock, +// returning its scheduled delay. +func (h *harness) fireTimer(kind string) time.Duration { + h.t.Helper() + h.awaitTimer(kind) + d, ok := h.clock.FireTimer(kind) + if !ok { + h.t.Fatalf("no outstanding %q timer to fire", kind) + } + return d +} + +// liveConn awaits the current socket (staleness arms at socket open) and +// returns it. +func (h *harness) liveConn() *feedtest.Conn { + h.t.Helper() + h.awaitTimer(timerStaleness) + conn := h.tr.LastConn() + if conn == nil { + h.t.Fatal("no connection was dialed") + } + return conn +} + +// awaitBoundary waits for the confirmed subscription to reach the catch-up +// hand-off. +func (h *harness) awaitBoundary() eventfeed.CatchUpBoundary { + h.t.Helper() + select { + case b := <-h.boundary: + return b + case <-time.After(watchdog): + h.t.Fatal("the catch-up boundary was not reached within the watchdog") + return eventfeed.CatchUpBoundary{} + } +} + +// awaitStreaming consumes state notifications until Streaming is entered — +// the rendezvous for "the walk finished and the buffer drained", after which +// the walk's deliveries, saves, and timer set are all settled. +func (h *harness) awaitStreaming() { + h.t.Helper() + deadline := time.After(watchdog) + for { + select { + case s := <-h.states: + if s == stateStreaming { + return + } + case <-deadline: + h.t.Fatalf("state %q was not entered within the watchdog", stateStreaming) + } + } +} + +// deliveredIDs returns the ids delivered so far, in delivery order. +func (h *harness) deliveredIDs() []int64 { + events, _, _ := h.snapshot() + ids := make([]int64, len(events)) + for i, ev := range events { + ids[i] = ev.ID + } + return ids +} + +// serveSettled serves frames and returns once the pump has taken them all: +// a trailing ping is served last, so an empty conn queue proves every earlier +// frame already reached the state machine's hand-off queue (the pump reads, +// resets staleness, and hands off in that order). +func (h *harness) serveSettled(conn *feedtest.Conn, frames ...[]byte) { + h.t.Helper() + for _, f := range frames { + conn.Serve(f) + } + conn.Serve(framePing()) + h.waitUntil("pump took the served frames", func() bool { return conn.Pending() == 0 }) +} + +// awaitFrameHandled consumes handled-frame notifications until the given +// kind is seen. +func (h *harness) awaitFrameHandled(kind string) { + h.t.Helper() + deadline := time.After(watchdog) + for { + select { + case k := <-h.handled: + if k == kind { + return + } + case <-deadline: + h.t.Fatalf("frame kind %q was not handled within the watchdog", kind) + } + } +} + +// waitUntil polls cond under a bounded watchdog — a rendezvous on fake-side +// state (never on virtual time). +func (h *harness) waitUntil(desc string, cond func() bool) { + h.t.Helper() + deadline := time.Now().Add(watchdog) + for !cond() { + if time.Now().After(deadline) { + h.t.Fatalf("condition %q not reached within the watchdog", desc) + } + time.Sleep(time.Millisecond) + } +} + +// assertTimers asserts the exact outstanding timer multiset. +func assertTimers(t *testing.T, clock *feedtest.Clock, want map[string]int) { + t.Helper() + got := map[string]int{} + for _, name := range clock.Outstanding() { + got[name]++ + } + if len(got) != len(want) { + t.Fatalf("outstanding timers = %v, want %v", got, want) + } + for name, n := range want { + if got[name] != n { + t.Fatalf("outstanding timers = %v, want %v", got, want) + } + } +} + +// assertGoroutinesSettle waits (bounded) for the goroutine count to return +// to the pre-test base. +func assertGoroutinesSettle(t *testing.T, base int) { + t.Helper() + deadline := time.Now().Add(watchdog) + for { + if runtime.NumGoroutine() <= base { + return + } + if time.Now().After(deadline) { + t.Fatalf("goroutines did not settle: base %d, now %d", base, runtime.NumGoroutine()) + } + time.Sleep(10 * time.Millisecond) + } +} + +// Timer kind literals (SPEC.md §23, kebab-case, verbatim). +const ( + timerHandshakeDeadline = "handshake-deadline" + timerConfirmationDeadline = "confirmation-deadline" + timerBackoff = "backoff" + timerStaleness = "staleness" + timerRepairPoll = "repair-poll" + timerPollRetry = "poll-retry" +) + +// stateStreaming is the fixture spelling of the steady state. +const stateStreaming = "streaming" + +// ticket builds the nth scripted stream ticket; URLs are distinct per index +// so fresh-ticket assertions have teeth. +func ticket(n int) eventfeed.StreamTicket { + return eventfeed.StreamTicket{ + Ticket: fmt.Sprintf("ticket-%d", n), + ExpiresIn: 120, + URL: fmt.Sprintf("wss://cable.example.com/cable?ticket=ticket-%d", n), + } +} + +func frameWelcome() []byte { return []byte(`{"type":"welcome"}`) } + +func framePing() []byte { return []byte(`{"type":"ping","message":1690000000}`) } + +func frameConfirm(identifier string) []byte { + b, err := json.Marshal(map[string]string{"type": "confirm_subscription", "identifier": identifier}) + if err != nil { + panic(err) + } + return b +} + +func frameReject(identifier string) []byte { + b, err := json.Marshal(map[string]string{"type": "reject_subscription", "identifier": identifier}) + if err != nil { + panic(err) + } + return b +} + +func frameDisconnect(reason string, reconnect bool) []byte { + b, err := json.Marshal(map[string]any{"type": "disconnect", "reason": reason, "reconnect": reconnect}) + if err != nil { + panic(err) + } + return b +} + +func frameMessage(identifier string, id int64) []byte { + payload := map[string]any{ + "id": id, + "kind": "message", + "event_type": "message.created", + "action": "created", + "created_at": "2026-08-01T12:00:00Z", + "bucket_id": 2, + "creator_id": 3, + "recording_id": 900, + "visible_to_clients": false, + } + b, err := json.Marshal(map[string]any{"identifier": identifier, "message": payload}) + if err != nil { + panic(err) + } + return b +} + +var noFilterIdentifier = eventfeed.ExportSubscribeIdentifier(eventfeed.Filters{}) + +// driveToSubscribed mints, dials, serves welcome, and waits for the +// subscribe write (the confirmation deadline arms with it). +func (h *harness) driveToSubscribed() *feedtest.Conn { + h.t.Helper() + conn := h.liveConn() + conn.Serve(frameWelcome()) + h.awaitTimer(timerConfirmationDeadline) + return conn +} + +// TestHappyPathReachesCatchUpBoundary drives mint → dial → welcome → +// subscribe → confirm and asserts the confirmed subscription reaches the +// slice-4b boundary with the right entry cursor, exact subscribe bytes, a +// verbatim dial of the mint's URL, and only the staleness timer surviving +// (fixture 01's lifecycle prefix). +func TestHappyPathReachesCatchUpBoundary(t *testing.T) { + var mu sync.Mutex + var lifecycle []string + obs := eventfeed.Observer{ + Connecting: func(attempt int, delay time.Duration) { + mu.Lock() + lifecycle = append(lifecycle, fmt.Sprintf("connecting %d %s", attempt, delay)) + mu.Unlock() + }, + Connected: func() { mu.Lock(); lifecycle = append(lifecycle, "connected"); mu.Unlock() }, + Confirmed: func() { mu.Lock(); lifecycle = append(lifecycle, "confirmed"); mu.Unlock() }, + } + h := newHarness(t, eventfeed.WithObserver(obs)) + h.minter.ScriptTicket(ticket(1)) + // The confirmed subscription walks straight into catch-up (catchup.go); + // this test's subject is the lifecycle prefix, so the walk is one empty + // page at the frozen head. + h.polls.ScriptPage(eventfeed.PollPage{Position: "pos-1"}) + base := runtime.NumGoroutine() + h.start() + + conn := h.driveToSubscribed() + if got, want := h.tr.DialedURLs(), []string{ticket(1).URL}; len(got) != 1 || got[0] != want[0] { + t.Fatalf("dialed URLs = %v, want %v (the mint's url, verbatim)", got, want) + } + writes := conn.Writes() + if len(writes) != 1 || string(writes[0]) != string(eventfeed.ExportSubscribeFrame(eventfeed.Filters{})) { + t.Fatalf("subscribe writes = %q, want exactly %q", writes, eventfeed.ExportSubscribeFrame(eventfeed.Filters{})) + } + + conn.Serve(frameConfirm(noFilterIdentifier)) + b := h.awaitBoundary() + if b.Entry != (eventfeed.Cursor{}) { + t.Fatalf("entry cursor = %+v, want the zero Cursor (bare present entry)", b.Entry) + } + if !b.PresentClass { + t.Fatal("entry should be present-class") + } + if len(b.Buffered) != 0 { + t.Fatalf("buffered = %v, want none", b.Buffered) + } + if got := h.minter.Calls(); got != 1 { + t.Fatalf("mint seam calls = %d, want 1", got) + } + h.awaitStreaming() + assertTimers(t, h.clock, map[string]int{timerStaleness: 1, timerRepairPoll: 1}) + + mu.Lock() + wantLifecycle := []string{"connecting 1 0s", "connected", "confirmed"} + if len(lifecycle) != len(wantLifecycle) || lifecycle[0] != wantLifecycle[0] || lifecycle[1] != wantLifecycle[1] || lifecycle[2] != wantLifecycle[2] { + mu.Unlock() + t.Fatalf("observer lifecycle = %v, want %v", lifecycle, wantLifecycle) + } + mu.Unlock() + + h.conn.Close() + h.join() + if _, terminal, elements := h.snapshot(); terminal != nil || elements != 0 { + t.Fatalf("clean close must end iteration with no elements; got %d elements, terminal %v", elements, terminal) + } + if !conn.Closed() { + t.Fatal("the connector must close the socket on Close") + } + assertTimers(t, h.clock, map[string]int{}) + assertGoroutinesSettle(t, base) +} + +// TestConfirmationGating is acceptance (b) (fixture 02): a message frame +// landing before confirm_subscription is buffered — zero deliveries, zero +// poll seam calls — and the buffered event is carried into the catch-up +// hand-off. +func TestConfirmationGating(t *testing.T) { + h := newHarness(t) + h.minter.ScriptTicket(ticket(1)) + h.polls.ScriptPage(eventfeed.PollPage{Position: "pos-1"}) + h.start() + + conn := h.driveToSubscribed() + conn.Serve(frameMessage(noFilterIdentifier, 201)) + h.awaitFrameHandled("message") + + if events, terminal, _ := h.snapshot(); len(events) != 0 || terminal != nil { + t.Fatalf("nothing may be delivered before confirmation; got %v / %v", events, terminal) + } + if got := h.polls.CallCount(); got != 0 { + t.Fatalf("poll seam calls before confirmation = %d, want 0", got) + } + assertTimers(t, h.clock, map[string]int{timerConfirmationDeadline: 1, timerStaleness: 1}) + + conn.Serve(frameConfirm(noFilterIdentifier)) + b := h.awaitBoundary() + if len(b.Buffered) != 1 || b.Buffered[0].ID != 201 { + t.Fatalf("buffered at hand-off = %v, want exactly event 201", b.Buffered) + } + // Confirmation is what unblocks the walk: the entry poll happens only + // now, and the buffered event drains exactly once. + h.awaitStreaming() + if got := h.polls.CallCount(); got != 1 { + t.Fatalf("poll seam calls after confirmation = %d, want the one entry poll", got) + } + assertIDs(t, h.deliveredIDs(), 201) + + h.conn.Close() + h.join() + if _, terminal, elements := h.snapshot(); terminal != nil || elements != 1 { + t.Fatalf("clean close must end iteration with no error element; got %d elements, terminal %v", elements, terminal) + } +} + +// TestConfirmationDeadlineTeardown is acceptance (c) (fixture 03): the +// lapsed confirmation deadline disposes the whole attempt — socket closed, +// exactly {backoff} outstanding — and the retry mints a FRESH ticket and +// dials the new URL. +func TestConfirmationDeadlineTeardown(t *testing.T) { + h := newHarness(t) + h.minter.ScriptTicket(ticket(1)) + h.minter.ScriptTicket(ticket(2)) + base := runtime.NumGoroutine() + h.start() + + conn1 := h.driveToSubscribed() + if d := h.fireTimer(timerConfirmationDeadline); d != eventfeed.DefaultConfirmationDeadline { + t.Fatalf("confirmation deadline armed for %s, want %s", d, eventfeed.DefaultConfirmationDeadline) + } + h.awaitTimer(timerBackoff) + if !conn1.Closed() { + t.Fatal("deadline teardown must close the socket") + } + assertTimers(t, h.clock, map[string]int{timerBackoff: 1}) + + if d := h.fireTimer(timerBackoff); d < 0 || d > time.Second { + t.Fatalf("first backoff delay = %s, want within [0, 1s] (full jitter, n=1)", d) + } + conn2 := h.driveToSubscribed() + if got := h.minter.Calls(); got != 2 { + t.Fatalf("mint seam calls = %d, want 2 (a fresh ticket on every pass)", got) + } + urls := h.tr.DialedURLs() + if len(urls) != 2 || urls[0] != ticket(1).URL || urls[1] != ticket(2).URL { + t.Fatalf("dialed URLs = %v, want the two mints' URLs in order", urls) + } + assertTimers(t, h.clock, map[string]int{timerConfirmationDeadline: 1, timerStaleness: 1}) + + h.conn.Close() + h.join() + if !conn2.Closed() { + t.Fatal("Close must close the live socket") + } + assertGoroutinesSettle(t, base) +} + +// TestTerminalRejection is acceptance (d) (fixture 04): reject_subscription +// is always terminal — the connector explicitly closes the still-open +// socket, surfaces subscription_rejected as the single error element, makes +// zero reconnect seam calls, and leaves the timer set empty. +func TestTerminalRejection(t *testing.T) { + h := newHarness(t) + h.minter.ScriptTicket(ticket(1)) + h.start() + + conn := h.driveToSubscribed() + conn.Serve(frameReject(noFilterIdentifier)) + h.join() + + _, terminal, elements := h.snapshot() + if terminal == nil || terminal.Reason != eventfeed.ReasonSubscriptionRejected { + t.Fatalf("terminal = %v, want reason %q", terminal, eventfeed.ReasonSubscriptionRejected) + } + if elements != 1 { + t.Fatalf("iteration elements = %d, want exactly the one error element", elements) + } + if !conn.Closed() { + t.Fatal("the connector must explicitly close the rejected socket (Action Cable leaves it open)") + } + if got := h.minter.Calls(); got != 1 { + t.Fatalf("mint seam calls = %d, want 1 (ZERO reconnect attempts)", got) + } + if got := len(h.tr.Dials()); got != 1 { + t.Fatalf("connect seam calls = %d, want 1", got) + } + assertTimers(t, h.clock, map[string]int{}) +} + +// TestHandshakeDeadline covers transition 9 (no welcome before the deadline) +// and transition 7's deadline-expiry-mid-dial (the pending dial is +// cancelled); both recover through Backoff with a fresh ticket. +func TestHandshakeDeadline(t *testing.T) { + t.Run("no welcome", func(t *testing.T) { + h := newHarness(t) + h.minter.ScriptTicket(ticket(1)) + h.minter.ScriptTicket(ticket(2)) + h.start() + + conn1 := h.liveConn() + if d := h.fireTimer(timerHandshakeDeadline); d != 10*time.Second { + t.Fatalf("handshake deadline armed for %s, want 10s", d) + } + h.awaitTimer(timerBackoff) + if !conn1.Closed() { + t.Fatal("handshake-deadline teardown must close the socket") + } + assertTimers(t, h.clock, map[string]int{timerBackoff: 1}) + h.fireTimer(timerBackoff) + h.liveConn() + if got := h.minter.Calls(); got != 2 { + t.Fatalf("mint seam calls = %d, want 2", got) + } + }) + + t.Run("mid-dial", func(t *testing.T) { + h := newHarness(t) + h.minter.ScriptTicket(ticket(1)) + h.minter.ScriptTicket(ticket(2)) + h.tr.StallNextDial() + h.start() + + // The deadline arms on entry to Connecting, BEFORE dial. + h.fireTimer(timerHandshakeDeadline) + h.awaitTimer(timerBackoff) + if got := len(h.tr.Dials()); got != 1 { + t.Fatalf("connect seam calls = %d, want 1 (the stalled dial still counts)", got) + } + h.fireTimer(timerBackoff) + h.liveConn() + urls := h.tr.DialedURLs() + if len(urls) != 2 || urls[1] != ticket(2).URL { + t.Fatalf("dialed URLs = %v, want the second mint's URL on retry", urls) + } + }) +} + +// TestProtocolFatalDisconnect (fixture 06 + the state-generic rule): a raw +// disconnect frame with reason invalid_event_stream_command is +// Terminal(protocol_fatal) from both pre-confirm socket-open states, never +// retried into. +func TestProtocolFatalDisconnect(t *testing.T) { + for _, tc := range []struct { + name string + welcome bool + }{ + {"awaiting confirmation", true}, + {"awaiting welcome", false}, + } { + t.Run(tc.name, func(t *testing.T) { + h := newHarness(t) + h.minter.ScriptTicket(ticket(1)) + h.start() + + var conn *feedtest.Conn + if tc.welcome { + conn = h.driveToSubscribed() + } else { + conn = h.liveConn() + } + conn.Serve(frameDisconnect("invalid_event_stream_command", false)) + h.join() + + _, terminal, _ := h.snapshot() + if terminal == nil || terminal.Reason != eventfeed.ReasonProtocolFatal { + t.Fatalf("terminal = %v, want reason %q", terminal, eventfeed.ReasonProtocolFatal) + } + if !conn.Closed() { + t.Fatal("the connector must close the socket") + } + if got := h.minter.Calls(); got != 1 { + t.Fatalf("mint seam calls = %d, want 1 (zero further mints)", got) + } + assertTimers(t, h.clock, map[string]int{}) + }) + } +} + +// TestUnauthorizedDisconnectReconnects (fixture 07): the same +// reconnect:false flag as protocol-fatal, the opposite response — dispatch +// is on the REASON string. A pre-welcome `unauthorized` disconnect retries +// with a fresh ticket. +func TestUnauthorizedDisconnectReconnects(t *testing.T) { + h := newHarness(t) + h.minter.ScriptTicket(ticket(1)) + h.minter.ScriptTicket(ticket(2)) + h.start() + + conn1 := h.liveConn() + conn1.Serve(frameDisconnect("unauthorized", false)) + h.awaitTimer(timerBackoff) + if !conn1.Closed() { + t.Fatal("teardown must close the socket") + } + assertTimers(t, h.clock, map[string]int{timerBackoff: 1}) + h.fireTimer(timerBackoff) + + h.driveToSubscribed() + if got := h.minter.Calls(); got != 2 { + t.Fatalf("mint seam calls = %d, want 2 (fresh-ticket retry, not terminal)", got) + } + urls := h.tr.DialedURLs() + if len(urls) != 2 || urls[1] != ticket(2).URL { + t.Fatalf("dialed URLs = %v, want the fresh mint's URL second", urls) + } + if _, terminal, _ := h.snapshot(); terminal != nil { + t.Fatalf("unauthorized below the threshold must not be terminal; got %v", terminal) + } +} + +// TestRemoteDisconnectRemints (fixture 17's lifecycle half): a `remote` / +// reconnect:true disconnect rides the existing Backoff transitions — +// teardown, backoff, fresh mint, reconnect to the new URL. +func TestRemoteDisconnectRemints(t *testing.T) { + h := newHarness(t) + h.minter.ScriptTicket(ticket(1)) + h.minter.ScriptTicket(ticket(2)) + h.start() + + conn1 := h.driveToSubscribed() + conn1.Serve(frameDisconnect("remote", true)) + h.awaitTimer(timerBackoff) + if !conn1.Closed() { + t.Fatal("teardown must close the socket") + } + h.fireTimer(timerBackoff) + h.driveToSubscribed() + urls := h.tr.DialedURLs() + if len(urls) != 2 || urls[1] != ticket(2).URL { + t.Fatalf("dialed URLs = %v, want a fresh mint's URL on reconnect", urls) + } + if _, terminal, _ := h.snapshot(); terminal != nil { + t.Fatalf("`remote` must not be terminal; got %v", terminal) + } +} + +// TestAuthorizationFailureThreshold: the shared connection-level counter — +// unauthorized mints and `unauthorized` disconnects — surfaces +// authorization_failed on the 3rd consecutive failure (transitions 4/5, +// 9/10), including across mixed shapes. +func TestAuthorizationFailureThreshold(t *testing.T) { + unauthorized := func() error { + return &eventfeed.MintError{Kind: eventfeed.MintUnauthorized, Err: errors.New("401")} + } + + t.Run("three unauthorized mints", func(t *testing.T) { + h := newHarness(t) + h.minter.ScriptError(unauthorized()) + h.minter.ScriptError(unauthorized()) + h.minter.ScriptError(unauthorized()) + h.start() + + h.fireTimer(timerBackoff) // after mint 1 + h.fireTimer(timerBackoff) // after mint 2 + h.join() // mint 3 → terminal + + _, terminal, _ := h.snapshot() + if terminal == nil || terminal.Reason != eventfeed.ReasonAuthorizationFailed { + t.Fatalf("terminal = %v, want reason %q", terminal, eventfeed.ReasonAuthorizationFailed) + } + if got := h.minter.Calls(); got != 3 { + t.Fatalf("mint seam calls = %d, want 3", got) + } + if got := len(h.tr.Dials()); got != 0 { + t.Fatalf("connect seam calls = %d, want 0", got) + } + assertTimers(t, h.clock, map[string]int{}) + }) + + t.Run("mixed mint and disconnect failures share the counter", func(t *testing.T) { + h := newHarness(t) + h.minter.ScriptError(unauthorized()) // failure 1 + h.minter.ScriptTicket(ticket(2)) + h.minter.ScriptError(unauthorized()) // failure 3 + h.start() + + h.fireTimer(timerBackoff) + conn := h.liveConn() + conn.Serve(frameDisconnect("unauthorized", false)) // failure 2 + h.fireTimer(timerBackoff) + h.join() // third failure → terminal + + _, terminal, _ := h.snapshot() + if terminal == nil || terminal.Reason != eventfeed.ReasonAuthorizationFailed { + t.Fatalf("terminal = %v, want reason %q (one SHARED counter)", terminal, eventfeed.ReasonAuthorizationFailed) + } + if got := h.minter.Calls(); got != 3 { + t.Fatalf("mint seam calls = %d, want 3", got) + } + }) +} + +// TestMintFailureClassification: transient/throttled ride Backoff +// (Retry-After floors the delay and wins outright past the cap); +// unrecoverable — and unclassified — mint errors are Terminal(mint_failed) +// with the generated error attached. +func TestMintFailureClassification(t *testing.T) { + t.Run("throttled Retry-After floors the backoff", func(t *testing.T) { + h := newHarness(t) + h.minter.ScriptError(&eventfeed.MintError{Kind: eventfeed.MintThrottled, RetryAfter: 30 * time.Second}) + h.start() + if d := h.fireTimer(timerBackoff); d != 30*time.Second { + t.Fatalf("backoff delay = %s, want exactly the 30s Retry-After floor", d) + } + }) + + t.Run("Retry-After beyond the cap wins outright", func(t *testing.T) { + h := newHarness(t) + h.minter.ScriptError(&eventfeed.MintError{Kind: eventfeed.MintThrottled, RetryAfter: 120 * time.Second}) + h.start() + if d := h.fireTimer(timerBackoff); d != 120*time.Second { + t.Fatalf("backoff delay = %s, want 120s (server-directed waits are cap-exempt)", d) + } + }) + + t.Run("transient rides backoff to a fresh mint", func(t *testing.T) { + h := newHarness(t) + h.minter.ScriptError(&eventfeed.MintError{Kind: eventfeed.MintTransient, Err: errors.New("boom")}) + h.minter.ScriptTicket(ticket(2)) + h.start() + h.fireTimer(timerBackoff) + h.liveConn() + if got := h.minter.Calls(); got != 2 { + t.Fatalf("mint seam calls = %d, want 2", got) + } + }) + + t.Run("unrecoverable is terminal with the cause attached", func(t *testing.T) { + sentinel := errors.New("422 unprocessable") + h := newHarness(t) + h.minter.ScriptError(&eventfeed.MintError{Kind: eventfeed.MintUnrecoverable, Err: sentinel}) + h.start() + h.join() + _, terminal, _ := h.snapshot() + if terminal == nil || terminal.Reason != eventfeed.ReasonMintFailed { + t.Fatalf("terminal = %v, want reason %q", terminal, eventfeed.ReasonMintFailed) + } + if !errors.Is(terminal, sentinel) { + t.Fatalf("terminal must wrap the generated error; got %v", terminal) + } + assertTimers(t, h.clock, map[string]int{}) + }) + + t.Run("an unclassified mint error is terminal", func(t *testing.T) { + h := newHarness(t) + h.minter.ScriptError(errors.New("adapter bug: unclassified")) + h.start() + h.join() + _, terminal, _ := h.snapshot() + if terminal == nil || terminal.Reason != eventfeed.ReasonMintFailed { + t.Fatalf("terminal = %v, want reason %q", terminal, eventfeed.ReasonMintFailed) + } + }) +} + +// TestInvalidCableURL: a mint URL failing cable-URL policy — and a +// policy-kind dial error — are Terminal(invalid_cable_url), never Backoff; +// a transient dial failure rides Backoff to a fresh mint (transition 7). +func TestInvalidCableURL(t *testing.T) { + t.Run("non-wss scheme, zero dials", func(t *testing.T) { + h := newHarness(t) + h.minter.ScriptTicket(eventfeed.StreamTicket{Ticket: "t", ExpiresIn: 120, URL: "https://cable.example.com/cable"}) + h.start() + h.join() + _, terminal, _ := h.snapshot() + if terminal == nil || terminal.Reason != eventfeed.ReasonInvalidCableURL { + t.Fatalf("terminal = %v, want reason %q", terminal, eventfeed.ReasonInvalidCableURL) + } + if got := len(h.tr.Dials()); got != 0 { + t.Fatalf("connect seam calls = %d, want 0 (policy pre-check refuses the dial)", got) + } + assertTimers(t, h.clock, map[string]int{}) + }) + + t.Run("ws outside loopback, zero dials", func(t *testing.T) { + h := newHarness(t) + h.minter.ScriptTicket(eventfeed.StreamTicket{Ticket: "t", ExpiresIn: 120, URL: "ws://cable.example.com/cable"}) + h.start() + h.join() + if _, terminal, _ := h.snapshot(); terminal == nil || terminal.Reason != eventfeed.ReasonInvalidCableURL { + t.Fatalf("terminal = %v, want reason %q", terminal, eventfeed.ReasonInvalidCableURL) + } + }) + + t.Run("policy dial error is terminal", func(t *testing.T) { + h := newHarness(t) + h.minter.ScriptTicket(ticket(1)) + h.tr.FailNextDial(&eventfeed.DialError{Kind: eventfeed.DialPolicy, Reason: "redirect encountered"}) + h.start() + h.join() + if _, terminal, _ := h.snapshot(); terminal == nil || terminal.Reason != eventfeed.ReasonInvalidCableURL { + t.Fatalf("terminal = %v, want reason %q", terminal, eventfeed.ReasonInvalidCableURL) + } + if got := len(h.tr.Dials()); got != 1 { + t.Fatalf("connect seam calls = %d, want 1", got) + } + assertTimers(t, h.clock, map[string]int{}) + }) + + t.Run("transient dial failure rides backoff", func(t *testing.T) { + h := newHarness(t) + h.minter.ScriptTicket(ticket(1)) + h.minter.ScriptTicket(ticket(2)) + h.tr.FailNextDial(errors.New("connection refused")) + h.start() + h.fireTimer(timerBackoff) + h.liveConn() + if got := h.minter.Calls(); got != 2 { + t.Fatalf("mint seam calls = %d, want 2 (fresh ticket after a failed dial)", got) + } + urls := h.tr.DialedURLs() + if len(urls) != 2 || urls[1] != ticket(2).URL { + t.Fatalf("dialed URLs = %v, want the fresh mint's URL second", urls) + } + }) +} + +// TestInvalidFramesPreConfirm: the invalid-frame class (unparseable JSON, a +// correlated message failing Event decode) dispatches as a SOCKET FAILURE — +// Backoff, never terminal — while unknown types, pings, and +// foreign-identifier confirm/reject frames are ignored. +func TestInvalidFramesPreConfirm(t *testing.T) { + t.Run("unparseable frame tears down to backoff", func(t *testing.T) { + var mu sync.Mutex + var disconnects []error + obs := eventfeed.Observer{Disconnected: func(reason string, err error) { + mu.Lock() + disconnects = append(disconnects, err) + mu.Unlock() + }} + h := newHarness(t, eventfeed.WithObserver(obs)) + h.minter.ScriptTicket(ticket(1)) + h.minter.ScriptTicket(ticket(2)) + h.start() + + conn := h.liveConn() + conn.Serve([]byte("not json at all")) + h.awaitTimer(timerBackoff) + if !conn.Closed() { + t.Fatal("an invalid frame must tear the socket down") + } + if _, terminal, _ := h.snapshot(); terminal != nil { + t.Fatalf("an invalid frame is never terminal; got %v", terminal) + } + mu.Lock() + if len(disconnects) != 1 || disconnects[0] == nil || !strings.Contains(disconnects[0].Error(), "invalid inbound frame") { + mu.Unlock() + t.Fatalf("Disconnected must carry the invalid-frame indication; got %v", disconnects) + } + mu.Unlock() + h.fireTimer(timerBackoff) + h.liveConn() + if got := h.minter.Calls(); got != 2 { + t.Fatalf("mint seam calls = %d, want 2 (the reconnect cycle recovers)", got) + } + }) + + t.Run("correlated message failing event decode tears down", func(t *testing.T) { + h := newHarness(t) + h.minter.ScriptTicket(ticket(1)) + h.minter.ScriptTicket(ticket(2)) + h.start() + + conn := h.driveToSubscribed() + bad, err := json.Marshal(map[string]any{"identifier": noFilterIdentifier, "message": map[string]any{"kind": "message"}}) + if err != nil { + t.Fatal(err) + } + conn.Serve(bad) + h.awaitTimer(timerBackoff) + if _, terminal, _ := h.snapshot(); terminal != nil { + t.Fatalf("a decode failure is never terminal; got %v", terminal) + } + if !conn.Closed() { + t.Fatal("teardown must close the socket") + } + }) + + t.Run("unknown types, pings, and foreign identifiers are ignored", func(t *testing.T) { + h := newHarness(t) + h.minter.ScriptTicket(ticket(1)) + h.start() + + conn := h.driveToSubscribed() + conn.Serve([]byte(`{"type":"transmissions_ahoy"}`)) + conn.Serve(framePing()) + conn.Serve(frameConfirm(`{"channel":"SomeOtherChannel"}`)) + conn.Serve(frameReject(`{"channel":"SomeOtherChannel"}`)) + h.awaitFrameHandled("reject_subscription") + select { + case b := <-h.boundary: + t.Fatalf("a foreign-identifier confirm must not confirm; boundary %v", b) + default: + } + if _, terminal, _ := h.snapshot(); terminal != nil { + t.Fatalf("a foreign-identifier reject must be ignored; got %v", terminal) + } + conn.Serve(frameConfirm(noFilterIdentifier)) + h.awaitBoundary() + }) +} + +// TestNullFrameTearsDownToBackoff: a top-level `null` frame is the +// invalid-frame class's parse shape, not an unknown-type frame. Classified as +// unknown it would be liveness-only, so a peer sending nothing but `null` +// could hold the socket open forever — the pump re-arms staleness before the +// frame is parsed — while delivering no protocol traffic. The disposition is +// the class's: socket failure, Backoff, never terminal. +func TestNullFrameTearsDownToBackoff(t *testing.T) { + var mu sync.Mutex + var disconnects []error + obs := eventfeed.Observer{Disconnected: func(_ string, err error) { + mu.Lock() + disconnects = append(disconnects, err) + mu.Unlock() + }} + h := newHarness(t, eventfeed.WithObserver(obs)) + h.minter.ScriptTicket(ticket(1)) + h.minter.ScriptTicket(ticket(2)) + h.start() + + conn := h.liveConn() + conn.Serve([]byte(`null`)) + h.awaitTimer(timerBackoff) + if !conn.Closed() { + t.Fatal("a null frame must tear the socket down") + } + if _, terminal, _ := h.snapshot(); terminal != nil { + t.Fatalf("an invalid frame is never terminal; got %v", terminal) + } + mu.Lock() + defer mu.Unlock() + if len(disconnects) != 1 || disconnects[0] == nil || !eventfeed.ExportIsInvalidFrameError(disconnects[0]) { + t.Fatalf("Disconnected must carry the invalid-frame indication; got %v", disconnects) + } +} + +// TestFrameDerivedObserverTextIsBounded: both frame-derived strings +// Disconnected can carry — a raw disconnect frame's reason, and the +// invalid-frame rendering of a decoder error that quotes frame bytes — are +// bounded by §9's MAX_ERROR_MESSAGE_LENGTH, which §23's Security Invariants +// apply to any rendering of frame contents. Unbounded, either is a 1 MiB +// attacker-chosen string in the consumer's logs. +func TestFrameDerivedObserverTextIsBounded(t *testing.T) { + oversized := strings.Repeat("a", 4096) + + t.Run("disconnect reason", func(t *testing.T) { + var mu sync.Mutex + var reasons []string + obs := eventfeed.Observer{Disconnected: func(reason string, _ error) { + mu.Lock() + reasons = append(reasons, reason) + mu.Unlock() + }} + h := newHarness(t, eventfeed.WithObserver(obs)) + h.minter.ScriptTicket(ticket(1)) + h.minter.ScriptTicket(ticket(2)) + h.start() + + conn := h.liveConn() + conn.Serve(frameDisconnect(oversized, true)) + h.awaitTimer(timerBackoff) + if _, terminal, _ := h.snapshot(); terminal != nil { + t.Fatalf("an unrecognized reason is a socket drop, never terminal; got %v", terminal) + } + mu.Lock() + defer mu.Unlock() + if len(reasons) != 1 { + t.Fatalf("Disconnected calls = %d, want 1", len(reasons)) + } + if got := len(reasons[0]); got > 500 { + t.Fatalf("observed reason length = %d bytes, want at most 500", got) + } + }) + + t.Run("event decode error", func(t *testing.T) { + var mu sync.Mutex + var disconnects []error + obs := eventfeed.Observer{Disconnected: func(_ string, err error) { + mu.Lock() + disconnects = append(disconnects, err) + mu.Unlock() + }} + h := newHarness(t, eventfeed.WithObserver(obs)) + h.minter.ScriptTicket(ticket(1)) + h.minter.ScriptTicket(ticket(2)) + h.start() + + conn := h.driveToSubscribed() + // time.Time's decoder embeds the offending value in its parse error. + bad, err := json.Marshal(map[string]any{ + "identifier": noFilterIdentifier, + "message": map[string]any{ + "id": 1, "kind": "message", "event_type": "message.created", + "action": "created", "created_at": oversized, + "bucket_id": 2, "creator_id": 3, "recording_id": 4, + }, + }) + if err != nil { + t.Fatal(err) + } + conn.Serve(bad) + h.awaitTimer(timerBackoff) + mu.Lock() + defer mu.Unlock() + if len(disconnects) != 1 || disconnects[0] == nil { + t.Fatalf("Disconnected must report the decode failure; got %v", disconnects) + } + if got := len(disconnects[0].Error()); got > 500 { + t.Fatalf("rendered error length = %d bytes, want at most 500", got) + } + // Bounding only the outermost rendering is not a fix: the observer + // holds the error, and errors.Unwrap walks straight back to the raw + // time.Time parse error carrying the full frame-derived value. An + // error carrying frame-derived text is FLAT (redactDialErr's + // precedent) — the whole chain is checked, not just its head. + for err := disconnects[0]; err != nil; err = errors.Unwrap(err) { + if strings.Contains(err.Error(), oversized) { + t.Fatalf("the error chain re-exposes the frame-supplied value at %T", err) + } + if got := len(err.Error()); got > 500 { + t.Fatalf("chained error %T renders %d bytes, want at most 500", err, got) + } + } + // The invalid-frame classification the tier-2 driver matches on must + // survive the flattening. + if !eventfeed.ExportIsInvalidFrameError(disconnects[0]) { + t.Fatalf("Disconnected error %T lost its invalid-frame classification", disconnects[0]) + } + }) +} + +// TestDuplicateWelcomeResendsSubscribe: subscribe is sent on each welcome, +// byte-identical (the server absorbs identical retransmits). +func TestDuplicateWelcomeResendsSubscribe(t *testing.T) { + h := newHarness(t) + h.minter.ScriptTicket(ticket(1)) + h.start() + + conn := h.driveToSubscribed() + conn.Serve(frameWelcome()) + h.waitUntil("second subscribe write", func() bool { return len(conn.Writes()) == 2 }) + writes := conn.Writes() + if string(writes[0]) != string(writes[1]) { + t.Fatalf("resubscribe must be byte-identical: %q vs %q", writes[0], writes[1]) + } + assertTimers(t, h.clock, map[string]int{timerConfirmationDeadline: 1, timerStaleness: 1}) + conn.Serve(frameConfirm(noFilterIdentifier)) + h.awaitBoundary() +} + +// TestSubscribeWriteFailure: a failed subscribe write takes the socket- +// failure path — Backoff, never terminal. +func TestSubscribeWriteFailure(t *testing.T) { + h := newHarness(t) + h.minter.ScriptTicket(ticket(1)) + h.start() + + conn := h.liveConn() + conn.FailWrites(errors.New("broken pipe")) + conn.Serve(frameWelcome()) + h.awaitTimer(timerBackoff) + if !conn.Closed() { + t.Fatal("teardown must close the socket") + } + if _, terminal, _ := h.snapshot(); terminal != nil { + t.Fatalf("a write failure is never terminal; got %v", terminal) + } +} + +// TestStaleness: the staleness timer (armed at socket open, reset pump-side +// per inbound frame of any kind) tears the socket down on expiry — and a +// frame genuinely re-arms it. +func TestStaleness(t *testing.T) { + t.Run("expiry pre-welcome tears down", func(t *testing.T) { + var mu sync.Mutex + var stale []time.Duration + obs := eventfeed.Observer{StaleConnection: func(since time.Duration) { + mu.Lock() + stale = append(stale, since) + mu.Unlock() + }} + h := newHarness(t, eventfeed.WithObserver(obs)) + h.minter.ScriptTicket(ticket(1)) + h.start() + + conn := h.liveConn() + h.clock.Advance(7500 * time.Millisecond) + h.awaitTimer(timerBackoff) + if !conn.Closed() { + t.Fatal("staleness must tear the socket down") + } + assertTimers(t, h.clock, map[string]int{timerBackoff: 1}) + mu.Lock() + if len(stale) != 1 || stale[0] < 7500*time.Millisecond { + mu.Unlock() + t.Fatalf("StaleConnection = %v, want one callback with >= 7.5s of silence", stale) + } + mu.Unlock() + }) + + t.Run("frames reset the window", func(t *testing.T) { + h := newHarness(t, eventfeed.WithConfirmationDeadline(time.Hour)) + h.minter.ScriptTicket(ticket(1)) + h.start() + + conn := h.driveToSubscribed() + h.clock.Advance(5 * time.Second) + conn.Serve(framePing()) + h.awaitFrameHandled("ping") + h.clock.Advance(5 * time.Second) + // 10s since socket open, but only 5s since the ping: still alive. + assertTimers(t, h.clock, map[string]int{timerConfirmationDeadline: 1, timerStaleness: 1}) + h.clock.Advance(2500 * time.Millisecond) + h.awaitTimer(timerBackoff) + if !conn.Closed() { + t.Fatal("the reset window's expiry must still tear down") + } + }) + + // SPEC.md §23 "Cable Protocol Details": the reset happens pump-side, at + // frame receipt, "so frame-vs-deadline ordering is well-defined at the + // transport boundary regardless of queue depth or consumer latency: a + // fired staleness deadline observed on return from a slow delivery is + // AUTHORITATIVE". A frame received after the deadline fired is not one the + // pump "received first", so it cannot supersede the firing — and the state + // machine, parked inside the delivery, has not had a chance to observe it + // yet. The pump is reading throughout here (the queue never fills), so the + // suspension rule does not apply and the window testifies. + t.Run("a frame arriving after the deadline does not supersede it", func(t *testing.T) { + var mu sync.Mutex + var stale []time.Duration + obs := eventfeed.Observer{StaleConnection: func(since time.Duration) { + mu.Lock() + stale = append(stale, since) + mu.Unlock() + }} + h := newHarness(t, eventfeed.WithObserver(obs)) + h.pauseAfter = 1 + h.minter.ScriptTicket(ticket(1)) + h.minter.ScriptTicket(ticket(2)) + h.polls.ScriptPage(eventfeed.PollPage{Position: "pos-1"}) + h.start() + + conn := h.driveToSubscribed() + conn.Serve(frameConfirm(noFilterIdentifier)) + h.awaitStreaming() + + // The consumer parks inside the delivery of the first live event, so + // the deadline fires with the state machine unable to observe it. + conn.Serve(frameMessage(noFilterIdentifier, 1)) + h.waitUntil("the consumer parked mid-delivery", func() bool { return len(h.deliveredIDs()) == 1 }) + h.clock.Advance(staleAfter) + // The late frame lands after the firing, before control returns. + h.serveSettled(conn, framePing()) + h.resume() + + h.awaitTimer(timerBackoff) + if !conn.Closed() { + t.Fatal("a frame received after the deadline fired must not erase the expiry") + } + assertTimers(t, h.clock, map[string]int{timerBackoff: 1}) + if _, terminal, _ := h.snapshot(); terminal != nil { + t.Fatalf("staleness is never terminal; got %v", terminal) + } + mu.Lock() + defer mu.Unlock() + if len(stale) != 1 || stale[0] < staleAfter { + t.Fatalf("StaleConnection = %v, want one callback with >= %s of silence", stale, staleAfter) + } + }) +} + +// TestBufferOverflowPreConfirm: overflow of the live buffer dispatches the +// BufferOverflow semantic signal at drop time — default-terminal with no +// handler, terminal on Terminate, continuing on Accept with the invocation +// recorded (SPEC.md §23 "Semantic Signals"). +func TestBufferOverflowPreConfirm(t *testing.T) { + serveThree := func(h *harness) *feedtest.Conn { + conn := h.driveToSubscribed() + conn.Serve(frameMessage(noFilterIdentifier, 1)) + conn.Serve(frameMessage(noFilterIdentifier, 2)) + conn.Serve(frameMessage(noFilterIdentifier, 3)) + return conn + } + + t.Run("no handler is terminal", func(t *testing.T) { + var droppedCounts []int + var mu sync.Mutex + obs := eventfeed.Observer{BufferOverflow: func(n int) { + mu.Lock() + droppedCounts = append(droppedCounts, n) + mu.Unlock() + }} + h := newHarness(t, eventfeed.WithLiveBufferCapacity(2), eventfeed.WithObserver(obs)) + h.minter.ScriptTicket(ticket(1)) + h.start() + conn := serveThree(h) + h.join() + _, terminal, _ := h.snapshot() + if terminal == nil || terminal.Reason != eventfeed.ReasonBufferOverflow { + t.Fatalf("terminal = %v, want reason %q", terminal, eventfeed.ReasonBufferOverflow) + } + if !conn.Closed() { + t.Fatal("teardown must close the socket") + } + mu.Lock() + if len(droppedCounts) != 1 || droppedCounts[0] != 1 { + mu.Unlock() + t.Fatalf("Observer.BufferOverflow = %v, want one callback for 1 drop", droppedCounts) + } + mu.Unlock() + assertTimers(t, h.clock, map[string]int{}) + }) + + t.Run("handler Accept continues with the invocation recorded", func(t *testing.T) { + var mu sync.Mutex + var signals []eventfeed.Signal + handler := func(s eventfeed.Signal) eventfeed.Disposition { + mu.Lock() + signals = append(signals, s) + mu.Unlock() + return eventfeed.Accept + } + h := newHarness(t, eventfeed.WithLiveBufferCapacity(2), eventfeed.WithSignalHandler(handler)) + h.minter.ScriptTicket(ticket(1)) + h.start() + conn := serveThree(h) + h.awaitFrameHandled("message") + h.awaitFrameHandled("message") + h.awaitFrameHandled("message") + conn.Serve(frameConfirm(noFilterIdentifier)) + b := h.awaitBoundary() + if len(b.Buffered) != 2 || b.Buffered[0].ID != 2 || b.Buffered[1].ID != 3 { + t.Fatalf("buffered = %v, want the newest two events (oldest dropped)", b.Buffered) + } + mu.Lock() + defer mu.Unlock() + if len(signals) != 1 { + t.Fatalf("handler invocations = %d, want exactly 1 per signal", len(signals)) + } + ov, ok := signals[0].(eventfeed.BufferOverflow) + if !ok || ov.DroppedCount != 1 || len(ov.DroppedIDs) != 1 || ov.DroppedIDs[0] != 1 { + t.Fatalf("signal = %+v, want BufferOverflow{DroppedIDs:[1], DroppedCount:1}", signals[0]) + } + }) + + t.Run("handler Terminate is terminal", func(t *testing.T) { + var invocations int + var mu sync.Mutex + handler := func(s eventfeed.Signal) eventfeed.Disposition { + mu.Lock() + invocations++ + mu.Unlock() + return eventfeed.Terminate + } + h := newHarness(t, eventfeed.WithLiveBufferCapacity(2), eventfeed.WithSignalHandler(handler)) + h.minter.ScriptTicket(ticket(1)) + h.start() + serveThree(h) + h.join() + _, terminal, _ := h.snapshot() + if terminal == nil || terminal.Reason != eventfeed.ReasonBufferOverflow { + t.Fatalf("terminal = %v, want reason %q", terminal, eventfeed.ReasonBufferOverflow) + } + mu.Lock() + defer mu.Unlock() + if invocations != 1 { + t.Fatalf("handler invocations = %d, want 1 (Terminate is the handler's verdict, not a bypass)", invocations) + } + }) +} + +// TestStartModes: each entry mode maps to its SPEC-defined cursor and +// present-class flag at the catch-up hand-off. +func TestStartModes(t *testing.T) { + cases := []struct { + name string + start eventfeed.Start + cursor eventfeed.Cursor + present bool + }{ + {"resume without a store is the bare present", eventfeed.StartResume(), eventfeed.Cursor{}, true}, + {"present", eventfeed.StartPresent(), eventfeed.Cursor{Since: "now"}, true}, + {"beginning", eventfeed.StartBeginning(), eventfeed.Cursor{Since: "0"}, false}, + {"after id", eventfeed.StartAfter(42), eventfeed.Cursor{Since: "42"}, false}, + {"at position", eventfeed.StartAtPosition("pos-1"), eventfeed.Cursor{Position: "pos-1"}, false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + h := newHarness(t, eventfeed.WithStart(tc.start)) + h.minter.ScriptTicket(ticket(1)) + h.start() + conn := h.driveToSubscribed() + conn.Serve(frameConfirm(noFilterIdentifier)) + b := h.awaitBoundary() + if b.Entry != tc.cursor || b.PresentClass != tc.present { + t.Fatalf("boundary = {%+v present=%v}, want {%+v present=%v}", b.Entry, b.PresentClass, tc.cursor, tc.present) + } + }) + } +} + +// TestFilteredSubscribeIdentifier: the subscribe command carries the ordered, +// comma-joined filter params, byte-exact, and confirmation correlates on +// that identifier. +func TestFilteredSubscribeIdentifier(t *testing.T) { + filters := eventfeed.Filters{Types: []string{"message.created", "todo.completed"}, Buckets: []int64{2, 1}} + h := newHarness(t, eventfeed.WithFilters(filters)) + h.minter.ScriptTicket(ticket(1)) + h.start() + + conn := h.driveToSubscribed() + want := string(eventfeed.ExportSubscribeFrame(filters)) + if !strings.Contains(want, `message.created,todo.completed`) || !strings.Contains(want, `2,1`) { + t.Fatalf("test premise: subscribe frame %q should join filters in configured order", want) + } + writes := conn.Writes() + if len(writes) != 1 || string(writes[0]) != want { + t.Fatalf("subscribe = %q, want %q", writes, want) + } + conn.Serve(frameConfirm(eventfeed.ExportSubscribeIdentifier(filters))) + h.awaitBoundary() +} + +// TestTeardownCancelsInFlight: Close (and caller cancellation) during an +// in-flight mint, a stalled dial, and an awaited confirmation each end +// iteration cleanly — no error element, no surviving timers, no leaked +// goroutines. +func TestTeardownCancelsInFlight(t *testing.T) { + t.Run("mid-mint", func(t *testing.T) { + h := newHarness(t) + h.minter.StallNext() + base := runtime.NumGoroutine() + h.start() + h.waitUntil("mint in flight", func() bool { return h.minter.Calls() == 1 }) + h.conn.Close() + h.join() + if _, terminal, elements := h.snapshot(); terminal != nil || elements != 0 { + t.Fatalf("clean close: got %d elements, terminal %v", elements, terminal) + } + if got := len(h.tr.Dials()); got != 0 { + t.Fatalf("connect seam calls = %d, want 0", got) + } + assertTimers(t, h.clock, map[string]int{}) + assertGoroutinesSettle(t, base) + }) + + t.Run("mid-dial", func(t *testing.T) { + h := newHarness(t) + h.minter.ScriptTicket(ticket(1)) + h.tr.StallNextDial() + base := runtime.NumGoroutine() + h.start() + h.waitUntil("dial in flight", func() bool { return len(h.tr.Dials()) == 1 }) + h.conn.Close() + h.join() + if _, terminal, elements := h.snapshot(); terminal != nil || elements != 0 { + t.Fatalf("clean close: got %d elements, terminal %v", elements, terminal) + } + assertTimers(t, h.clock, map[string]int{}) + assertGoroutinesSettle(t, base) + }) + + t.Run("mid-await-confirmation", func(t *testing.T) { + h := newHarness(t) + h.minter.ScriptTicket(ticket(1)) + base := runtime.NumGoroutine() + h.start() + conn := h.driveToSubscribed() + h.conn.Close() + h.join() + if !conn.Closed() { + t.Fatal("Close must close the live socket") + } + if _, terminal, elements := h.snapshot(); terminal != nil || elements != 0 { + t.Fatalf("clean close: got %d elements, terminal %v", elements, terminal) + } + assertTimers(t, h.clock, map[string]int{}) + assertGoroutinesSettle(t, base) + }) + + t.Run("caller cancellation mid-await", func(t *testing.T) { + h := newHarness(t) + h.minter.ScriptTicket(ticket(1)) + base := runtime.NumGoroutine() + h.start() + conn := h.driveToSubscribed() + h.cancel() + h.join() + if !conn.Closed() { + t.Fatal("cancellation must close the live socket") + } + if _, terminal, elements := h.snapshot(); terminal != nil || elements != 0 { + t.Fatalf("clean cancel: got %d elements, terminal %v", elements, terminal) + } + assertTimers(t, h.clock, map[string]int{}) + assertGoroutinesSettle(t, base) + }) +} + +// TestStartModesWithAPopulatedStore: only StartResume is defined as "the +// stored position if any" (SPEC.md §23 "Consumer Ergonomics"). StartPresent, +// StartBeginning and StartAfter promise since=now, since=0 and since=, +// and a caller pairing one with a configured store means it — otherwise every +// explicit mode silently becomes resume the moment the store has anything in +// it, and a checkpointed feed can never be deliberately replayed or reset. +// The load still runs under those modes (its failure edge and its lineage +// identity are not mode-dependent); the value simply is not what the entry is +// taken from. +func TestStartModesWithAPopulatedStore(t *testing.T) { + cases := []struct { + name string + start eventfeed.Start + cursor eventfeed.Cursor + present bool + }{ + {"resume takes the stored position", eventfeed.StartResume(), eventfeed.Cursor{Position: "pos-0"}, false}, + {"present ignores it", eventfeed.StartPresent(), eventfeed.Cursor{Since: "now"}, true}, + {"beginning ignores it", eventfeed.StartBeginning(), eventfeed.Cursor{Since: "0"}, false}, + {"after id ignores it", eventfeed.StartAfter(42), eventfeed.Cursor{Since: "42"}, false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + store := feedtest.NewStore() + store.Stored("pos-0") + h := storedHarness(t, store, eventfeed.WithStart(tc.start)) + h.minter.ScriptTicket(ticket(1)) + h.start() + conn := h.driveToSubscribed() + conn.Serve(frameConfirm(noFilterIdentifier)) + b := h.awaitBoundary() + if b.Entry != tc.cursor || b.PresentClass != tc.present { + t.Fatalf("boundary = {%+v present=%v}, want {%+v present=%v}", b.Entry, b.PresentClass, tc.cursor, tc.present) + } + if got := len(store.Loads()); got != 1 { + t.Fatalf("checkpoint loads = %d, want exactly 1 — the load runs under every mode", got) + } + }) + } + + // The other half of the rule: a position ACCEPTED during this run is + // authoritative within the run, so a reconnect under an explicit mode + // resumes where the feed actually got to rather than re-entering at the + // mode's original cursor. + t.Run("a position accepted during the run wins on reconnect", func(t *testing.T) { + store := feedtest.NewStore() + store.Stored("pos-0") + h := storedHarness(t, store, eventfeed.WithStart(eventfeed.StartPresent())) + h.minter.ScriptTicket(ticket(1)) + h.minter.ScriptTicket(ticket(2)) + h.polls.ScriptPage(eventfeed.PollPage{Position: "pos-1"}) + h.polls.ScriptPage(eventfeed.PollPage{Position: "pos-2"}) + h.start() + + conn := h.driveToSubscribed() + conn.Serve(frameConfirm(noFilterIdentifier)) + h.awaitStreaming() + conn.FailReads(errors.New("connection reset")) + h.awaitTimer(timerBackoff) + h.fireTimer(timerBackoff) + conn2 := h.driveToSubscribed() + conn2.Serve(frameConfirm(noFilterIdentifier)) + h.awaitStreaming() + + calls := h.polls.Calls() + if len(calls) != 2 { + t.Fatalf("poll seam calls = %d, want 2", len(calls)) + } + if calls[0].Cursor != (eventfeed.Cursor{Since: "now"}) { + t.Fatalf("entry cursor = %+v, want the configured present entry", calls[0].Cursor) + } + if calls[1].Cursor != (eventfeed.Cursor{Position: "pos-1"}) { + t.Fatalf("reconnect cursor = %+v, want the position accepted during this run", calls[1].Cursor) + } + }) +} + +// TestRejectBeforeSubscribeIsIgnored gates transition 12 on the state §23 +// draws it from. `reject_subscription` is the connector's single most severe +// verdict — Terminal, ZERO reconnects, no backoff — and ungated it can be +// produced by one unsolicited frame from a peer that has done nothing but +// complete the WebSocket handshake, against zero subscription attempts. The +// neighbouring confirm branch already gates on AwaitingConfirmation; this is +// the same gate on the same transition's other half. +func TestRejectBeforeSubscribeIsIgnored(t *testing.T) { + h := newHarness(t) + h.minter.ScriptTicket(ticket(1)) + h.polls.ScriptPage(eventfeed.PollPage{Position: "pos-1"}) + h.start() + + conn := h.liveConn() + // The connector's OWN identifier, so correlation is not what saves it, + // and before any subscribe command has been written. + conn.Serve(frameReject(noFilterIdentifier)) + h.awaitFrameHandled("reject_subscription") + if _, terminal, _ := h.snapshot(); terminal != nil { + t.Fatalf("terminal = %v, want none — transition 12 exists only from AwaitingConfirmation", terminal) + } + if conn.Closed() { + t.Fatal("a premature rejection must not tear the socket down") + } + + // The handshake then proceeds normally on the same socket. + conn.Serve(frameWelcome()) + h.awaitTimer(timerConfirmationDeadline) + conn.Serve(frameConfirm(noFilterIdentifier)) + h.awaitStreaming() + if _, terminal, _ := h.snapshot(); terminal != nil { + t.Fatalf("terminal = %v, want none", terminal) + } +} + +// TestStalledSubscribeWriteStillLapsesTheHandshakeDeadline bounds the one +// place the state machine hands control to the transport synchronously while +// a deadline governs the phase. A CableConn.WriteFrame that blocks — a peer +// whose receive window has shut, or a seam implementation that returns only +// on cancellation — cannot be interrupted from inside the write: at.ctx is +// cancelled by a teardown the write is itself blocking, so neither the +// handshake nor the confirmation deadline can ever be observed and the feed +// hangs after `welcome` until the consumer closes the connector. +func TestStalledSubscribeWriteStillLapsesTheHandshakeDeadline(t *testing.T) { + h := newHarness(t) + h.minter.ScriptTicket(ticket(1)) + h.minter.ScriptTicket(ticket(2)) + h.start() + + conn := h.liveConn() + conn.StallWrites() + conn.Serve(frameWelcome()) + // The frame-handled hook fires before the switch, so past it the only + // path is the subscribe write: firing the deadline here cannot be taken + // by the outer select instead. + h.awaitFrameHandled("welcome") + h.fireTimer(timerHandshakeDeadline) + + h.awaitTimer(timerBackoff) + if !conn.Closed() { + t.Fatal("the lapse must dispose the attempt, closing the socket") + } + if _, terminal, _ := h.snapshot(); terminal != nil { + t.Fatalf("a handshake lapse is never terminal; got %v", terminal) + } + assertTimers(t, h.clock, map[string]int{timerBackoff: 1}) +} + +// TestWelcomeRacingAnExpiredHandshakeDeadlineLapses reads Stop's result as +// the ordering discriminator, the same way staleHolder.arm does. When the +// welcome frame and the handshake expiry are both ready the select picks one +// at random; if the frame wins, an unchecked Stop swallows the firing and +// replaces the expired handshake timer with a fresh confirmation window — so +// a welcome that arrived past the deadline is accepted instead of taking +// transition 9, half the time it happens. +// +// The hook fires in exactly that window (write returned, timer not yet +// stopped), which is the only way to make the race deterministic from +// outside: serving the frame and firing the timer from the test leaves the +// select genuinely free to pick either. +func TestWelcomeRacingAnExpiredHandshakeDeadlineLapses(t *testing.T) { + h := newHarness(t) + h.minter.ScriptTicket(ticket(1)) + h.minter.ScriptTicket(ticket(2)) + fired := make(chan struct{}, 1) + h.conn.OnSubscribeWritten(func() { + if _, ok := h.clock.FireTimer(timerHandshakeDeadline); ok { + select { + case fired <- struct{}{}: + default: + } + } + }) + h.start() + + conn := h.liveConn() + conn.Serve(frameWelcome()) + select { + case <-fired: + case <-time.After(watchdog): + t.Fatal("the handshake deadline was never fired inside the write→Stop window") + } + + h.awaitTimer(timerBackoff) + if !conn.Closed() { + t.Fatal("a lapsed handshake must dispose the attempt, closing the socket") + } + if _, terminal, _ := h.snapshot(); terminal != nil { + t.Fatalf("a handshake lapse is never terminal; got %v", terminal) + } + // The decisive assertion: no confirmation-deadline was armed over the + // expired handshake timer. + assertTimers(t, h.clock, map[string]int{timerBackoff: 1}) +} + +// TestConfirmRacingAnExpiredConfirmationDeadlineLapses is the confirmation +// half of the Stop-as-ordering-discriminator rule, and the more consequential +// half. Where an unchecked Stop on the handshake deadline merely swallowed the +// firing, here the connector would go on to announce Confirmed and enter +// catch-up on a subscription whose confirmation arrived past transition 14's +// deadline — the lapse not just swallowed but overwritten by a successful +// handoff. +// +// The frame-handled hook fires before the switch, so firing the deadline from +// it lands in the window between the frame being classified and the Stop that +// cancels it: the same window the select leaves open when both are ready, made +// deterministic. +func TestConfirmRacingAnExpiredConfirmationDeadlineLapses(t *testing.T) { + h := newHarness(t) + h.minter.ScriptTicket(ticket(1)) + h.minter.ScriptTicket(ticket(2)) + h.polls.ScriptPage(eventfeed.PollPage{Position: "pos-1"}) + fired := make(chan struct{}, 1) + h.conn.OnFrameHandled(func(kind string) { + if kind != "confirm_subscription" { + return + } + if _, ok := h.clock.FireTimer(timerConfirmationDeadline); ok { + select { + case fired <- struct{}{}: + default: + } + } + }) + h.start() + + conn := h.driveToSubscribed() + conn.Serve(frameConfirm(noFilterIdentifier)) + select { + case <-fired: + case <-time.After(watchdog): + t.Fatal("the confirmation deadline was never fired inside the classify→Stop window") + } + + h.awaitTimer(timerBackoff) + if !conn.Closed() { + t.Fatal("a lapsed confirmation must dispose the attempt, closing the socket") + } + if _, terminal, _ := h.snapshot(); terminal != nil { + t.Fatalf("a confirmation lapse is never terminal; got %v", terminal) + } + // The decisive assertion: catch-up was never entered, so neither the + // staleness nor the repair-poll timer survives. + assertTimers(t, h.clock, map[string]int{timerBackoff: 1}) + if got := h.polls.CallCount(); got != 0 { + t.Fatalf("poll seam calls = %d, want 0 — the confirmation lapsed", got) + } +} diff --git a/go/pkg/basecamp/eventfeed/reconnect_test.go b/go/pkg/basecamp/eventfeed/reconnect_test.go new file mode 100644 index 0000000000..7b2750193e --- /dev/null +++ b/go/pkg/basecamp/eventfeed/reconnect_test.go @@ -0,0 +1,453 @@ +// Slice-4c acceptance tests: the reconnect cycle now that catch-up, the +// entry boundary, and streaming exist — fresh-ticket reconnect (acceptance +// (a)), the full-jitter backoff envelope and its counter discipline, what a +// reconnect preserves versus rebuilds, and the staleness policy's evaluation +// rule (SPEC.md §23 transitions 2/4/7/9/14/15/21/25 and the addenda's +// pump-side reset, full-queue suspension, and Draining deferral). Structured +// to mirror tier-2 fixtures 05, 07, and 17; all time flows through the +// injected feedtest.Clock, and mint/connect/poll counts count seam calls. +package eventfeed_test + +import ( + "errors" + "runtime" + "sync" + "testing" + "time" + + "github.com/basecamp/basecamp-sdk/go/pkg/basecamp/eventfeed" + "github.com/basecamp/basecamp-sdk/go/pkg/basecamp/eventfeed/feedtest" +) + +// staleAfter is SPEC §23's pinned staleness window (EVENT_FEED_STALE_AFTER). +const staleAfter = 7500 * time.Millisecond + +// TestFreshTicketReconnectAfterTTL is acceptance (a) (fixture 05): a stream +// runs past the first ticket's server-owned expires_in — which must schedule +// nothing client-side — and is then severed. The reconnect pass mints a FRESH +// ticket and dials the SECOND mint's URL, and one catch-up poll from the +// in-memory position precedes any live delivery. Staleness and repair-poll +// are configured out of the window, as the fixture does, so the sever is the +// only thing that ends the socket. +func TestFreshTicketReconnectAfterTTL(t *testing.T) { + store := feedtest.NewStore() + store.Stored("pos-0") + h := storedHarness(t, store, eventfeed.WithRepairInterval(999_999*time.Second)) + h.conn.SetStaleAfter(999_999 * time.Second) + h.minter.ScriptTicket(ticket(1)) + h.minter.ScriptTicket(ticket(2)) + h.polls.ScriptPage(eventfeed.PollPage{Position: "pos-1"}) + h.polls.ScriptPage(eventfeed.PollPage{Position: "pos-2"}) + base := runtime.NumGoroutine() + h.start() + + conn1 := h.driveToSubscribed() + conn1.Serve(frameConfirm(noFilterIdentifier)) + h.awaitStreaming() + assertPositions(t, store.Saves(), "pos-1") + + // 121s of virtual silence: past the ~120s ticket TTL, which is + // server-owned and never used for client-side scheduling — nothing is due. + h.clock.Advance(121 * time.Second) + if got := h.minter.Calls(); got != 1 { + t.Fatalf("mint seam calls after the TTL elapsed = %d, want 1 (expires_in schedules nothing)", got) + } + + conn1.FailReads(errors.New("connection reset by peer")) + h.awaitTimer(timerBackoff) + if !conn1.Closed() { + t.Fatal("the sever must dispose the attempt's socket") + } + // The whole attempt is disposed before Backoff is entered: no repair-poll + // or staleness timer survives it. + assertTimers(t, h.clock, map[string]int{timerBackoff: 1}) + if d := h.fireTimer(timerBackoff); d < 0 || d > time.Second { + t.Fatalf("first backoff delay = %s, want within [0, 1s] (full jitter, n=1)", d) + } + + conn2 := h.driveToSubscribed() + conn2.Serve(frameConfirm(noFilterIdentifier)) + h.awaitStreaming() + + urls := h.tr.DialedURLs() + if len(urls) != 2 || urls[0] != ticket(1).URL || urls[1] != ticket(2).URL { + t.Fatalf("dialed URLs = %v, want each mint's url verbatim, in order", urls) + } + if got := h.minter.Calls(); got != 2 { + t.Fatalf("mint seam calls = %d, want 2 (a fresh ticket on EVERY pass)", got) + } + calls := h.polls.Calls() + if len(calls) != 2 || calls[1].Cursor != (eventfeed.Cursor{Position: "pos-1"}) { + t.Fatalf("poll calls = %+v, want the reconnect walking from the in-memory position", calls) + } + assertPositions(t, store.Saves(), "pos-1", "pos-2") + if got := len(store.Loads()); got != 1 { + t.Fatalf("store loads = %d, want 1 (the store is never re-read within a run)", got) + } + assertIDs(t, h.deliveredIDs()) + assertTimers(t, h.clock, map[string]int{timerStaleness: 1, timerRepairPoll: 1}) + if conn2.Closed() { + t.Fatal("the reconnected socket must be open") + } + + h.conn.Close() + h.join() + assertGoroutinesSettle(t, base) +} + +// TestReconnectResumesFromTheInMemoryPosition (fixture 17's catch-up half): a +// `remote` disconnect re-enters at the position the connector already holds — +// not the stored one, and not the configured Start mode — as a POSITION-RESUME +// entry, with the delivered-id LRU carried across the reconnect so a re-served +// page repeats nothing. +func TestReconnectResumesFromTheInMemoryPosition(t *testing.T) { + store := feedtest.NewStore() + store.Stored("pos-0") + h := storedHarness(t, store) + h.minter.ScriptTicket(ticket(1)) + h.minter.ScriptTicket(ticket(2)) + h.polls.ScriptPage(eventfeed.PollPage{Events: []eventfeed.Event{pollEvent(101)}, Position: "pos-1"}) + // The re-entry page re-serves 101 (it sits at the safety horizon) plus 102. + h.polls.ScriptPage(eventfeed.PollPage{ + Events: []eventfeed.Event{pollEvent(101), pollEvent(102)}, + Position: "pos-2", + }) + h.start() + + conn1 := h.driveToSubscribed() + conn1.Serve(frameConfirm(noFilterIdentifier)) + h.awaitStreaming() + first := h.awaitBoundary() + if first.Entry != (eventfeed.Cursor{Position: "pos-0"}) || first.PresentClass { + t.Fatalf("first entry = {%+v present=%v}, want the loaded position, position-resume", first.Entry, first.PresentClass) + } + + conn1.Serve(frameDisconnect("remote", true)) + h.awaitTimer(timerBackoff) + h.fireTimer(timerBackoff) + conn2 := h.driveToSubscribed() + conn2.Serve(frameConfirm(noFilterIdentifier)) + h.awaitStreaming() + + second := h.awaitBoundary() + if second.Entry != (eventfeed.Cursor{Position: "pos-1"}) || second.PresentClass { + t.Fatalf("re-entry = {%+v present=%v}, want the in-memory position, position-resume", second.Entry, second.PresentClass) + } + if got := len(store.Loads()); got != 1 { + t.Fatalf("store loads = %d, want 1 (in-memory position is authoritative for resume within a run)", got) + } + assertIDs(t, h.deliveredIDs(), 101, 102) + assertLedger(t, h.ledger(), []string{"event 101", "save pos-1", "event 102", "save pos-2"}) +} + +// TestReconnectCarriesTheAdmittedStragglerIntoTheNextEntry: the live buffer is +// state-machine-owned, not socket-owned. An event admitted before the first +// socket died is still the only carrier of that straggler — the poll lane +// never serves an id at or below a present-class entry's cursor — so it is +// delivered on the next attempt BEFORE that entry's held position saves. The +// conjunctive save-ordering invariant holds across the reconnect. +func TestReconnectCarriesTheAdmittedStragglerIntoTheNextEntry(t *testing.T) { + store := feedtest.NewStore() // Missing → a present-class entry + h := storedHarness(t, store) + h.minter.ScriptTicket(ticket(1)) + h.minter.ScriptTicket(ticket(2)) + h.polls.ScriptPage(eventfeed.PollPage{Position: "pos-1"}) + h.start() + + conn1 := h.driveToSubscribed() + h.serveSettled(conn1, frameMessage(noFilterIdentifier, 41)) + h.awaitFrameHandled("message") + conn1.FailReads(errors.New("connection reset by peer")) + h.awaitTimer(timerBackoff) + if got := h.polls.CallCount(); got != 0 { + t.Fatalf("poll seam calls = %d, want 0 (the first attempt never confirmed)", got) + } + h.fireTimer(timerBackoff) + + conn2 := h.driveToSubscribed() + conn2.Serve(frameConfirm(noFilterIdentifier)) + h.awaitStreaming() + + b := h.awaitBoundary() + if len(b.Buffered) != 1 || b.Buffered[0].ID != 41 { + t.Fatalf("buffered at the second hand-off = %v, want the straggler admitted before the first socket died", b.Buffered) + } + assertLedger(t, h.ledger(), []string{"event 41", "save pos-1"}) + assertIDs(t, h.deliveredIDs(), 41) +} + +// TestBackoffEnvelopeGrowsAndResetsOnConfirmation: the reconnect-cycle delay +// is full-jitter over the failed-cycle count n — uniform(0, min(60s, 1s × +// 2^(n−1))) — so with the draw pinned at its upper bound the envelope doubles +// per failed cycle; confirmation resets that counter (transition 11) and the +// next failure starts at n=1 again. +func TestBackoffEnvelopeGrowsAndResetsOnConfirmation(t *testing.T) { + h := newHarness(t) + // A degenerate always-1 draw makes the envelope itself assertable; the + // formula's bounds are pinned in backoff_test.go. + h.conn.SetRand(func() float64 { return 1 }) + for i := 1; i <= 5; i++ { + h.minter.ScriptTicket(ticket(i)) + } + for i := 0; i < 3; i++ { + h.tr.FailNextDial(errors.New("dial tcp: connection refused")) + } + h.polls.ScriptPage(eventfeed.PollPage{Position: "pos-1"}) + h.start() + + for _, want := range []time.Duration{time.Second, 2 * time.Second, 4 * time.Second} { + if d := h.fireTimer(timerBackoff); d != want { + t.Fatalf("backoff delay = %s, want %s (full-jitter envelope, doubling per failed cycle)", d, want) + } + } + + conn := h.driveToSubscribed() + conn.Serve(frameConfirm(noFilterIdentifier)) + h.awaitStreaming() + if got := len(h.tr.Dials()); got != 4 { + t.Fatalf("connect seam calls = %d, want 4 (three refused dials, then the live one)", got) + } + + conn.FailReads(errors.New("connection reset by peer")) + if d := h.fireTimer(timerBackoff); d != time.Second { + t.Fatalf("post-confirmation backoff delay = %s, want 1s — confirmation resets the failed-cycle count", d) + } +} + +// TestStalenessSuspendedWhileThePumpIsBlocked is the G-R16/17/18 evaluation +// rule: staleness is suspended while the pump is blocked on a full hand-off +// queue — a full queue is a fast peer, not a dead one, and a pump that is not +// reading cannot observe the resets that would prove it. Suspension is an +// EVALUATION rule, not a timer-set change: the timer stays armed, and a firing +// whose window overlapped a pump-blocked interval is disregarded and re-armed +// rather than dispatched. A firing over a window the pump spent reading is +// authoritative. +func TestStalenessSuspendedWhileThePumpIsBlocked(t *testing.T) { + h := newHarness(t) + blocked := make(chan struct{}, 1) + // Blocks and releases are counted, not merely signalled: a release is what + // re-arms the window unsuspended, and the second half of the test is only + // meaningful once every block the pump took has been released. + var mu sync.Mutex + var blocks, releases int + h.conn.OnPumpBlocked(func() { + mu.Lock() + blocks++ + mu.Unlock() + select { + case blocked <- struct{}{}: + default: + } + }) + h.conn.OnPumpReleased(func() { + mu.Lock() + releases++ + mu.Unlock() + }) + h.pauseAfter = 1 + h.minter.ScriptTicket(ticket(1)) + h.polls.ScriptPage(eventfeed.PollPage{Position: "pos-1"}) + h.start() + + conn := h.driveToSubscribed() + conn.Serve(frameConfirm(noFilterIdentifier)) + h.awaitStreaming() + + // The consumer parks inside the delivery of the first live event, so the + // state machine stops dequeuing while the peer keeps sending. + conn.Serve(frameMessage(noFilterIdentifier, 1)) + h.waitUntil("the consumer parked mid-delivery", func() bool { return len(h.deliveredIDs()) == 1 }) + for i := 0; i <= eventfeed.ExportPumpDepth; i++ { + conn.Serve(framePing()) + } + select { + case <-blocked: + case <-time.After(watchdog): + t.Fatal("the pump never blocked on a full hand-off queue") + } + + h.clock.Advance(staleAfter) + h.resume() + + // The queue drains and the socket survives: the firing is disregarded, not + // dispatched. A dispatched firing disposes the attempt, so the socket + // closes and the queued frames are never handled — checked on every turn + // so that failure reports itself rather than timing out. + deadline := time.After(watchdog) + for pings := 0; pings <= eventfeed.ExportPumpDepth; { + if conn.Closed() { + t.Fatal("a staleness firing whose window overlapped a pump block must be disregarded, not dispatched") + } + select { + case kind := <-h.handled: + if kind == "ping" { + pings++ + } + case <-time.After(time.Millisecond): + case <-deadline: + t.Fatal("the hand-off queue never drained") + } + } + if _, terminal, _ := h.snapshot(); terminal != nil { + t.Fatalf("a suspended staleness firing is never terminal; got %v", terminal) + } + // The last ping being HANDLED only proves the state machine dequeued it — + // the dequeue is what unblocks the pump, so the pump's release still races + // this point, and the release is what re-arms the window unsuspended. An + // advance taken before it lands fires a window the release then supersedes + // on generation grounds, leaving nothing to tear the socket down. All the + // pings are handled here, so no further hand-off can block: once every + // block has been released, the armed window is settled. + h.waitUntil("every blocked hand-off was released", func() bool { + mu.Lock() + defer mu.Unlock() + return blocks > 0 && releases == blocks + }) + // Disregarded AND re-armed — the exact per-state set is unchanged. + assertTimers(t, h.clock, map[string]int{timerStaleness: 1, timerRepairPoll: 1}) + + // The pump is reading again: this window is authoritative. + h.clock.Advance(staleAfter) + h.awaitTimer(timerBackoff) + if !conn.Closed() { + t.Fatal("a firing over a reading window must tear the socket down") + } + assertTimers(t, h.clock, map[string]int{timerBackoff: 1}) +} + +// TestStalenessDuringDrainDefersToTheStreamingBoundary: Draining is a bounded +// in-memory replay with no failure edge of its own, so a staleness expiry +// observed while it runs is consumed at the Streaming boundary — transition 23 +// completes the drain (retained deliveries, then the held position's save and +// caught_up), and only then does transition 25 handle the failure. +func TestStalenessDuringDrainDefersToTheStreamingBoundary(t *testing.T) { + var mu sync.Mutex + var caughtUp int + obs := eventfeed.Observer{CaughtUp: func() { + mu.Lock() + caughtUp++ + mu.Unlock() + }} + store := feedtest.NewStore() + h := storedHarness(t, store, eventfeed.WithObserver(obs)) + h.pauseAfter = 1 + h.minter.ScriptTicket(ticket(1)) + h.minter.ScriptTicket(ticket(2)) + h.polls.ScriptPage(eventfeed.PollPage{Position: "pos-1"}) + h.start() + + conn := h.driveToSubscribed() + h.serveSettled(conn, frameMessage(noFilterIdentifier, 41)) + h.awaitFrameHandled("message") + conn.Serve(frameConfirm(noFilterIdentifier)) + + // The consumer parks inside the drain's delivery of the retained event. + h.waitUntil("the drain parked mid-delivery", func() bool { return len(h.deliveredIDs()) == 1 }) + h.clock.Advance(staleAfter) + h.resume() + + h.awaitTimer(timerBackoff) + assertLedger(t, h.ledger(), []string{"event 41", "save pos-1"}) + assertPositions(t, store.Saves(), "pos-1") + mu.Lock() + if caughtUp != 1 { + mu.Unlock() + t.Fatalf("caught_up announcements = %d, want 1 (the drain completes before the failure is handled)", caughtUp) + } + mu.Unlock() + if !conn.Closed() { + t.Fatal("the deferred staleness expiry must tear the socket down at the Streaming boundary") + } + assertTimers(t, h.clock, map[string]int{timerBackoff: 1}) + if _, terminal, _ := h.snapshot(); terminal != nil { + t.Fatalf("staleness is never terminal; got %v", terminal) + } +} + +// TestWalkFailureBetweenPages is transition 21 from inside the walk itself: a +// socket death and a staleness expiry that arise while a multi-page walk is in +// flight are both observed at the page boundary — the walk stops there rather +// than following `next` on a socket that is already gone — and the reconnect +// re-enters at the last accepted page's position with the saved checkpoints +// kept. +func TestWalkFailureBetweenPages(t *testing.T) { + // setup drives a two-page walk whose FIRST poll seam call runs interrupt + // while it is in flight, so the failure is queued for the state machine + // before the first page is even accepted — the walk's own page boundary is + // then the only place it can be observed. + setup := func(t *testing.T, interrupt func(h *harness, conn *feedtest.Conn, errQueued <-chan struct{})) *harness { + t.Helper() + store := feedtest.NewStore() + store.Stored("pos-0") + h := storedHarness(t, store) + h.minter.ScriptTicket(ticket(1)) + h.minter.ScriptTicket(ticket(2)) + next := testOrigin + "/999/events.json?after=101" + h.polls.ScriptPage(eventfeed.PollPage{ + Events: []eventfeed.Event{pollEvent(101)}, + Position: "pos-1", + Next: next, + }) + h.polls.ScriptPage(eventfeed.PollPage{Events: []eventfeed.Event{pollEvent(102)}, Position: "pos-2"}) + + errQueued := make(chan struct{}, 1) + h.conn.OnPumpHandedOff(func(isErr bool) { + if !isErr { + return + } + select { + case errQueued <- struct{}{}: + default: + } + }) + var conn *feedtest.Conn + var once sync.Once + h.polls.OnCall(func(feedtest.PollCall) { + once.Do(func() { interrupt(h, conn, errQueued) }) + }) + h.start() + conn = h.driveToSubscribed() + conn.Serve(frameConfirm(noFilterIdentifier)) + h.awaitTimer(timerBackoff) + + if got := h.polls.CallCount(); got != 1 { + t.Fatalf("poll seam calls = %d, want 1 — the walk must not follow `next` on a dead socket", got) + } + if !conn.Closed() { + t.Fatal("the mid-walk failure must dispose the attempt") + } + assertTimers(t, h.clock, map[string]int{timerBackoff: 1}) + if _, terminal, _ := h.snapshot(); terminal != nil { + t.Fatalf("a mid-walk failure is never terminal; got %v", terminal) + } + assertPositions(t, store.Saves(), "pos-1") + return h + } + + t.Run("socket drop", func(t *testing.T) { + setup(t, func(_ *harness, conn *feedtest.Conn, errQueued <-chan struct{}) { + conn.FailReads(errors.New("connection reset by peer")) + select { + case <-errQueued: + case <-time.After(watchdog): + t.Error("the pump never handed the socket failure to the state machine") + } + }) + }) + + t.Run("staleness expiry", func(t *testing.T) { + h := setup(t, func(h *harness, _ *feedtest.Conn, _ <-chan struct{}) { h.clock.Advance(staleAfter) }) + + // The reconnect re-enters at the accepted page's position, keeping the + // checkpoint the walk already saved. + h.fireTimer(timerBackoff) + conn := h.driveToSubscribed() + conn.Serve(frameConfirm(noFilterIdentifier)) + h.awaitStreaming() + calls := h.polls.Calls() + if len(calls) != 2 || calls[1].Cursor != (eventfeed.Cursor{Position: "pos-1"}) { + t.Fatalf("poll calls = %+v, want the re-entry at the last accepted position", calls) + } + assertIDs(t, h.deliveredIDs(), 101, 102) + }) +} diff --git a/go/pkg/basecamp/eventfeed/recovery.go b/go/pkg/basecamp/eventfeed/recovery.go new file mode 100644 index 0000000000..b5ecc508f3 --- /dev/null +++ b/go/pkg/basecamp/eventfeed/recovery.go @@ -0,0 +1,181 @@ +package eventfeed + +import ( + "fmt" + "strconv" +) + +// The recovery matrix: the repair poll a fired `repair-poll` timer drives +// (SPEC.md §23 transition 24) and the re-entries a poll error whose +// disposition is a NEW cursor takes (transitions 17/18/19 — 410 `gone`, +// 400-position, 409 `filter_changed`). Both run on the consumer's goroutine, +// inside the walk that reached them, so nothing here is concurrent with a +// delivery and the handler contract's "synchronously, on the consumer's +// execution context" is structural. +// +// The three re-entries share one shape and differ only in where the new +// cursor comes from: a 410's is the resume URL the SERVER provided, gated +// behind the FeedGap semantic signal; a 400-position's and a 409's is the +// poll-lane-only reset cursor below. All three restart the entry boundary — +// the walk drops any held position and re-takes the ownership cut on the new +// entry — because a re-entry IS an entry; only the positions already saved +// survive it. +// +// A re-entry re-polls IMMEDIATELY: §23 models rows 17/18/19 as CatchingUp +// self-loops with no wait of their own (only transients and throttles take +// the `poll-retry` timer), and the tier-2 gap fixtures script the re-entry +// poll with no intervening timer to fire. The connector therefore relies on +// the server not answering a reset cursor with the same rejection that +// produced it — a `since=` cursor carries no position token for a 400-position +// or 409 to be about, and a resume URL is the server's own. No local bound is +// imposed here: one would have to invent a threshold and a terminal reason +// §23 does not publish, and would diverge from the other five SDKs. + +// repairWalk is transition 24: one walk from the connector's current +// position, driven by a fired `repair-poll` timer. It is the same walk, drain +// and entry discipline the post-confirmation catch-up runs, so it returns to +// Streaming through Draining — which is what makes a live frame arriving +// mid-walk buffered rather than delivered (CatchingUp delivers poll pages +// only) and replayed on the way back. +// +// It reports done=true with the cycle's outcome when the walk ended the +// cycle; otherwise the caller re-arms the next jittered cadence and +// re-announces Streaming. +func (l *loop) repairWalk(at *attempt) (cycleOutcome, bool) { + cursor, presentClass := l.repairCursor() + l.setState(stateCatchingUp) + if l.cfg.observer.CatchUpStarted != nil { + l.cfg.observer.CatchUpStarted(cursor) + } + return l.walkThenDrain(at, cursor, presentClass) +} + +// repairCursor selects the repair walk's entry. The connector's IN-MEMORY +// position is authoritative for repair within the run — the store is never +// re-read after its one load, so a failed save neither regresses nor blanks +// what the repair polls from. With no position at all — an entry whose pages +// have not moved one, or one a 409 discarded — the repair re-enters at the +// present, which is present-class. +func (l *loop) repairCursor() (Cursor, bool) { + if l.position != "" { + return Cursor{Position: l.position}, false + } + return Cursor{Since: sincePresent}, true +} + +// reenterWalk dispatches the poll errors whose disposition is a new cursor +// (transitions 17/18/19). It returns the walk's next step, or done=true with +// the outcome that ended the cycle. +func (l *loop) reenterWalk(at *attempt, pe *PollError) (walkStep, cycleOutcome, bool) { + switch pe.Kind { + case PollGone: + return l.recoverGone(at, pe) + case PollFilterChanged: + // Transition 19: the held position is DISCARDED before the re-entry. + // Its lineage belongs to a filter set that is not this connector's, so + // it can never be resumed from again — an attempt torn down before the + // re-entry serves a page must not reconnect back onto it. (Transition + // 18 deliberately does not discard: §23 gives only the 409 that step, + // and a re-entry's first accepted page overwrites the in-memory cursor + // either way.) No store delete is needed or offered: the next page's + // save overwrites under the same key. + l.position = "" + return l.reenterAtResetCursor(pe.Kind), cycleOutcome{}, false + default: // PollPositionInvalid + // Transition 18: the position the server refused is replaced by the + // reset cursor; nothing durable moves and the socket is kept. + return l.reenterAtResetCursor(pe.Kind), cycleOutcome{}, false + } +} + +// recoverGone dispatches transition 17's FeedGap semantic signal. Observer.Gap +// fires first and unconditionally — observability is not disposition — and a +// REGISTERED handler is then invoked exactly once, synchronously, before its +// disposition takes effect. Accept re-enters via the URL the server provided, +// which the walk validates under §8 before issuing any request to it; +// Terminate and no-handler are both Terminal(feed_gap) with NO save. A 410 +// never silently auto-continues: with no handler the feed ends here rather +// than following `resume` on the consumer's behalf. +func (l *loop) recoverGone(at *attempt, pe *PollError) (walkStep, cycleOutcome, bool) { + signal := FeedGap{EpochAfterID: pe.EpochAfterID, ResumeURL: pe.ResumeURL} + if l.hooks.signalRaised != nil { + l.hooks.signalRaised(signal) + } + if l.cfg.observer.Gap != nil { + l.cfg.observer.Gap(pe.EpochAfterID, pe.ResumeURL) + } + if l.cfg.handler != nil && l.cfg.handler(signal) == Accept { + if pe.ResumeURL == "" { + // Accepting a 410 that carried no resume URL cannot be honored: + // the disposition names a URL the body did not supply, and a bare + // present entry is emphatically not what was accepted — it would + // silently reposition the feed at the present head. An empty + // string fails the same continuation validation every followed URL + // passes (it is not an absolute URL), so it takes that edge. + l.disposeAttempt(at, nil) + return walkStep{}, cycleOutcome{kind: outcomeTerminal, term: &TerminalError{ + Reason: ReasonInvalidContinuation, + Msg: "the 410 carried no resume URL to continue from", + }}, true + } + // The resume URL is a present-class entry (the server documents it as + // since=now with the canonical filter set preserved), so the entry + // boundary's hold-then-save discipline governs the pages it serves — + // and the old cursor is unusable, which is precisely the durability + // boundary the invariant's exclusion names. Unusable means DISCARDED, + // not merely superseded: leaving it in memory would let a socket torn + // down before the resume poll's first page re-select it on the next + // connection, drawing the same 410 and re-invoking a gap handler that + // has already been asked and has already answered Accept. The resume + // URL takes its place as latched reconnect state so the accepted + // present-class reset is what continues. + l.position = "" + l.reentry = &reentryCursor{cursor: Cursor{PageURL: pe.ResumeURL}, presentClass: true} + return walkStep{ + cursor: l.reentry.cursor, + presentClass: true, + reentry: true, + }, cycleOutcome{}, false + } + // Terminate, or no handler: the typed terminal, and no save — the error + // names the epoch and never the resume URL. + l.disposeAttempt(at, nil) + return walkStep{}, cycleOutcome{kind: outcomeTerminal, term: &TerminalError{ + Reason: ReasonFeedGap, + Msg: fmt.Sprintf("the feed's served history before event %d is gone", pe.EpochAfterID), + }}, true +} + +// reenterAtResetCursor builds the re-entry rows 18/19 share: `since=`, or `since=now` — a present-class entry — when the poll +// lane has served no id. +// +// The reset cursor is POLL-LANE-ONLY. A live-delivered id is never one: a +// live id above the durable position would re-enter past the un-polled gap +// behind it and permanently skip everything in between. Empty pages serve no +// ids and advance nothing. +// +// The selected cursor is LATCHED as reconnect state, not just handed to the +// walk. A reset cursor is the connector's whole answer to a rejected +// position, and it is chosen from run state (`lastPollServedID`) that no +// later entry re-derives: if the socket dies before the re-entry's first page +// lands, the next connection would otherwise fall back to a position the +// server refused — or, after a 409 discarded it, to the configured start +// mode, which for a resume feed is the bare present. That is a silent skip of +// everything between the poll-served id and the present, on a path whose +// entire purpose was not to skip anything. +func (l *loop) reenterAtResetCursor(kind PollErrorKind) walkStep { + if l.cfg.observer.PositionRejected != nil { + l.cfg.observer.PositionRejected(kind) + } + if l.lastPollServedID > 0 { + l.reentry = &reentryCursor{cursor: Cursor{Since: strconv.FormatInt(l.lastPollServedID, 10)}} + } else { + l.reentry = &reentryCursor{cursor: Cursor{Since: sincePresent}, presentClass: true} + } + return walkStep{ + cursor: l.reentry.cursor, + presentClass: l.reentry.presentClass, + reentry: true, + } +} diff --git a/go/pkg/basecamp/eventfeed/recovery_test.go b/go/pkg/basecamp/eventfeed/recovery_test.go new file mode 100644 index 0000000000..77b9127598 --- /dev/null +++ b/go/pkg/basecamp/eventfeed/recovery_test.go @@ -0,0 +1,760 @@ +// Slice-4d acceptance tests: the repair poll (SPEC.md §23 transition 24) and +// the re-entry matrix a poll's 410, 400-position, or 409 takes (transitions +// 17/18/19), plus the shared authorization counter's poll-lane share and the +// two-lane poll-retry timing. Structured to mirror tier-2 fixtures 16, 23, +// 25, 27, and 29's repair half — the 400/409 rows have no PR-2 fixture (they +// are pinned at PR 4), so their coverage here is written to the same shape. +package eventfeed_test + +import ( + "errors" + "fmt" + "runtime" + "strings" + "sync" + "testing" + "time" + + "github.com/basecamp/basecamp-sdk/go/pkg/basecamp/eventfeed" + "github.com/basecamp/basecamp-sdk/go/pkg/basecamp/eventfeed/feedtest" +) + +// signalLedger is the tier-2 driver's handlerInvocations record expressed as a +// Go SignalHandler: it returns a scripted disposition and appends one +// {kind, disposition} entry per invocation, in the fixtures' own spellings. +// The record is the ONLY thing distinguishing handler-Terminate from +// no-handler-terminal (their visible outcomes are identical), so the +// assertions check the exact set, never mere presence — which is what makes +// them red against a bypassed handler. +type signalLedger struct { + mu sync.Mutex + give eventfeed.Disposition + entries []string + signals []eventfeed.Signal +} + +// handler is the registered SignalHandler: invoked exactly once per signal, +// synchronously, on the consumer's goroutine, before its disposition takes +// effect. +func (l *signalLedger) handler(s eventfeed.Signal) eventfeed.Disposition { + l.mu.Lock() + defer l.mu.Unlock() + l.signals = append(l.signals, s) + l.entries = append(l.entries, signalKind(s)+"/"+dispositionName(l.give)) + return l.give +} + +func (l *signalLedger) invocations() []string { + l.mu.Lock() + defer l.mu.Unlock() + return append([]string(nil), l.entries...) +} + +func (l *signalLedger) first() eventfeed.Signal { + l.mu.Lock() + defer l.mu.Unlock() + if len(l.signals) == 0 { + return nil + } + return l.signals[0] +} + +// signalKind renders a signal's fixture spelling. +func signalKind(s eventfeed.Signal) string { + switch s.(type) { + case eventfeed.FeedGap: + return "feedGap" + case eventfeed.BufferOverflow: + return "bufferOverflow" + default: + return fmt.Sprintf("%T", s) + } +} + +// dispositionName renders a disposition's fixture spelling. +func dispositionName(d eventfeed.Disposition) string { + if d == eventfeed.Accept { + return "accept" + } + return "terminate" +} + +func assertInvocations(t *testing.T, got []string, want ...string) { + t.Helper() + if len(got) != len(want) { + t.Fatalf("handler invocations = %v, want exactly %v", got, want) + } + for i := range want { + if got[i] != want[i] { + t.Fatalf("handler invocations = %v, want exactly %v", got, want) + } + } +} + +// gonePoll builds the 410 the poll seam reports: the body's epoch_after_id and +// resume URL, classified. +func gonePoll(resume string) error { + return &eventfeed.PollError{Kind: eventfeed.PollGone, EpochAfterID: 100, ResumeURL: resume} +} + +// TestFeedGapAcceptedResumesViaProvidedURL is fixture 16: a 410 with an +// accepting handler fires Observer.Gap, invokes the handler exactly once, and +// re-enters via the URL the SERVER provided — verbatim, and as a present-class +// entry, so the resume page's position is held until the admitted straggler +// has been delivered. +func TestFeedGapAcceptedResumesViaProvidedURL(t *testing.T) { + store := feedtest.NewStore() + store.Stored("pos-0") + ledger := &signalLedger{give: eventfeed.Accept} + var gaps []string + obs := eventfeed.Observer{Gap: func(epochAfterID int64, resumeURL string) { + gaps = append(gaps, fmt.Sprintf("gap %d %s", epochAfterID, resumeURL)) + }} + h := storedHarness(t, store, + eventfeed.WithSignalHandler(ledger.handler), + eventfeed.WithObserver(obs)) + h.minter.ScriptTicket(ticket(1)) + resume := testOrigin + "/999/events.json?since=now" + h.polls.ScriptError(gonePoll(resume)) + h.polls.ScriptPage(eventfeed.PollPage{Position: "pos-1"}) + h.start() + + conn := h.driveToSubscribed() + h.serveSettled(conn, frameMessage(noFilterIdentifier, 41)) + conn.Serve(frameConfirm(noFilterIdentifier)) + h.awaitStreaming() + + if len(gaps) != 1 || gaps[0] != fmt.Sprintf("gap 100 %s", resume) { + t.Fatalf("Observer.Gap = %v, want exactly one firing carrying the epoch and resume URL", gaps) + } + assertInvocations(t, ledger.invocations(), "feedGap/accept") + calls := h.polls.Calls() + if len(calls) != 2 { + t.Fatalf("poll seam calls = %d, want 2", len(calls)) + } + if calls[1].Cursor != (eventfeed.Cursor{PageURL: resume}) { + t.Fatalf("re-entry cursor = %+v, want the provided resume URL verbatim", calls[1].Cursor) + } + // The resume is a present-class entry: the held position saves only after + // the buffered straggler is delivered. A position-resume treatment would + // save first and fail this ledger. + assertLedger(t, h.ledger(), []string{"event 41", "save pos-1"}) + assertPositions(t, store.Saves(), "pos-1") + if conn.Closed() { + t.Fatal("an accepted gap continues the feed on the same socket") + } + assertTimers(t, h.clock, map[string]int{timerStaleness: 1, timerRepairPoll: 1}) + if got := h.minter.Calls(); got != 1 { + t.Fatalf("mint seam calls = %d, want 1", got) + } +} + +// TestFeedGapDefaultTerminal is fixture 23: with no handler registered a 410 +// is the typed terminal — it never silently auto-continues. Observer.gap still +// fires (observability is not disposition), the resume URL is never requested, +// and nothing is saved. +func TestFeedGapDefaultTerminal(t *testing.T) { + store := feedtest.NewStore() + store.Stored("pos-0") + var gaps int + h := storedHarness(t, store, eventfeed.WithObserver(eventfeed.Observer{ + Gap: func(int64, string) { gaps++ }, + })) + h.minter.ScriptTicket(ticket(1)) + resume := testOrigin + "/999/events.json?since=now" + h.polls.ScriptError(gonePoll(resume)) + h.polls.ScriptPage(eventfeed.PollPage{Position: "pos-1"}) // never reached + base := runtime.NumGoroutine() + h.start() + + conn := h.driveToSubscribed() + conn.Serve(frameConfirm(noFilterIdentifier)) + h.join() + assertGoroutinesSettle(t, base) + + _, terminal, elements := h.snapshot() + if terminal == nil || terminal.Reason != eventfeed.ReasonFeedGap { + t.Fatalf("terminal = %v, want reason %q", terminal, eventfeed.ReasonFeedGap) + } + if elements != 1 { + t.Fatalf("iteration elements = %d, want exactly one terminal element", elements) + } + if gaps != 1 { + t.Fatalf("Observer.Gap firings = %d, want 1 (observability is not disposition)", gaps) + } + if got := h.polls.CallCount(); got != 1 { + t.Fatalf("poll seam calls = %d, want 1 (the resume URL is never followed)", got) + } + assertPositions(t, store.Saves()) + if got := h.minter.Calls(); got != 1 { + t.Fatalf("mint seam calls = %d, want 1 (terminal, never retried into)", got) + } + if !conn.Closed() { + t.Fatal("teardown must close the socket") + } + assertTimers(t, h.clock, map[string]int{}) +} + +// TestFeedGapHandlerTerminate is fixture 25: the Terminate branch reaches the +// same visible outcome as fixture 23 — Terminal(feed_gap), no save, the resume +// never followed — and is distinguished from it ONLY by the invocation record. +func TestFeedGapHandlerTerminate(t *testing.T) { + store := feedtest.NewStore() + store.Stored("pos-0") + ledger := &signalLedger{give: eventfeed.Terminate} + h := storedHarness(t, store, eventfeed.WithSignalHandler(ledger.handler)) + h.minter.ScriptTicket(ticket(1)) + resume := testOrigin + "/999/events.json?since=now" + h.polls.ScriptError(gonePoll(resume)) + h.start() + + conn := h.driveToSubscribed() + conn.Serve(frameConfirm(noFilterIdentifier)) + h.join() + + assertInvocations(t, ledger.invocations(), "feedGap/terminate") + gap, ok := ledger.first().(eventfeed.FeedGap) + if !ok || gap.EpochAfterID != 100 || gap.ResumeURL != resume { + t.Fatalf("signal = %+v, want FeedGap{EpochAfterID:100, ResumeURL:%q}", ledger.first(), resume) + } + _, terminal, _ := h.snapshot() + if terminal == nil || terminal.Reason != eventfeed.ReasonFeedGap { + t.Fatalf("terminal = %v, want reason %q", terminal, eventfeed.ReasonFeedGap) + } + if strings.Contains(terminal.Msg, resume) { + t.Fatalf("terminal message %q must not carry the resume URL", terminal.Msg) + } + if got := h.polls.CallCount(); got != 1 { + t.Fatalf("poll seam calls = %d, want 1 (Terminate never follows the resume)", got) + } + assertPositions(t, store.Saves()) + if !conn.Closed() { + t.Fatal("teardown must close the socket") + } + assertTimers(t, h.clock, map[string]int{}) +} + +// TestFeedGapAcceptedHostileResumeIsInvalidContinuation is fixture 27: the +// handler accepts — which is what reaches the resume-follow path at all — and +// the cross-origin resume URL then fails §8 validation BEFORE any request. +// The terminal is invalid_continuation, not feed_gap: the handler accepted; +// the URL failed. +func TestFeedGapAcceptedHostileResumeIsInvalidContinuation(t *testing.T) { + store := feedtest.NewStore() + store.Stored("pos-0") + ledger := &signalLedger{give: eventfeed.Accept} + h := storedHarness(t, store, eventfeed.WithSignalHandler(ledger.handler)) + h.minter.ScriptTicket(ticket(1)) + h.polls.ScriptError(gonePoll("https://attacker.example.com/999/events.json?since=now&token=secret")) + h.start() + + conn := h.driveToSubscribed() + conn.Serve(frameConfirm(noFilterIdentifier)) + h.join() + + assertInvocations(t, ledger.invocations(), "feedGap/accept") + _, terminal, _ := h.snapshot() + if terminal == nil || terminal.Reason != eventfeed.ReasonInvalidContinuation { + t.Fatalf("terminal = %v, want reason %q", terminal, eventfeed.ReasonInvalidContinuation) + } + if !strings.Contains(terminal.Msg, "https://attacker.example.com") { + t.Fatalf("terminal message %q should name the rejected origin", terminal.Msg) + } + if strings.Contains(terminal.Msg, "token=secret") || strings.Contains(terminal.Msg, "/999/") { + t.Fatalf("terminal message %q must be redacted to the origin", terminal.Msg) + } + if got := h.polls.CallCount(); got != 1 { + t.Fatalf("poll seam calls = %d, want 1 (zero requests to the foreign origin)", got) + } + assertPositions(t, store.Saves()) + if !conn.Closed() { + t.Fatal("teardown must close the socket") + } + assertTimers(t, h.clock, map[string]int{}) +} + +// TestFeedGapAcceptedWithNoResumeURLIsInvalidContinuation: accepting a 410 +// whose body carried no resume URL cannot be honored — the connector must not +// silently reposition at the present head — so it takes the same +// invalid_continuation edge an unfollowable URL does, before any request. +func TestFeedGapAcceptedWithNoResumeURLIsInvalidContinuation(t *testing.T) { + store := feedtest.NewStore() + store.Stored("pos-0") + ledger := &signalLedger{give: eventfeed.Accept} + h := storedHarness(t, store, eventfeed.WithSignalHandler(ledger.handler)) + h.minter.ScriptTicket(ticket(1)) + h.polls.ScriptError(&eventfeed.PollError{Kind: eventfeed.PollGone, EpochAfterID: 100}) + h.start() + + conn := h.driveToSubscribed() + conn.Serve(frameConfirm(noFilterIdentifier)) + h.join() + + assertInvocations(t, ledger.invocations(), "feedGap/accept") + _, terminal, _ := h.snapshot() + if terminal == nil || terminal.Reason != eventfeed.ReasonInvalidContinuation { + t.Fatalf("terminal = %v, want reason %q", terminal, eventfeed.ReasonInvalidContinuation) + } + if got := h.polls.CallCount(); got != 1 { + t.Fatalf("poll seam calls = %d, want 1 (no present-class fallback poll)", got) + } + assertPositions(t, store.Saves()) + if !conn.Closed() { + t.Fatal("teardown must close the socket") + } + assertTimers(t, h.clock, map[string]int{}) +} + +// TestRepairPollWalksFromTheInMemoryPosition is fixture 29's repair half: the +// fired `repair-poll` timer drives one walk from the connector's in-memory +// position — authoritative for repair within the run even though its save +// FAILED — delivers and checkpoints the repaired page, returns to Streaming +// with the cadence re-armed, and the next cycle repairs from the position the +// last page moved it to. +func TestRepairPollWalksFromTheInMemoryPosition(t *testing.T) { + store := feedtest.NewStore() // Missing → present-class entry + store.FailNextSave(errors.New("disk full")) + var saveFailures int + h := storedHarness(t, store, eventfeed.WithObserver(eventfeed.Observer{ + CheckpointSaveFailed: func(error) { saveFailures++ }, + })) + h.minter.ScriptTicket(ticket(1)) + h.polls.ScriptPage(eventfeed.PollPage{Position: "pos-1"}) + h.polls.ScriptPage(eventfeed.PollPage{Events: []eventfeed.Event{pollEvent(71)}, Position: "pos-2"}) + h.polls.ScriptPage(eventfeed.PollPage{Position: "pos-3"}) + h.start() + + conn := h.driveToSubscribed() + conn.Serve(frameConfirm(noFilterIdentifier)) + h.awaitStreaming() + + if d := h.fireTimer(timerRepairPoll); d < 48*time.Second || d > 72*time.Second { + t.Fatalf("repair-poll delay = %s, want within 60s ± 20%%", d) + } + h.awaitStreaming() + if d := h.fireTimer(timerRepairPoll); d < 48*time.Second || d > 72*time.Second { + t.Fatalf("second repair-poll delay = %s, want within 60s ± 20%%", d) + } + h.awaitStreaming() + + calls := h.polls.Calls() + if len(calls) != 3 { + t.Fatalf("poll seam calls = %d, want 3 (entry + two repair walks)", len(calls)) + } + if calls[1].Cursor != (eventfeed.Cursor{Position: "pos-1"}) { + t.Fatalf("first repair cursor = %+v, want the in-memory position (the failed save never regresses it)", calls[1].Cursor) + } + if calls[2].Cursor != (eventfeed.Cursor{Position: "pos-2"}) { + t.Fatalf("second repair cursor = %+v, want the position the repaired page moved it to", calls[2].Cursor) + } + if saveFailures != 1 { + t.Fatalf("CheckpointSaveFailed firings = %d, want 1", saveFailures) + } + assertIDs(t, h.deliveredIDs(), 71) + assertPositions(t, store.Saves(), "pos-1", "pos-2", "pos-3") + if got := h.minter.Calls(); got != 1 { + t.Fatalf("mint seam calls = %d, want 1 (a repair walk keeps the socket)", got) + } + assertTimers(t, h.clock, map[string]int{timerStaleness: 1, timerRepairPoll: 1}) +} + +// TestRepairWalkDedupesAgainstStreamedDeliveries: a repair page re-serving an +// id streaming already delivered is suppressed by the delivered-id LRU — the +// rule is symmetric across lanes — while the page's own position still +// advances (a position is a poll cursor, not an event acknowledgment). +func TestRepairWalkDedupesAgainstStreamedDeliveries(t *testing.T) { + store := feedtest.NewStore() + store.Stored("pos-0") + h := storedHarness(t, store) + h.minter.ScriptTicket(ticket(1)) + h.polls.ScriptPage(eventfeed.PollPage{Position: "pos-1"}) + h.polls.ScriptPage(eventfeed.PollPage{ + Events: []eventfeed.Event{pollEvent(81), pollEvent(82)}, + Position: "pos-2", + }) + h.start() + + conn := h.driveToSubscribed() + conn.Serve(frameConfirm(noFilterIdentifier)) + h.awaitStreaming() + conn.Serve(frameMessage(noFilterIdentifier, 81)) + h.waitUntil("live delivery", func() bool { return len(h.deliveredIDs()) == 1 }) + + h.fireTimer(timerRepairPoll) + h.awaitStreaming() + + assertIDs(t, h.deliveredIDs(), 81, 82) + assertPositions(t, store.Saves(), "pos-1", "pos-2") +} + +// TestRepairWalkBuffersLiveFramesAndDrainsThem: while a repair walk runs the +// state is CatchingUp, whose delivery invariant is poll pages only — a live +// frame arriving mid-walk is admitted to the buffer and replayed on the way +// back through Draining, after the walk's own pages. +func TestRepairWalkBuffersLiveFramesAndDrainsThem(t *testing.T) { + store := feedtest.NewStore() + store.Stored("pos-0") + h := storedHarness(t, store) + h.minter.ScriptTicket(ticket(1)) + next := testOrigin + "/999/events.json?after=91" + h.polls.ScriptPage(eventfeed.PollPage{Position: "pos-1"}) + h.polls.ScriptPage(eventfeed.PollPage{ + Events: []eventfeed.Event{pollEvent(91)}, + Position: "pos-2", + Next: next, + }) + h.polls.ScriptPage(eventfeed.PollPage{Events: []eventfeed.Event{pollEvent(92)}, Position: "pos-3"}) + + var conn *feedtest.Conn + h.polls.OnCall(func(call feedtest.PollCall) { + if call.Cursor != (eventfeed.Cursor{Position: "pos-1"}) { + return + } + // Inside the repair walk's first seam call: serve a live event and wait + // for the pump to take it, so it is queued for the state machine before + // the page is accepted. + conn.Serve(frameMessage(noFilterIdentifier, 55)) + conn.Serve(framePing()) + deadline := time.Now().Add(watchdog) + for conn.Pending() > 0 && time.Now().Before(deadline) { + time.Sleep(time.Millisecond) + } + }) + h.start() + + conn = h.driveToSubscribed() + conn.Serve(frameConfirm(noFilterIdentifier)) + h.awaitStreaming() + h.fireTimer(timerRepairPoll) + h.awaitStreaming() + + assertLedger(t, h.ledger(), []string{ + "save pos-1", + "event 91", "save pos-2", + "event 92", "save pos-3", + "event 55", + }) +} + +// TestPositionInvalidReEntersAtLastPollServedID is transition 18: a +// 400-position re-enters at `since=` on the SAME socket, +// keeping the checkpoints the walk already saved. +func TestPositionInvalidReEntersAtLastPollServedID(t *testing.T) { + store := feedtest.NewStore() + store.Stored("pos-0") + var rejected []eventfeed.PollErrorKind + h := storedHarness(t, store, eventfeed.WithObserver(eventfeed.Observer{ + PositionRejected: func(kind eventfeed.PollErrorKind) { rejected = append(rejected, kind) }, + })) + h.minter.ScriptTicket(ticket(1)) + next := testOrigin + "/999/events.json?after=102" + h.polls.ScriptPage(eventfeed.PollPage{ + Events: []eventfeed.Event{pollEvent(101), pollEvent(102)}, + Position: "pos-1", + Next: next, + }) + h.polls.ScriptError(&eventfeed.PollError{Kind: eventfeed.PollPositionInvalid}) + h.polls.ScriptPage(eventfeed.PollPage{Events: []eventfeed.Event{pollEvent(103)}, Position: "pos-2"}) + h.start() + + conn := h.driveToSubscribed() + conn.Serve(frameConfirm(noFilterIdentifier)) + h.awaitStreaming() + + calls := h.polls.Calls() + if len(calls) != 3 { + t.Fatalf("poll seam calls = %d, want 3", len(calls)) + } + if calls[2].Cursor != (eventfeed.Cursor{Since: "102"}) { + t.Fatalf("re-entry cursor = %+v, want since=", calls[2].Cursor) + } + if len(rejected) != 1 || rejected[0] != eventfeed.PollPositionInvalid { + t.Fatalf("Observer.PositionRejected = %v, want one position_invalid", rejected) + } + assertIDs(t, h.deliveredIDs(), 101, 102, 103) + assertPositions(t, store.Saves(), "pos-1", "pos-2") + if got := h.minter.Calls(); got != 1 { + t.Fatalf("mint seam calls = %d, want 1 (a re-entry keeps the socket)", got) + } +} + +// TestPositionInvalidWithNoPollServedIDReEntersPresentClass: with no id ever +// served by the POLL lane, the re-entry falls back to `since=now` — a +// present-class entry, so its position is held until the buffered live event +// has been delivered. A live-delivered id is never a reset cursor. +func TestPositionInvalidWithNoPollServedIDReEntersPresentClass(t *testing.T) { + store := feedtest.NewStore() + store.Stored("pos-0") + h := storedHarness(t, store) + h.minter.ScriptTicket(ticket(1)) + h.polls.ScriptError(&eventfeed.PollError{Kind: eventfeed.PollPositionInvalid}) + h.polls.ScriptPage(eventfeed.PollPage{Position: "pos-1"}) + h.start() + + conn := h.driveToSubscribed() + // Admitted live BEFORE the re-entry: its id must not position the re-entry. + h.serveSettled(conn, frameMessage(noFilterIdentifier, 41)) + conn.Serve(frameConfirm(noFilterIdentifier)) + h.awaitStreaming() + + calls := h.polls.Calls() + if len(calls) != 2 { + t.Fatalf("poll seam calls = %d, want 2", len(calls)) + } + if calls[1].Cursor != (eventfeed.Cursor{Since: "now"}) { + t.Fatalf("re-entry cursor = %+v, want the present-class since=now fallback", calls[1].Cursor) + } + assertLedger(t, h.ledger(), []string{"event 41", "save pos-1"}) +} + +// TestResetCursorIgnoresLiveDeliveredIDs: the reset cursor is POLL-LANE-ONLY. +// A repair walk whose 400-position lands after streaming has DELIVERED a live +// id still re-enters present-class, because the poll lane has served no id — +// re-entering at the live id would skip the un-polled gap behind it forever. +func TestResetCursorIgnoresLiveDeliveredIDs(t *testing.T) { + h := newHarness(t) + h.minter.ScriptTicket(ticket(1)) + h.polls.ScriptPage(eventfeed.PollPage{Position: "pos-1"}) // entry: serves no id + h.polls.ScriptError(&eventfeed.PollError{Kind: eventfeed.PollPositionInvalid}) + h.polls.ScriptPage(eventfeed.PollPage{Position: "pos-2"}) + h.start() + + conn := h.driveToSubscribed() + conn.Serve(frameConfirm(noFilterIdentifier)) + h.awaitStreaming() + conn.Serve(frameMessage(noFilterIdentifier, 55)) + h.waitUntil("live delivery", func() bool { return len(h.deliveredIDs()) == 1 }) + + h.fireTimer(timerRepairPoll) + h.awaitStreaming() + + calls := h.polls.Calls() + if len(calls) != 3 { + t.Fatalf("poll seam calls = %d, want 3", len(calls)) + } + if calls[1].Cursor != (eventfeed.Cursor{Position: "pos-1"}) { + t.Fatalf("repair cursor = %+v, want the in-memory position", calls[1].Cursor) + } + if calls[2].Cursor != (eventfeed.Cursor{Since: "now"}) { + t.Fatalf("re-entry cursor = %+v, want since=now — a live-delivered id is never a reset cursor", calls[2].Cursor) + } + assertIDs(t, h.deliveredIDs(), 55) +} + +// TestFilterChangedDiscardsTheHeldPosition is transition 19: a 409 re-enters +// like a 400-position AND discards the held position — so an attempt torn down +// before the re-entry serves a page reconnects at the present rather than back +// at the position the server just refused. With no poll-served id the reset +// cursor IS the present, and the reconnect re-enters at the latched +// `since=now` spelling rather than the bare entry: §23 treats the two +// identically ("`since=now` — and the bare present entry, which the server +// treats identically"), and re-entering at the LATCHED cursor is what keeps +// the poll-served-id case (below) from decaying into a present-class jump. +func TestFilterChangedDiscardsTheHeldPosition(t *testing.T) { + store := feedtest.NewStore() + store.Stored("pos-0") + var rejected []eventfeed.PollErrorKind + h := storedHarness(t, store, eventfeed.WithObserver(eventfeed.Observer{ + PositionRejected: func(kind eventfeed.PollErrorKind) { rejected = append(rejected, kind) }, + })) + h.minter.ScriptTicket(ticket(1)) + h.minter.ScriptTicket(ticket(2)) + h.polls.ScriptError(&eventfeed.PollError{Kind: eventfeed.PollFilterChanged}) + h.polls.ScriptError(&eventfeed.PollError{Kind: eventfeed.PollTransient, Err: errors.New("502")}) + h.polls.ScriptPage(eventfeed.PollPage{Position: "pos-1"}) + h.start() + + conn1 := h.driveToSubscribed() + conn1.Serve(frameConfirm(noFilterIdentifier)) + // The re-entry poll fails transiently, parking the walk on `poll-retry`; + // the socket then dies, so no page is ever accepted at the new cursor. + h.awaitTimer(timerPollRetry) + conn1.FailReads(errors.New("connection reset")) + h.awaitTimer(timerBackoff) + h.fireTimer(timerBackoff) + + conn2 := h.driveToSubscribed() + conn2.Serve(frameConfirm(noFilterIdentifier)) + h.awaitStreaming() + + calls := h.polls.Calls() + if len(calls) != 3 { + t.Fatalf("poll seam calls = %d, want 3", len(calls)) + } + if calls[1].Cursor != (eventfeed.Cursor{Since: "now"}) { + t.Fatalf("re-entry cursor = %+v, want the present-class since=now fallback", calls[1].Cursor) + } + if calls[2].Cursor != (eventfeed.Cursor{Since: "now"}) { + t.Fatalf("reconnect entry cursor = %+v, want the latched present-class reset (the 409 discarded the position)", calls[2].Cursor) + } + if len(rejected) != 1 || rejected[0] != eventfeed.PollFilterChanged { + t.Fatalf("Observer.PositionRejected = %v, want one filter_changed", rejected) + } + assertPositions(t, store.Saves(), "pos-1") +} + +// TestAuthorizationCounterCountsAllThreeShapes: the shared connection-level +// counter is one counter over three failure shapes — an unauthorized mint, an +// `unauthorized` disconnect at connect, and an unauthorized poll — so a MIXED +// sequence reaches the threshold-3 terminal. +func TestAuthorizationCounterCountsAllThreeShapes(t *testing.T) { + h := newHarness(t) + h.minter.ScriptError(&eventfeed.MintError{Kind: eventfeed.MintUnauthorized}) // 1 + h.minter.ScriptTicket(ticket(2)) + h.polls.ScriptError(&eventfeed.PollError{Kind: eventfeed.PollUnauthorized, Err: errors.New("401")}) // 2 + h.minter.ScriptTicket(ticket(3)) + h.start() + + h.fireTimer(timerBackoff) + + conn2 := h.driveToSubscribed() + conn2.Serve(frameConfirm(noFilterIdentifier)) + h.fireTimer(timerBackoff) + + conn3 := h.liveConn() + conn3.Serve(frameDisconnect("unauthorized", false)) // 3, pre-welcome + h.join() + + _, terminal, _ := h.snapshot() + if terminal == nil || terminal.Reason != eventfeed.ReasonAuthorizationFailed { + t.Fatalf("terminal = %v, want reason %q", terminal, eventfeed.ReasonAuthorizationFailed) + } + if got := h.minter.Calls(); got != 3 { + t.Fatalf("mint seam calls = %d, want 3", got) + } + assertTimers(t, h.clock, map[string]int{}) +} + +// TestPollRetryIndexGrowsAndResets pins the consecutive-poll-failure index k +// behind the `poll-retry` timer: the full-jitter envelope doubles per +// consecutive failure, and a successful page resets the index — a counter +// separate from the reconnect cycle's. +func TestPollRetryIndexGrowsAndResets(t *testing.T) { + h := newHarness(t) + h.conn.SetRand(func() float64 { return 0.5 }) + h.minter.ScriptTicket(ticket(1)) + transient := func() error { + return &eventfeed.PollError{Kind: eventfeed.PollTransient, Err: errors.New("502")} + } + next := testOrigin + "/999/events.json?after=101" + h.polls.ScriptError(transient()) // k=1 + h.polls.ScriptError(transient()) // k=2 + h.polls.ScriptPage(eventfeed.PollPage{Events: []eventfeed.Event{pollEvent(101)}, Position: "pos-1", Next: next}) + h.polls.ScriptError(transient()) // k=1 again: the page reset it + h.polls.ScriptPage(eventfeed.PollPage{Position: "pos-2"}) + h.start() + + conn := h.driveToSubscribed() + conn.Serve(frameConfirm(noFilterIdentifier)) + + if d := h.fireTimer(timerPollRetry); d != 500*time.Millisecond { + t.Fatalf("poll-retry delay (k=1) = %s, want 0.5 × 1s", d) + } + if d := h.fireTimer(timerPollRetry); d != time.Second { + t.Fatalf("poll-retry delay (k=2) = %s, want 0.5 × 2s", d) + } + if d := h.fireTimer(timerPollRetry); d != 500*time.Millisecond { + t.Fatalf("poll-retry delay after a successful page = %s, want the index reset to k=1", d) + } + h.awaitStreaming() + assertIDs(t, h.deliveredIDs(), 101) +} + +// TestResetCursorSurvivesAReconnect is the 409's reset cursor as RECONNECT +// state rather than walk-local state. Transition 19 discards the position, so +// once the socket dies before the re-entry has served a page there is nothing +// left in memory to re-enter from and the entry falls back to the configured +// start mode — the bare present for a resume feed. That silently skips +// everything between the last poll-served id and the present, on the one path +// whose whole purpose was not to skip anything. The cursor is chosen from run +// state (`lastPollServedID`) no later entry re-derives, so it has to be +// latched until a page replaces it. +func TestResetCursorSurvivesAReconnect(t *testing.T) { + store := feedtest.NewStore() + store.Stored("pos-0") + h := storedHarness(t, store) + h.minter.ScriptTicket(ticket(1)) + h.minter.ScriptTicket(ticket(2)) + next := testOrigin + "/999/events.json?after=101" + h.polls.ScriptPage(eventfeed.PollPage{ + Events: []eventfeed.Event{pollEvent(101)}, + Position: "pos-1", + Next: next, + }) + h.polls.ScriptError(&eventfeed.PollError{Kind: eventfeed.PollFilterChanged}) + h.polls.ScriptError(&eventfeed.PollError{Kind: eventfeed.PollTransient, Err: errors.New("502")}) + h.polls.ScriptPage(eventfeed.PollPage{Position: "pos-2"}) + h.start() + + conn1 := h.driveToSubscribed() + conn1.Serve(frameConfirm(noFilterIdentifier)) + // The re-entry poll fails transiently, parking the walk on `poll-retry`; + // the socket then dies, so no page is ever accepted at the reset cursor. + h.awaitTimer(timerPollRetry) + conn1.FailReads(errors.New("connection reset")) + h.awaitTimer(timerBackoff) + h.fireTimer(timerBackoff) + + conn2 := h.driveToSubscribed() + conn2.Serve(frameConfirm(noFilterIdentifier)) + h.awaitStreaming() + + calls := h.polls.Calls() + if len(calls) != 4 { + t.Fatalf("poll seam calls = %d, want 4", len(calls)) + } + if calls[2].Cursor != (eventfeed.Cursor{Since: "101"}) { + t.Fatalf("re-entry cursor = %+v, want the poll-served reset cursor", calls[2].Cursor) + } + if calls[3].Cursor != (eventfeed.Cursor{Since: "101"}) { + t.Fatalf("reconnect cursor = %+v, want the latched reset cursor, not a jump to the present", calls[3].Cursor) + } + assertPositions(t, store.Saves(), "pos-1", "pos-2") +} + +// TestAcceptedGapResumeSurvivesAReconnect is the same latch from the 410 side, +// plus the discard the Accept implies. The position the 410 refused is +// UNUSABLE, not merely superseded: left in memory it is what the next +// connection's entry selects, drawing the same 410 and re-invoking a gap +// handler that has already been asked and has already answered Accept. The +// accepted resume URL takes its place until a page replaces it, so the +// present-class reset the consumer accepted is what actually continues. +func TestAcceptedGapResumeSurvivesAReconnect(t *testing.T) { + store := feedtest.NewStore() + store.Stored("pos-0") + ledger := &signalLedger{give: eventfeed.Accept} + h := storedHarness(t, store, eventfeed.WithSignalHandler(ledger.handler)) + h.minter.ScriptTicket(ticket(1)) + h.minter.ScriptTicket(ticket(2)) + resume := testOrigin + "/999/events.json?since=now" + h.polls.ScriptError(gonePoll(resume)) + h.polls.ScriptError(&eventfeed.PollError{Kind: eventfeed.PollTransient, Err: errors.New("502")}) + h.polls.ScriptPage(eventfeed.PollPage{Position: "pos-1"}) + h.start() + + conn1 := h.driveToSubscribed() + conn1.Serve(frameConfirm(noFilterIdentifier)) + h.awaitTimer(timerPollRetry) + conn1.FailReads(errors.New("connection reset")) + h.awaitTimer(timerBackoff) + h.fireTimer(timerBackoff) + + conn2 := h.driveToSubscribed() + conn2.Serve(frameConfirm(noFilterIdentifier)) + h.awaitStreaming() + + calls := h.polls.Calls() + if len(calls) != 3 { + t.Fatalf("poll seam calls = %d, want 3", len(calls)) + } + if calls[1].Cursor != (eventfeed.Cursor{PageURL: resume}) { + t.Fatalf("re-entry cursor = %+v, want the provided resume URL", calls[1].Cursor) + } + if calls[2].Cursor != (eventfeed.Cursor{PageURL: resume}) { + t.Fatalf("reconnect cursor = %+v, want the latched resume URL, not the position the 410 refused", calls[2].Cursor) + } + // The handler is asked once, and the accepted disposition is not + // re-litigated by a reconnect. + assertInvocations(t, ledger.invocations(), "feedGap/accept") + assertPositions(t, store.Saves(), "pos-1") +} diff --git a/go/pkg/basecamp/eventfeed/scenario_conformance_test.go b/go/pkg/basecamp/eventfeed/scenario_conformance_test.go new file mode 100644 index 0000000000..9864b8ff96 --- /dev/null +++ b/go/pkg/basecamp/eventfeed/scenario_conformance_test.go @@ -0,0 +1,1077 @@ +// Tier-2 conformance driver: every fixture in `conformance/event-feed/` +// executed against the real connector through its seams. +// +// Strictness follows the family README's per-action-class rules exactly. +// Checkpoint saves and outbound frames are ARRIVAL-STRICT: a save carries the +// ordering witness (how many events had been delivered when it was made) and +// is matched against the deliveries its step's predecessors required, so a +// save that arrives while an `expectDelivered` rendezvous is still pending +// fails; outbound frames are matched in socket order and an unmatched one +// fails at `finally`. Mint and poll seam calls are PARKED: the call blocks +// inside the seam and its scripted response is released only when the driver +// reaches the matching expect step, so an early call cannot fail on arrival +// and one still parked at the end fails. Observation directives are +// rendezvous assertions under a wall-clock watchdog while virtual time stays +// frozen. +package eventfeed_test + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "maps" + "os" + "path/filepath" + "slices" + "strconv" + "strings" + "testing" + "time" + + "github.com/basecamp/basecamp-sdk/go/pkg/basecamp/eventfeed" + "github.com/basecamp/basecamp-sdk/go/pkg/basecamp/eventfeed/feedtest" +) + +// scenarioFixtureGlob discovers the family's fixtures by directory glob, so a +// fixture added by a later PR is executed without touching this driver. The +// sibling `pin-probes/` directory is deliberately NOT globbed: probes are +// inputs to the schema gate, not scenarios. +const scenarioFixtureGlob = "../../../../conformance/event-feed/fixtures/*.json" + +// TestEventFeedScenarioConformance runs every tier-2 scenario fixture, one +// subtest per fixture file. +func TestEventFeedScenarioConformance(t *testing.T) { + paths, err := filepath.Glob(scenarioFixtureGlob) + if err != nil { + t.Fatalf("globbing %s: %v", scenarioFixtureGlob, err) + } + if len(paths) == 0 { + t.Fatalf("no scenario fixtures matched %s — a glob that matches nothing proves nothing", scenarioFixtureGlob) + } + slices.Sort(paths) + t.Logf("tier-2 event feed scenarios: %d fixtures discovered under %s", len(paths), filepath.Dir(scenarioFixtureGlob)) + for _, path := range paths { + t.Run(filepath.Base(path), func(t *testing.T) { + raw, err := os.ReadFile(path) + if err != nil { + t.Fatalf("reading fixture: %v", err) + } + if err := runScenarioBytes(raw, filepath.Base(path)); err != nil { + t.Fatalf("%v", err) + } + }) + } +} + +// runScenarioBytes loads one fixture against a fresh harness and drives it. +func runScenarioBytes(raw []byte, file string) error { + h := newScenarioHarness() + defer h.close() + substituted, err := substitutePlaceholders(raw, h) + if err != nil { + return err + } + sc, err := parseScenario(substituted, file) + if err != nil { + return err + } + return runScenario(h, sc) +} + +// runScenario drives one loaded scenario to its `finally` block. +func runScenario(h *scenarioHarness, sc *scenario) (err error) { + h.connectPlan = planConnects(sc) + conn, cerr := h.newConnector(sc.Config) + if cerr != nil { + return fmt.Errorf("connector construction: %w", cerr) + } + ctx, cancel := context.WithCancel(context.Background()) + iteration := make(chan struct{}) + go func() { + defer close(iteration) + for event, iterErr := range conn.Events(ctx) { + if iterErr != nil { + h.recordTerminal(iterErr) + continue + } + h.recordDelivered(event.ID) + } + h.recordIterDone() + }() + defer func() { + cancel() + select { + case <-iteration: + case <-time.After(scenarioWatchdog): + if err == nil { + err = errors.New("the feed iteration did not end after cancellation") + } + } + }() + + d := &driver{h: h, sc: sc} + for i, step := range sc.Steps { + if stepErr := d.runStep(step); stepErr != nil { + return fmt.Errorf("step %d (%s): %w", i+1, step.Kind, stepErr) + } + } + return d.runFinally() +} + +// planConnects pre-scans the scenario's connect outcomes so the Nth handshake +// is answered as the Nth `expectConnect` scripts it. Dials happen strictly in +// order (each is gated behind its own parked mint), so the indices line up. +func planConnects(sc *scenario) []connectPlanEntry { + var plan []connectPlanEntry + for _, step := range sc.Steps { + connect, ok := step.Payload.(*expectConnectStep) + if !ok { + continue + } + entry := connectPlanEntry{kind: "accept"} + switch { + case connect.Outcome == nil: + case connect.Outcome.Refuse != "": + entry.kind = "refuse" + case connect.Outcome.RedirectTo != "": + entry.kind, entry.target = "redirect", connect.Outcome.RedirectTo + } + plan = append(plan, entry) + } + return plan +} + +// driver executes one scenario's steps against the harness. +type driver struct { + h *scenarioHarness + sc *scenario + + peer *cablePeer + connectsTaken int + identifier string + // requiredDeliveries is the cumulative delivery count the script has + // demanded so far — the ordering witness a checkpoint step is matched + // against. + requiredDeliveries int +} + +// runStep dispatches one directive. +// +// table: flat dispatch is what keeps the mapping auditable. +// +//nolint:gocyclo,cyclop // one arm per directive, mirroring the schema's step +func (d *driver) runStep(step scenarioStep) error { + switch payload := step.Payload.(type) { + case *expectMintStep: + return d.expectMint(payload) + case *expectConnectStep: + return d.expectConnect(payload) + case *serveStep: + return d.serve(payload) + case *serverCloseStep: + return d.serverClose(payload) + case *severStep: + return d.sever() + case *expectSubscribeStep: + return d.expectSubscribe(payload) + case *expectPollStep: + return d.expectPoll(payload) + case *expectClientCloseStep: + return d.expectClientClose() + case *advanceStep: + // Plain Advance: no fixture scripts a firing that arms a follow-on + // timer due inside the same window — 05, the suite's only advance, + // deliberately configures staleness and repair-poll out of it, so the + // window fires nothing. A script that did want a chained firing would + // pass feedtest.Clock.AdvanceSettling the rendezvous for the arming, + // since the connector arms on its own goroutine. + d.h.clock.Advance(millis(payload.Ms)) + return nil + case *fireTimerStep: + return d.fireTimer(payload) + case *exactIDs: + return d.expectDelivered(payload.Exact) + case *expectCheckpointStep: + return d.expectCheckpoint(payload.Position) + case *timerSet: + return d.awaitTimers(payload.Exact) + case *expectStateStep: + return d.awaitState(payload.Is) + case *errorExpect: + return d.awaitTerminal(payload.Reason) + case *expectGapStep: + return d.expectGap(payload.EpochAfterID) + case *expectBufferedStep: + return d.expectBuffered(payload.Count) + case *expectSignalStep: + return d.expectSignal(payload) + case *exactInvocations: + return d.expectInvocations(payload.Exact) + case *expectSaveFailedStep: + return d.expectSaveFailed() + case *expectPositionRejectedStep: + return d.expectPositionRejected(payload.Kind) + case *expectDisconnectedInvalidFrameStep: + return d.expectInvalidFrameDisconnect() + default: + return fmt.Errorf("the driver has no executor for directive %q", step.Kind) + } +} + +// --- protocol actions ---------------------------------------------------- + +func (d *driver) expectMint(step *expectMintStep) error { + var call *mintCall + err := d.h.await("a mint seam call", func() (bool, string) { + if len(d.h.pendingMints) == 0 { + return false, "" + } + call = d.h.pendingMints[0] + d.h.pendingMints = d.h.pendingMints[1:] + return true, "" + }) + if err != nil { + return err + } + outcome, err := mintOutcomeFrom(step.Respond) + if err != nil { + return err + } + call.release <- outcome + return nil +} + +func (d *driver) expectConnect(step *expectConnectStep) error { + index := d.connectsTaken + var attempt *connectAttempt + err := d.h.await(fmt.Sprintf("a cable dial to %s", step.URL), func() (bool, string) { + if len(d.h.connects) <= index { + return false, "" + } + attempt = d.h.connects[index] + if attempt.url != step.URL { + return false, fmt.Sprintf("the connector dialed\n %s\nwant the mint's url verbatim\n %s", attempt.url, step.URL) + } + if attempt.kind != "accept" { + return true, "" + } + return attempt.peer != nil, "" + }) + if err != nil { + return err + } + d.connectsTaken++ + if attempt.peer != nil { + d.peer = attempt.peer + } + return nil +} + +func (d *driver) serve(step *serveStep) error { + if d.peer == nil { + return errors.New("no cable connection is open to serve on") + } + frame, err := serveFrameBytes(step, d.identifier) + if err != nil { + return err + } + return d.h.writeFrame(d.peer, frame) +} + +func (d *driver) serverClose(step *serverCloseStep) error { + if d.peer == nil { + return errors.New("no cable connection is open to close") + } + return d.h.closePeer(d.peer, step.Code, step.Reason) +} + +// sever drops the TCP connection abruptly: no close frame, no disconnect +// frame. +func (d *driver) sever() error { + if d.peer == nil { + return errors.New("no cable connection is open to sever") + } + return d.peer.conn.CloseNow() +} + +func (d *driver) expectSubscribe(step *expectSubscribeStep) error { + frame, err := d.nextClientFrame("the subscribe command") + if err != nil { + return err + } + if frame.kind != "text" { + return fmt.Errorf("the connector closed the socket where the script expects a subscribe command") + } + var command struct { + Command string `json:"command"` + Identifier string `json:"identifier"` + } + if err := json.Unmarshal(frame.data, &command); err != nil { + return fmt.Errorf("the outbound frame is not a cable command: %w", err) + } + if command.Command != "subscribe" { + return fmt.Errorf("the connector sent a %q command; only `subscribe` is ever sent", command.Command) + } + identifier := map[string]string{} + if err := json.Unmarshal([]byte(command.Identifier), &identifier); err != nil { + return fmt.Errorf("the subscribe identifier is not a JSON object: %w", err) + } + want := map[string]string{"channel": step.Channel} + maps.Copy(want, step.Params) + if !maps.Equal(identifier, want) { + return fmt.Errorf("subscribe identifier %v, want %v", identifier, want) + } + if step.IdenticalToPrevious != nil { + if err := d.assertIdenticalSubscribe(frame.data); err != nil { + return err + } + } + d.identifier = command.Identifier + return nil +} + +// assertIdenticalSubscribe pins byte-identity with the previous subscribe on +// this connection (the retransmit-absorption case). +func (d *driver) assertIdenticalSubscribe(current []byte) error { + d.h.mu.Lock() + defer d.h.mu.Unlock() + for i := d.peer.taken - 2; i >= 0; i-- { + previous := d.peer.frames[i] + if previous.kind != "text" { + continue + } + if !slices.Equal(previous.data, current) { + return fmt.Errorf("the retransmitted subscribe differs from the previous one:\n %s\n %s", previous.data, current) + } + return nil + } + return errors.New("identicalToPrevious has no previous subscribe on this connection") +} + +func (d *driver) expectPoll(step *expectPollStep) error { + var call *pollCall + err := d.h.await("a poll seam call", func() (bool, string) { + if len(d.h.pendingPolls) == 0 { + return false, "" + } + call = d.h.pendingPolls[0] + d.h.pendingPolls = d.h.pendingPolls[1:] + return true, "" + }) + if err != nil { + return err + } + if err := assertPollTarget(step, call); err != nil { + return err + } + outcome, err := d.pollOutcomeFrom(step.Respond) + if err != nil { + return err + } + call.release <- outcome + return nil +} + +// assertPollTarget pins what the poll requested: an absolute continuation or +// resume URL followed verbatim, or the derived query params. +func assertPollTarget(step *expectPollStep, call *pollCall) error { + if step.URL != "" { + if call.cursor.PageURL != step.URL { + return fmt.Errorf("the poll targeted %+v, want the URL followed verbatim: %s", call.cursor, step.URL) + } + return nil + } + if call.cursor.PageURL != "" { + return fmt.Errorf("the poll followed the continuation %s where the script pins a query", call.cursor.PageURL) + } + got := derivedQuery(call) + if step.exactPin { + if !maps.Equal(got, step.params) { + return fmt.Errorf("poll query %v, want exactly %v", got, step.params) + } + return nil + } + for name, want := range step.params { + if got[name] != want { + return fmt.Errorf("poll query %v, want %s=%s", got, name, want) + } + } + return nil +} + +// derivedQuery renders the query the poll seam call would carry on the wire: +// the cursor's position/since plus the canonical comma-joined filter params. +func derivedQuery(call *pollCall) map[string]string { + query := map[string]string{} + if call.cursor.Position != "" { + query["position"] = call.cursor.Position + } + if call.cursor.Since != "" { + query["since"] = call.cursor.Since + } + if len(call.filters.Types) > 0 { + query["types"] = strings.Join(call.filters.Types, ",") + } + if len(call.filters.Buckets) > 0 { + query["buckets"] = joinIDs(call.filters.Buckets) + } + if len(call.filters.Creators) > 0 { + query["creators"] = joinIDs(call.filters.Creators) + } + return query +} + +func joinIDs(ids []int64) string { + parts := make([]string, len(ids)) + for i, id := range ids { + parts[i] = strconv.FormatInt(id, 10) + } + return strings.Join(parts, ",") +} + +func (d *driver) expectClientClose() error { + frame, err := d.nextClientFrame("the connector's explicit close of the open socket") + if err != nil { + return err + } + if frame.kind != "close" { + return fmt.Errorf("the connector sent a frame where the script expects its close: %s", frame.data) + } + return nil +} + +// nextClientFrame pops the next outbound action observed on the current +// socket, in order. +func (d *driver) nextClientFrame(what string) (clientFrame, error) { + if d.peer == nil { + return clientFrame{}, errors.New("no cable connection is open") + } + var frame clientFrame + err := d.h.await(what, func() (bool, string) { + if d.peer.taken >= len(d.peer.frames) { + if d.peer.dead { + return false, "the socket died with no further outbound frame" + } + return false, "" + } + frame = d.peer.frames[d.peer.taken] + d.peer.taken++ + return true, "" + }) + return frame, err +} + +// --- time ---------------------------------------------------------------- + +func (d *driver) fireTimer(step *fireTimerStep) error { + if err := d.awaitTimerArmed(step.Kind); err != nil { + return err + } + delay, ok := d.h.clock.FireTimer(step.Kind) + if !ok { + return fmt.Errorf("no %s timer is outstanding: outstanding %v", step.Kind, d.h.clock.Outstanding()) + } + if step.AssertDelayMs == nil { + return nil + } + low, high := millis(step.AssertDelayMs.Min), millis(step.AssertDelayMs.Max) + if delay < low || delay > high { + return fmt.Errorf("the %s timer was armed for %s, want [%s, %s]", step.Kind, delay, low, high) + } + return nil +} + +// awaitTimerArmed waits for a timer of the kind to exist. Timers are armed on +// the connector's goroutine, so a fireTimer directive that races the arming +// would otherwise fail spuriously. +func (d *driver) awaitTimerArmed(kind string) error { + return d.pollUntil(fmt.Sprintf("the %s timer to be armed", kind), func() bool { + return slices.Contains(d.h.clock.Outstanding(), kind) + }) +} + +// awaitTimers waits for the exact outstanding-timer set. +func (d *driver) awaitTimers(want map[string]int) error { + var got map[string]int + err := d.pollUntil(fmt.Sprintf("the outstanding timer set %v", want), func() bool { + got = timerCounts(d.h.clock) + return maps.Equal(got, want) + }) + if err != nil { + return fmt.Errorf("outstanding timers %v, want exactly %v", got, want) + } + return nil +} + +func timerCounts(clock *feedtest.Clock) map[string]int { + counts := map[string]int{} + for _, name := range clock.Outstanding() { + counts[name]++ + } + return counts +} + +// pollUntil is the bounded wall-clock watchdog for the clock registry, which +// is not part of the harness's broadcast. +func (d *driver) pollUntil(what string, satisfied func() bool) error { + deadline := time.Now().Add(scenarioWatchdog) + for { + if satisfied() { + return nil + } + if err := d.h.violation(); err != nil { + return fmt.Errorf("while waiting for %s: %w", what, err) + } + if time.Now().After(deadline) { + return fmt.Errorf("timed out after %s waiting for %s\n%s", scenarioWatchdog, what, d.h.describe()) + } + time.Sleep(time.Millisecond) + } +} + +// --- observations -------------------------------------------------------- + +func (d *driver) expectDelivered(want []int64) error { + err := d.h.await(fmt.Sprintf("delivered ids %v", want), func() (bool, string) { + if !prefixOfIDs(d.h.delivered, want) { + return false, fmt.Sprintf("delivered %v, want exactly %v", d.h.delivered, want) + } + return len(d.h.delivered) == len(want), "" + }) + if err != nil { + return err + } + d.requiredDeliveries = len(want) + return nil +} + +func (d *driver) expectCheckpoint(position string) error { + return d.h.await(fmt.Sprintf("a checkpoint save of %s", position), func() (bool, string) { + if d.h.savesTaken >= len(d.h.saves) { + return false, "" + } + save := d.h.saves[d.h.savesTaken] + if save.position != position { + return false, fmt.Sprintf("saved %q, want %q", save.position, position) + } + if save.deliveredAt < d.requiredDeliveries { + return false, fmt.Sprintf( + "the save of %q arrived after %d deliveries, but the script requires %d first — delivery strictly precedes that page's checkpoint", + save.position, save.deliveredAt, d.requiredDeliveries) + } + d.h.savesTaken++ + return true, "" + }) +} + +func (d *driver) awaitState(want string) error { + return d.h.await(fmt.Sprintf("state %q", want), func() (bool, string) { + return d.h.state == want, "" + }) +} + +func (d *driver) awaitTerminal(reason string) error { + return d.h.await(fmt.Sprintf("Terminal(%s)", reason), func() (bool, string) { + if d.h.terminal == nil { + return false, "" + } + if string(d.h.terminal.Reason) != reason { + return false, fmt.Sprintf("terminal reason %q, want %q (%v)", d.h.terminal.Reason, reason, d.h.terminal) + } + return true, "" + }) +} + +func (d *driver) expectGap(epochAfterID int64) error { + return d.h.await(fmt.Sprintf("Observer.gap(%d)", epochAfterID), func() (bool, string) { + if d.h.gapsTaken >= len(d.h.gaps) { + return false, "" + } + if got := d.h.gaps[d.h.gapsTaken]; got != epochAfterID { + return false, fmt.Sprintf("gap epoch_after_id %d, want %d", got, epochAfterID) + } + d.h.gapsTaken++ + return true, "" + }) +} + +// expectBuffered waits for the state-machine-owned live buffer's CURRENT +// occupancy — events admitted minus events dropped — to equal count. Every +// change is recorded, so a value reached and left again while the driver was +// scheduled out still satisfies the rendezvous. +func (d *driver) expectBuffered(count int) error { + d.h.mu.Lock() + from := len(d.h.occupancyHistory) + d.h.mu.Unlock() + return d.h.await(fmt.Sprintf("live buffer occupancy %d", count), func() (bool, string) { + if d.h.occupancy == count { + return true, "" + } + return slices.Contains(d.h.occupancyHistory[from:], count), "" + }) +} + +func (d *driver) expectSignal(step *expectSignalStep) error { + return d.h.await(fmt.Sprintf("the %s signal", step.Kind), func() (bool, string) { + if d.h.signalsTaken >= len(d.h.signals) { + return false, "" + } + signal := d.h.signals[d.h.signalsTaken] + if bad := matchSignal(step, signal); bad != "" { + return false, bad + } + d.h.signalsTaken++ + return true, "" + }) +} + +// matchSignal compares one raised signal against its expectation, returning a +// failure description or "". +func matchSignal(step *expectSignalStep, signal eventfeed.Signal) string { + switch got := signal.(type) { + case eventfeed.BufferOverflow: + if step.Kind != "bufferOverflow" { + return fmt.Sprintf("raised %T, want a %s signal", got, step.Kind) + } + if !slices.Equal(got.DroppedIDs, step.DroppedIDs) { + return fmt.Sprintf("dropped ids %v, want exactly %v", got.DroppedIDs, step.DroppedIDs) + } + if got.DroppedCount != *step.DroppedCount { + return fmt.Sprintf("dropped count %d, want %d", got.DroppedCount, *step.DroppedCount) + } + case eventfeed.FeedGap: + if step.Kind != "feedGap" { + return fmt.Sprintf("raised %T, want a %s signal", got, step.Kind) + } + if got.EpochAfterID != *step.EpochAfterID { + return fmt.Sprintf("epoch_after_id %d, want %d", got.EpochAfterID, *step.EpochAfterID) + } + default: + return fmt.Sprintf("unknown signal type %T", got) + } + return "" +} + +func (d *driver) expectInvocations(want []invocation) error { + return d.h.await(fmt.Sprintf("handler invocations %v", want), func() (bool, string) { + if !prefixOfInvocations(d.h.invocations, want) { + return false, fmt.Sprintf("handler invocations %v, want exactly %v", d.h.invocations, want) + } + return len(d.h.invocations) == len(want), "" + }) +} + +func (d *driver) expectSaveFailed() error { + return d.h.await("Observer.checkpoint_save_failed", func() (bool, string) { + if d.h.saveFailuresTaken >= d.h.saveFailures { + return false, "" + } + d.h.saveFailuresTaken++ + return true, "" + }) +} + +func (d *driver) expectPositionRejected(kind string) error { + return d.h.await(fmt.Sprintf("Observer.position_rejected(%s)", kind), func() (bool, string) { + if d.h.posRejectedTaken >= len(d.h.positionRejected) { + return false, "" + } + if got := d.h.positionRejected[d.h.posRejectedTaken]; got != kind { + return false, fmt.Sprintf("position_rejected kind %q, want %q", got, kind) + } + d.h.posRejectedTaken++ + return true, "" + }) +} + +func (d *driver) expectInvalidFrameDisconnect() error { + return d.h.await("Observer.disconnected carrying the invalid-frame indication", func() (bool, string) { + if d.h.invalidFramesTaken >= d.h.invalidFrames { + return false, "" + } + d.h.invalidFramesTaken++ + return true, "" + }) +} + +// --- finally ------------------------------------------------------------- + +// runFinally evaluates the end-of-scenario assertions, then the residue +// checks that make the strict interleave total: a parked seam call never +// matched, an outbound frame never matched, a save never matched, an +// unconsumed store script, an unmatched dial, or any recorded violation. +func (d *driver) runFinally() error { + fin := d.sc.Finally + if err := d.awaitState(fin.State); err != nil { + return fmt.Errorf("finally.state: %w", err) + } + if fin.Error != nil { + if err := d.awaitTerminal(fin.Error.Reason); err != nil { + return fmt.Errorf("finally.error: %w", err) + } + } else if terminal := d.h.terminalError(); terminal != nil { + return fmt.Errorf("finally: the feed terminated, but the fixture states no error: %w", terminal) + } + if fin.Delivered != nil { + if err := d.expectDelivered(fin.Delivered.Exact); err != nil { + return fmt.Errorf("finally.delivered: %w", err) + } + } + if fin.HandlerInvocations != nil { + if err := d.expectInvocations(fin.HandlerInvocations.Exact); err != nil { + return fmt.Errorf("finally.handlerInvocations: %w", err) + } + } + if fin.Checkpoints != nil { + if err := d.assertCheckpointLedger(fin.Checkpoints.Exact); err != nil { + return fmt.Errorf("finally.checkpoints: %w", err) + } + } + if fin.Timers != nil { + if err := d.awaitTimers(fin.Timers.Exact); err != nil { + return fmt.Errorf("finally.timers: %w", err) + } + } + if err := d.assertCounts(fin); err != nil { + return err + } + if err := d.assertSocket(fin.Socket); err != nil { + return fmt.Errorf("finally.socket: %w", err) + } + return d.assertNoResidue() +} + +func (d *driver) assertCheckpointLedger(want []string) error { + return d.h.await(fmt.Sprintf("the checkpoint ledger %v", want), func() (bool, string) { + got := make([]string, len(d.h.saves)) + for i, save := range d.h.saves { + got[i] = save.position + } + if !prefixOfStrings(got, want) { + return false, fmt.Sprintf("checkpoint ledger %v, want exactly %v", got, want) + } + return len(got) == len(want), "" + }) +} + +func (d *driver) assertCounts(fin scenarioFinally) error { + d.h.mu.Lock() + mints, polls, connects := d.h.mintCalls, d.h.pollCalls, len(d.h.connects) + d.h.mu.Unlock() + for _, check := range []struct { + name string + want *int + got int + }{ + {"finally.mintCount", fin.MintCount, mints}, + {"finally.connectCount", fin.ConnectCount, connects}, + {"finally.pollCount", fin.PollCount, polls}, + } { + if check.want != nil && check.got != *check.want { + return fmt.Errorf("%s: %d seam calls, want %d", check.name, check.got, *check.want) + } + } + return nil +} + +func (d *driver) assertSocket(disposition string) error { + if disposition == "" { + return nil + } + d.h.mu.Lock() + dialed := len(d.h.connects) + var last *cablePeer + if len(d.h.peers) > 0 { + last = d.h.peers[len(d.h.peers)-1] + } + d.h.mu.Unlock() + switch disposition { + case "none": + if dialed != 0 { + return fmt.Errorf("%d dials happened, want none", dialed) + } + return nil + case "closed": + if last == nil { + return errors.New("no socket was ever opened") + } + return d.pollUntil("the socket to be closed", func() bool { + d.h.mu.Lock() + defer d.h.mu.Unlock() + return last.dead + }) + default: // open + if last == nil { + return errors.New("no socket was ever opened") + } + d.h.mu.Lock() + defer d.h.mu.Unlock() + if last.dead { + return errors.New("the socket is closed, want it still open") + } + return nil + } +} + +func (d *driver) assertNoResidue() error { + d.h.mu.Lock() + defer d.h.mu.Unlock() + if len(d.h.violations) > 0 { + return errors.New(d.h.violations[0]) + } + if len(d.h.pendingMints) > 0 || len(d.h.pendingPolls) > 0 { + return fmt.Errorf("%d mint and %d poll seam calls are still parked, unmatched by any expect step", + len(d.h.pendingMints), len(d.h.pendingPolls)) + } + if d.connectsTaken != len(d.h.connects) { + return fmt.Errorf("%d cable dials happened, %d matched by an expectConnect step", + len(d.h.connects), d.connectsTaken) + } + for i, peer := range d.h.peers { + if peer.taken < len(peer.frames) { + return fmt.Errorf("socket %d has %d unmatched outbound action(s), first: %s", + i+1, len(peer.frames)-peer.taken, describeFrame(peer.frames[peer.taken])) + } + } + if d.h.savesTaken != len(d.h.saves) { + return fmt.Errorf("%d checkpoint saves happened, %d matched by an expectCheckpoint step", + len(d.h.saves), d.h.savesTaken) + } + if d.h.saveScripted && d.h.saveUsed != len(d.h.saveScript) { + return fmt.Errorf("the scripted store calls %v were consumed %d deep — an unconsumed outcome fails the scenario", + d.h.saveScript, d.h.saveUsed) + } + return nil +} + +func describeFrame(frame clientFrame) string { + if frame.kind == "close" { + return fmt.Sprintf("a client close (code %d)", frame.code) + } + return string(frame.data) +} + +// terminalError returns the terminal error element, if the feed yielded one. +func (h *scenarioHarness) terminalError() *eventfeed.TerminalError { + h.mu.Lock() + defer h.mu.Unlock() + return h.terminal +} + +// violation returns the first recorded harness violation, if any. +func (h *scenarioHarness) violation() error { + h.mu.Lock() + defer h.mu.Unlock() + if len(h.violations) == 0 { + return nil + } + return errors.New(h.violations[0]) +} + +// --- scripted responses -------------------------------------------------- + +// mintOutcomeFrom maps a scripted mint response onto the seam's outcome: a +// ticket, or the classified *MintError the Layer-1 adapter would produce. +func mintOutcomeFrom(respond mintRespond) (mintOutcome, error) { + if respond.Body != nil { + return mintOutcome{ticket: eventfeed.StreamTicket{ + Ticket: respond.Body.Ticket, + ExpiresIn: respond.Body.ExpiresIn, + URL: respond.Body.URL, + }}, nil + } + retryAfter, err := retryAfterFrom(respond.Headers) + if err != nil { + return mintOutcome{}, err + } + mintErr := &eventfeed.MintError{RetryAfter: retryAfter, Err: fmt.Errorf("mint responded %d", *respond.Status)} + switch *respond.Status { + case 401, 403: + mintErr.Kind = eventfeed.MintUnauthorized + case 404, 422: + mintErr.Kind = eventfeed.MintUnrecoverable + case 429, 503: + mintErr.Kind = eventfeed.MintThrottled + default: + mintErr.Kind = eventfeed.MintTransient + } + return mintOutcome{err: mintErr}, nil +} + +// pollOutcomeFrom maps a scripted poll response onto the seam's outcome: a +// page, or the classified *PollError the Layer-1 adapter would produce. +func (d *driver) pollOutcomeFrom(respond pollRespond) (pollOutcome, error) { + status := 200 + if respond.Status != nil { + status = *respond.Status + } + switch status { + case 200: + return pollPageFrom(respond.Body) + case 302: + return d.redirectRefusalFrom(respond.Headers) + case 400: + return pollOutcome{}, errors.New("400 poll responses are not modeled: the position-vs-filter discriminating bodies are class-1 literals still PROVISIONAL in the family README's dependency table, so a driver cannot discriminate them without guessing") + case 409: + return pollOutcome{err: &eventfeed.PollError{ + Kind: eventfeed.PollFilterChanged, + Err: errors.New("poll responded 409"), + }}, nil + case 410: + return goneOutcome(respond.Body) + case 401, 403: + return pollOutcome{err: &eventfeed.PollError{ + Kind: eventfeed.PollUnauthorized, + Err: errors.New("poll responded 401/403"), + }}, nil + default: + retryAfter, err := retryAfterFrom(respond.Headers) + if err != nil { + return pollOutcome{}, err + } + kind := eventfeed.PollTransient + if status == 429 || status == 503 { + kind = eventfeed.PollThrottled + } + return pollOutcome{err: &eventfeed.PollError{ + Kind: kind, + RetryAfter: retryAfter, + Err: fmt.Errorf("poll responded %d", status), + }}, nil + } +} + +func pollPageFrom(body json.RawMessage) (pollOutcome, error) { + envelope := pollEnvelope{} + if err := decodeStrict(body, &envelope); err != nil { + return pollOutcome{}, fmt.Errorf("poll envelope: %w", err) + } + page := eventfeed.PollPage{Position: envelope.Position, Next: envelope.Next} + for _, raw := range envelope.Events { + row := pollEventRow{} + if err := decodeStrict(raw, &row); err != nil { + return pollOutcome{}, fmt.Errorf("poll event row: %w", err) + } + createdAt, err := time.Parse(time.RFC3339, row.CreatedAt) + if err != nil { + return pollOutcome{}, fmt.Errorf("poll event created_at: %w", err) + } + page.Events = append(page.Events, eventfeed.Event{ + ID: row.ID, + Kind: row.Kind, + EventType: row.EventType, + Action: row.Action, + CreatedAt: createdAt, + BucketID: row.BucketID, + CreatorID: row.CreatorID, + RecordingID: row.RecordingID, + }) + } + return pollOutcome{page: page}, nil +} + +func goneOutcome(body json.RawMessage) (pollOutcome, error) { + gone := goneBody{} + if err := decodeStrict(body, &gone); err != nil { + return pollOutcome{}, fmt.Errorf("410 body: %w", err) + } + return pollOutcome{err: &eventfeed.PollError{ + Kind: eventfeed.PollGone, + EpochAfterID: gone.EpochAfterID, + ResumeURL: gone.Resume, + Err: errors.New("poll responded 410"), + }}, nil +} + +// redirectRefusalFrom models the poll seam's suppressed redirect: the seam +// never follows automatically, and a Location that fails per-hop +// same-origin/no-downgrade validation is refused inside the call, with the +// Location redacted to its origin. No request is ever issued to it — the +// driver is the seam, so the foreign origin is unreachable by construction. +func (d *driver) redirectRefusalFrom(headers map[string]string) (pollOutcome, error) { + location := headers["Location"] + if location == "" { + return pollOutcome{}, errors.New("a 302 poll response needs a Location header") + } + origin, err := eventfeed.CanonicalOrigin(location) + if err != nil { + return pollOutcome{}, fmt.Errorf("302 Location: %w", err) + } + if origin == d.h.apiOrigin { + return pollOutcome{}, errors.New("a same-origin 302 is not modeled: the seam would follow it, and no fixture scripts that hop") + } + return pollOutcome{err: &eventfeed.PollError{ + Kind: eventfeed.PollRedirectRefused, + LocationOrigin: origin, + Err: errors.New("poll refused a redirect"), + }}, nil +} + +func retryAfterFrom(headers map[string]string) (time.Duration, error) { + value, ok := headers["Retry-After"] + if !ok { + return 0, nil + } + seconds, err := strconv.Atoi(value) + if err != nil { + return 0, fmt.Errorf("Retry-After %q is not a delta-seconds value: %w", value, err) + } + return time.Duration(seconds) * time.Second, nil +} + +// --- server frames ------------------------------------------------------- + +// serveFrameBytes renders one scripted server frame. confirm/reject/message +// auto-echo the captured subscribe identifier unless the fixture overrides it. +func serveFrameBytes(step *serveStep, identifier string) ([]byte, error) { + echo := identifier + if step.Identifier != nil { + echo = *step.Identifier + } + switch step.Frame { + case "welcome": + return []byte(`{"type":"welcome"}`), nil + case "ping": + if step.Message == nil { + return []byte(`{"type":"ping"}`), nil + } + return json.Marshal(map[string]any{"type": "ping", "message": *step.Message}) + case "confirm", "reject": + if echo == "" { + return nil, fmt.Errorf("no subscribe identifier has been captured to echo") + } + frameType := "confirm_subscription" + if step.Frame == "reject" { + frameType = "reject_subscription" + } + return json.Marshal(map[string]any{"type": frameType, "identifier": echo}) + case "disconnect": + return json.Marshal(map[string]any{ + "type": "disconnect", + "reason": step.Reason, + "reconnect": *step.Reconnect, + }) + case "message": + if echo == "" { + return nil, fmt.Errorf("no subscribe identifier has been captured to echo") + } + return json.Marshal(map[string]any{"identifier": echo, "message": step.Event}) + default: // raw + return []byte(step.Text), nil + } +} + +// --- small helpers ------------------------------------------------------- + +func prefixOfIDs(got, want []int64) bool { + return len(got) <= len(want) && slices.Equal(got, want[:len(got)]) +} + +func prefixOfStrings(got, want []string) bool { + return len(got) <= len(want) && slices.Equal(got, want[:len(got)]) +} + +func prefixOfInvocations(got, want []invocation) bool { + return len(got) <= len(want) && slices.Equal(got, want[:len(got)]) +} diff --git a/go/pkg/basecamp/eventfeed/scenario_fixture_test.go b/go/pkg/basecamp/eventfeed/scenario_fixture_test.go new file mode 100644 index 0000000000..9dca83dbe4 --- /dev/null +++ b/go/pkg/basecamp/eventfeed/scenario_fixture_test.go @@ -0,0 +1,807 @@ +// Tier-2 conformance: the scenario fixture model. +// +// This file loads `conformance/event-feed/fixtures/*.json` — merged contract, +// never edited here — into typed directives. Loading is STRICT in both +// directions: an unknown step directive, an unknown object key, a construct +// the driver does not model, or a fixture-level invariant the JSON Schema +// cannot express (droppedCount vs droppedIds, a fireTimer envelope with +// min > max, a name that disagrees with the filename) fails the fixture at +// load with a named error. Nothing is ever silently ignored: that is what +// keeps the driver honest as PR 4 adds fixtures and schema constructs. +package eventfeed_test + +import ( + "bytes" + "encoding/json" + "fmt" + "regexp" + "strconv" + "strings" +) + +// scenario is one loaded tier-2 fixture. +type scenario struct { + Name string + Description string + Config scenarioConfig + Steps []scenarioStep + Finally scenarioFinally +} + +// scenarioStep is one directive: its schema key plus the decoded payload. +type scenarioStep struct { + Kind string + Payload any +} + +// scenarioConfig is the schema's `config` object — the connector construction +// options a scenario selects. +type scenarioConfig struct { + Types []string `json:"types"` + Buckets []int64 `json:"buckets"` + Creators []int64 `json:"creators"` + Position string `json:"position"` + ConfirmationDeadlineMs int `json:"confirmationDeadlineMs"` + RepairPollBaseMs int `json:"repairPollBaseMs"` + BackoffBaseMs int `json:"backoffBaseMs"` + BackoffCapMs int `json:"backoffCapMs"` + StalenessMs int `json:"stalenessMs"` + LiveBufferCapacity int `json:"liveBufferCapacity"` + DedupeCapacity int `json:"dedupeCapacity"` + SignalDisposition map[string]string `json:"signalDisposition"` + CheckpointStore *storeScript `json:"checkpointStore"` +} + +// storeScript is the schema's scripted CheckpointStore. +type storeScript struct { + Load string `json:"load"` + Save []string `json:"save"` +} + +// scenarioFinally is the schema's `finally` object. +type scenarioFinally struct { + State string `json:"state"` + MintCount *int `json:"mintCount"` + ConnectCount *int `json:"connectCount"` + PollCount *int `json:"pollCount"` + Timers *timerSet `json:"timers"` + Socket string `json:"socket"` + Delivered *exactIDs `json:"delivered"` + Checkpoints *exactPositions `json:"checkpoints"` + Error *errorExpect `json:"error"` + HandlerInvocations *exactInvocations `json:"handlerInvocations"` +} + +type timerSet struct { + Exact map[string]int `json:"exact"` +} + +type exactIDs struct { + Exact []int64 `json:"exact"` +} + +type exactPositions struct { + Exact []string `json:"exact"` +} + +type exactInvocations struct { + Exact []invocation `json:"exact"` +} + +type invocation struct { + Kind string `json:"kind"` + Disposition string `json:"disposition"` +} + +type errorExpect struct { + Reason string `json:"reason"` + Category string `json:"category"` +} + +// --- step payloads ------------------------------------------------------- + +type expectMintStep struct { + Respond mintRespond `json:"respond"` +} + +type mintRespond struct { + Status *int `json:"status"` + Body *mintBody `json:"body"` + Headers map[string]string `json:"headers"` +} + +type mintBody struct { + Ticket string `json:"ticket"` + ExpiresIn int `json:"expires_in"` + URL string `json:"url"` +} + +type expectConnectStep struct { + URL string `json:"url"` + Outcome *connectOutcome `json:"outcome"` +} + +type connectOutcome struct { + Accept *bool `json:"accept"` + Refuse string `json:"refuse"` + RedirectTo string `json:"redirectTo"` +} + +type serveStep struct { + Frame string `json:"frame"` + Message *int `json:"message"` + Identifier *string `json:"identifier"` + Reason string `json:"reason"` + Reconnect *bool `json:"reconnect"` + Event json.RawMessage `json:"event"` + Text string `json:"text"` +} + +type serverCloseStep struct { + Code int `json:"code"` + Reason string `json:"reason"` +} + +type expectSubscribeStep struct { + Channel string `json:"channel"` + Params map[string]string `json:"params"` + IdenticalToPrevious *bool `json:"identicalToPrevious"` +} + +type expectPollStep struct { + URL string `json:"url"` + Query json.RawMessage `json:"query"` + Respond pollRespond `json:"respond"` + + // query, decoded: exactPin selects equality over the whole derived query. + params map[string]string + exactPin bool +} + +type pollRespond struct { + Status *int `json:"status"` + Headers map[string]string `json:"headers"` + Body json.RawMessage `json:"body"` +} + +type pollEnvelope struct { + Events []json.RawMessage `json:"events"` + Position string `json:"position"` + Next string `json:"next"` +} + +type pollEventRow struct { + ID int64 `json:"id"` + Kind string `json:"kind"` + EventType string `json:"event_type"` + Action string `json:"action"` + CreatedAt string `json:"created_at"` + BucketID int64 `json:"bucket_id"` + CreatorID int64 `json:"creator_id"` + RecordingID int64 `json:"recording_id"` +} + +type goneBody struct { + Error string `json:"error"` + EpochAfterID int64 `json:"epoch_after_id"` + Resume string `json:"resume"` +} + +type advanceStep struct { + Ms int `json:"ms"` +} + +type fireTimerStep struct { + Kind string `json:"kind"` + AssertDelayMs *delayEnvelope `json:"assertDelayMs"` +} + +type delayEnvelope struct { + Min int `json:"min"` + Max int `json:"max"` +} + +type expectCheckpointStep struct { + Position string `json:"position"` +} + +type expectStateStep struct { + Is string `json:"is"` +} + +type expectGapStep struct { + EpochAfterID int64 `json:"epochAfterId"` +} + +type expectBufferedStep struct { + Count int `json:"count"` +} + +type expectSignalStep struct { + Kind string `json:"kind"` + DroppedIDs []int64 `json:"droppedIds"` + DroppedCount *int `json:"droppedCount"` + EpochAfterID *int64 `json:"epochAfterId"` +} + +type expectPositionRejectedStep struct { + Kind string `json:"kind"` +} + +// --- loading ------------------------------------------------------------- + +// parseScenario decodes one substituted fixture, failing on anything the +// driver does not model. +func parseScenario(raw []byte, file string) (*scenario, error) { + top, err := objectKeys(raw) + if err != nil { + return nil, err + } + if err := allowedKeys("fixture", top, "name", "description", "config", "steps", "finally"); err != nil { + return nil, err + } + sc := &scenario{} + if err := decodeStrict(top["name"], &sc.Name); err != nil { + return nil, fmt.Errorf("name: %w", err) + } + if err := decodeStrict(top["description"], &sc.Description); err != nil { + return nil, fmt.Errorf("description: %w", err) + } + if want := strings.TrimSuffix(file, ".json"); sc.Name != want { + return nil, fmt.Errorf("fixture name %q must match its filename %q", sc.Name, want) + } + if cfgRaw, ok := top["config"]; ok { + if err := decodeStrict(cfgRaw, &sc.Config); err != nil { + return nil, fmt.Errorf("config: %w", err) + } + if err := validateConfig(sc.Config); err != nil { + return nil, fmt.Errorf("config: %w", err) + } + } + var steps []json.RawMessage + if err := decodeStrict(top["steps"], &steps); err != nil { + return nil, fmt.Errorf("steps: %w", err) + } + for i, rawStep := range steps { + step, err := parseStep(rawStep) + if err != nil { + return nil, fmt.Errorf("step %d: %w", i+1, err) + } + sc.Steps = append(sc.Steps, step) + } + finRaw, ok := top["finally"] + if !ok { + return nil, fmt.Errorf("fixture is missing its `finally` block") + } + if err := decodeStrict(finRaw, &sc.Finally); err != nil { + return nil, fmt.Errorf("finally: %w", err) + } + if err := validateFinally(sc.Finally); err != nil { + return nil, fmt.Errorf("finally: %w", err) + } + return sc, nil +} + +// parseStep decodes one step's single directive. +func parseStep(raw json.RawMessage) (scenarioStep, error) { + keys, err := objectKeys(raw) + if err != nil { + return scenarioStep{}, err + } + if len(keys) != 1 { + return scenarioStep{}, fmt.Errorf("a step carries exactly one directive, got %d", len(keys)) + } + for kind, body := range keys { + payload, err := decodeDirective(kind, body) + if err != nil { + return scenarioStep{}, fmt.Errorf("%s: %w", kind, err) + } + return scenarioStep{Kind: kind, Payload: payload}, nil + } + return scenarioStep{}, fmt.Errorf("unreachable") +} + +// decodeDirective decodes one directive body. An unrecognized directive key is +// a hard failure — never a skip — so a fixture using a construct this driver +// has not implemented fails loudly instead of passing vacuously. +// +// dispatch table is exactly the shape that makes an unimplemented directive +// impossible to miss. +// +//nolint:gocyclo,cyclop // one arm per schema directive: a flat, exhaustive +func decodeDirective(kind string, body json.RawMessage) (any, error) { + switch kind { + case "expectMint": + step := &expectMintStep{} + if err := decodeStrict(body, step); err != nil { + return nil, err + } + return step, validateMintRespond(step.Respond) + case "expectConnect": + step := &expectConnectStep{} + if err := decodeStrict(body, step); err != nil { + return nil, err + } + return step, validateConnectOutcome(step.Outcome) + case "serve": + step := &serveStep{} + if err := decodeStrict(body, step); err != nil { + return nil, err + } + return step, validateServe(step) + case "serverClose": + step := &serverCloseStep{} + return step, decodeStrict(body, step) + case "sever": + var flag bool + if err := decodeStrict(body, &flag); err != nil { + return nil, err + } + if !flag { + return nil, fmt.Errorf("only `true` is valid") + } + return &severStep{}, nil + case "expectSubscribe": + step := &expectSubscribeStep{} + if err := decodeStrict(body, step); err != nil { + return nil, err + } + if step.Channel != "EventsChannel" { + return nil, fmt.Errorf("channel %q is not EventsChannel", step.Channel) + } + return step, nil + case "expectPoll": + step := &expectPollStep{} + if err := decodeStrict(body, step); err != nil { + return nil, err + } + return step, decodePollQuery(step) + case "expectClientClose": + var empty struct{} + return &expectClientCloseStep{}, decodeStrict(body, &empty) + case "advance": + step := &advanceStep{} + return step, decodeStrict(body, step) + case "fireTimer": + step := &fireTimerStep{} + if err := decodeStrict(body, step); err != nil { + return nil, err + } + if step.AssertDelayMs != nil && step.AssertDelayMs.Min > step.AssertDelayMs.Max { + return nil, fmt.Errorf("assertDelayMs min %d exceeds max %d", step.AssertDelayMs.Min, step.AssertDelayMs.Max) + } + return step, validateTimerKind(step.Kind) + case "expectDelivered": + step := &exactIDs{} + return step, decodeStrict(body, step) + case "expectCheckpoint": + step := &expectCheckpointStep{} + return step, decodeStrict(body, step) + case "expectTimers": + step := &timerSet{} + if err := decodeStrict(body, step); err != nil { + return nil, err + } + return step, validateTimerSet(*step) + case "expectState": + step := &expectStateStep{} + return step, decodeStrict(body, step) + case "expectError": + step := &errorExpect{} + if err := decodeStrict(body, step); err != nil { + return nil, err + } + return step, validateErrorExpect(*step) + case "expectGap": + step := &expectGapStep{} + return step, decodeStrict(body, step) + case "expectBuffered": + step := &expectBufferedStep{} + return step, decodeStrict(body, step) + case "expectSignal": + step := &expectSignalStep{} + if err := decodeStrict(body, step); err != nil { + return nil, err + } + return step, validateSignalExpect(step) + case "expectHandlerInvocations": + step := &exactInvocations{} + if err := decodeStrict(body, step); err != nil { + return nil, err + } + return step, validateInvocations(step.Exact) + case "expectSaveFailed": + var flag bool + if err := decodeStrict(body, &flag); err != nil { + return nil, err + } + if !flag { + return nil, fmt.Errorf("only `true` is valid") + } + return &expectSaveFailedStep{}, nil + case "expectPositionRejected": + step := &expectPositionRejectedStep{} + if err := decodeStrict(body, step); err != nil { + return nil, err + } + if step.Kind != "position_invalid" && step.Kind != "filter_changed" { + return nil, fmt.Errorf("unknown kind %q", step.Kind) + } + return step, nil + case "expectDisconnectedInvalidFrame": + var flag bool + if err := decodeStrict(body, &flag); err != nil { + return nil, err + } + if !flag { + return nil, fmt.Errorf("only `true` is valid") + } + return &expectDisconnectedInvalidFrameStep{}, nil + default: + return nil, fmt.Errorf("unknown step directive %q: the driver refuses to skip a directive it does not model", kind) + } +} + +// Parameterless directives, each a distinct type so the executor's type switch +// stays exhaustive. +type ( + severStep struct{} + expectClientCloseStep struct{} + expectSaveFailedStep struct{} + expectDisconnectedInvalidFrameStep struct{} +) + +// --- validation ---------------------------------------------------------- + +func validateConfig(cfg scenarioConfig) error { + if cfg.BackoffBaseMs != 0 || cfg.BackoffCapMs != 0 { + return fmt.Errorf("backoffBaseMs/backoffCapMs are not modeled: SPEC §23 pins the Go connector's full-jitter base and cap as constants, with no construction option to override") + } + for kind, disposition := range cfg.SignalDisposition { + if kind != "bufferOverflow" && kind != "feedGap" { + return fmt.Errorf("signalDisposition: unknown signal kind %q", kind) + } + if disposition != "accept" && disposition != "terminate" { + return fmt.Errorf("signalDisposition[%s]: unknown disposition %q", kind, disposition) + } + } + if cfg.CheckpointStore != nil { + load := cfg.CheckpointStore.Load + switch { + case load == "missing", load == "failed", strings.HasPrefix(load, "loaded:"): + default: + return fmt.Errorf("checkpointStore.load %q is not `loaded:`, `missing`, or `failed`", load) + } + for _, outcome := range cfg.CheckpointStore.Save { + if outcome != "saved" && outcome != "failed" { + return fmt.Errorf("checkpointStore.save: unknown outcome %q", outcome) + } + } + if cfg.Position != "" { + return fmt.Errorf("position and checkpointStore are mutually exclusive") + } + } + return nil +} + +func validateFinally(fin scenarioFinally) error { + if err := validateStateName(fin.State); err != nil { + return err + } + switch fin.Socket { + case "", "none", "open", "closed": + default: + return fmt.Errorf("unknown socket disposition %q", fin.Socket) + } + if fin.Timers != nil { + if err := validateTimerSet(*fin.Timers); err != nil { + return err + } + } + if fin.Error != nil { + if err := validateErrorExpect(*fin.Error); err != nil { + return err + } + } + if fin.HandlerInvocations != nil { + return validateInvocations(fin.HandlerInvocations.Exact) + } + return nil +} + +func validateMintRespond(r mintRespond) error { + if r.Body != nil { + if r.Status != nil && *r.Status != 200 { + return fmt.Errorf("a mint body with status %d is not a modeled response", *r.Status) + } + if r.Body.Ticket == "" || r.Body.URL == "" || r.Body.ExpiresIn <= 0 { + return fmt.Errorf("a successful mint body needs ticket, expires_in, and url") + } + return nil + } + if r.Status == nil { + return fmt.Errorf("a mint response needs a body or a status") + } + switch *r.Status { + case 401, 403, 404, 422, 429, 500, 502, 503, 504: + return nil + default: + return fmt.Errorf("unknown mint status %d", *r.Status) + } +} + +func validateConnectOutcome(o *connectOutcome) error { + if o == nil { + return nil + } + set := 0 + if o.Accept != nil { + if !*o.Accept { + return fmt.Errorf("outcome.accept: only `true` is valid") + } + set++ + } + if o.Refuse != "" { + if o.Refuse != "upgrade-failure" { + return fmt.Errorf("outcome.refuse %q is not `upgrade-failure`", o.Refuse) + } + set++ + } + if o.RedirectTo != "" { + set++ + } + if set != 1 { + return fmt.Errorf("outcome names exactly one of accept/refuse/redirectTo") + } + return nil +} + +func validateServe(s *serveStep) error { + switch s.Frame { + case "welcome", "ping", "confirm", "reject": + return nil + case "disconnect": + if s.Reason == "" || s.Reconnect == nil { + return fmt.Errorf("a disconnect frame needs reason and reconnect") + } + return nil + case "message": + if len(s.Event) == 0 { + return fmt.Errorf("a message frame needs an event") + } + return validatePushEvent(s.Event) + case "raw": + return nil + default: + return fmt.Errorf("unknown frame %q", s.Frame) + } +} + +// validatePushEvent pins the 9-key push payload: a fixture's event body is +// forwarded to the connector VERBATIM, so the driver only checks that the +// contract's keys are all present (the schema requires them; a driver that +// never looked would let a rewritten fixture through). +func validatePushEvent(raw json.RawMessage) error { + keys, err := objectKeys(raw) + if err != nil { + return err + } + for _, key := range []string{ + "id", "kind", "event_type", "action", "created_at", + "bucket_id", "creator_id", "recording_id", "visible_to_clients", + } { + if _, ok := keys[key]; !ok { + return fmt.Errorf("push event is missing required key %q", key) + } + } + return allowedKeys("push event", keys, + "id", "kind", "event_type", "action", "created_at", + "bucket_id", "creator_id", "recording_id", "visible_to_clients") +} + +// decodePollQuery decodes expectPoll.query into its subset/exact form. +func decodePollQuery(step *expectPollStep) error { + if step.URL == "" && len(step.Query) == 0 { + return fmt.Errorf("a poll expectation pins nothing: give url or query") + } + if len(step.Query) == 0 { + return nil + } + keys, err := objectKeys(step.Query) + if err != nil { + return err + } + body := step.Query + if exact, ok := keys["exact"]; ok { + if len(keys) != 1 { + return fmt.Errorf("query.exact is exclusive with subset pins") + } + step.exactPin = true + body = exact + } + params := map[string]string{} + if err := decodeStrict(body, ¶ms); err != nil { + return err + } + for name := range params { + switch name { + case "position", "since", "types", "buckets", "creators": + default: + return fmt.Errorf("unknown query param %q", name) + } + } + step.params = params + return nil +} + +func validateTimerKind(kind string) error { + switch kind { + case "handshake-deadline", "confirmation-deadline", "backoff", + "staleness", "repair-poll", "poll-retry": + return nil + default: + return fmt.Errorf("unknown timer kind %q", kind) + } +} + +func validateTimerSet(set timerSet) error { + for kind, count := range set.Exact { + if err := validateTimerKind(kind); err != nil { + return err + } + if count < 1 { + return fmt.Errorf("timer %q count %d must be positive", kind, count) + } + } + return nil +} + +func validateStateName(state string) error { + switch state { + case "idle", "backoff", "minting", "connecting", "awaiting_welcome", + "awaiting_confirmation", "catching_up", "draining", "streaming", + "terminal", "closed": + return nil + default: + return fmt.Errorf("unknown state %q", state) + } +} + +func validateErrorExpect(e errorExpect) error { + switch e.Reason { + case "subscription_rejected", "protocol_fatal", "filter_invalid", + "authorization_failed", "checkpoint_load", "usage", "buffer_overflow", + "feed_gap", "invalid_continuation", "poll_failed", "mint_failed", + "invalid_cable_url": + default: + return fmt.Errorf("unknown terminal reason %q", e.Reason) + } + if e.Category != "" { + return fmt.Errorf("error.category is not modeled: the Go TerminalError carries a reason, not a §6 category") + } + return nil +} + +func validateSignalExpect(s *expectSignalStep) error { + switch s.Kind { + case "bufferOverflow": + if s.DroppedCount == nil || len(s.DroppedIDs) == 0 { + return fmt.Errorf("a bufferOverflow signal needs droppedIds and droppedCount") + } + if *s.DroppedCount != len(s.DroppedIDs) { + return fmt.Errorf("droppedCount %d disagrees with %d droppedIds", *s.DroppedCount, len(s.DroppedIDs)) + } + if s.EpochAfterID != nil { + return fmt.Errorf("a bufferOverflow signal carries no epochAfterId") + } + case "feedGap": + if s.EpochAfterID == nil { + return fmt.Errorf("a feedGap signal needs epochAfterId") + } + if len(s.DroppedIDs) > 0 || s.DroppedCount != nil { + return fmt.Errorf("a feedGap signal carries no dropped ids") + } + default: + return fmt.Errorf("unknown signal kind %q", s.Kind) + } + return nil +} + +func validateInvocations(records []invocation) error { + for _, rec := range records { + if rec.Kind != "bufferOverflow" && rec.Kind != "feedGap" { + return fmt.Errorf("unknown handler invocation kind %q", rec.Kind) + } + if rec.Disposition != "accept" && rec.Disposition != "terminate" { + return fmt.Errorf("unknown handler disposition %q", rec.Disposition) + } + } + return nil +} + +// --- decoding helpers ---------------------------------------------------- + +// decodeStrict decodes exactly one JSON value into dst, rejecting unknown +// object keys and trailing content. +func decodeStrict(raw json.RawMessage, dst any) error { + if len(raw) == 0 { + return fmt.Errorf("missing value") + } + dec := json.NewDecoder(bytes.NewReader(raw)) + dec.DisallowUnknownFields() + if err := dec.Decode(dst); err != nil { + return err + } + if dec.More() { + return fmt.Errorf("trailing JSON content") + } + return nil +} + +// objectKeys decodes a JSON object into its raw members. +func objectKeys(raw json.RawMessage) (map[string]json.RawMessage, error) { + keys := map[string]json.RawMessage{} + if err := json.Unmarshal(raw, &keys); err != nil { + return nil, fmt.Errorf("not a JSON object: %w", err) + } + return keys, nil +} + +// allowedKeys rejects any key outside the allowed set. +func allowedKeys(what string, keys map[string]json.RawMessage, allowed ...string) error { + for key := range keys { + found := false + for _, ok := range allowed { + if key == ok { + found = true + break + } + } + if !found { + return fmt.Errorf("%s: unknown key %q", what, key) + } + } + return nil +} + +// --- placeholder substitution -------------------------------------------- + +// placeholderPattern matches the family's `{{NAME}}` / `{{NAME:n}}` tokens. +var placeholderPattern = regexp.MustCompile(`\{\{([A-Z_]+)(?::(\d+))?\}\}`) + +// substitutePlaceholders replaces every fixture placeholder with this +// harness's own loopback origins and tokens, guaranteeing distinctness across +// indices ({{CABLE_URL:1}} ≠ {{CABLE_URL:2}} is what gives the fresh-ticket +// assertions teeth). Literal foreign origins are deliberately left alone. An +// unknown placeholder fails the fixture rather than surviving into a URL. +func substitutePlaceholders(raw []byte, h *scenarioHarness) ([]byte, error) { + var failure error + out := placeholderPattern.ReplaceAllFunc(raw, func(token []byte) []byte { + groups := placeholderPattern.FindSubmatch(token) + name := string(groups[1]) + index := 0 + if len(groups[2]) > 0 { + index, _ = strconv.Atoi(string(groups[2])) + } + switch name { + case "API_ORIGIN": + return []byte(h.apiOrigin) + case "TICKET": + return []byte(ticketToken(index)) + case "CABLE_URL": + return []byte(h.cableURL(index)) + case "POS": + return []byte(fmt.Sprintf("pos-%d", index)) + case "NEXT": + return []byte(fmt.Sprintf("%s/events.json?continuation=%d", h.apiOrigin, index)) + default: + if failure == nil { + failure = fmt.Errorf("unknown placeholder %s", token) + } + return token + } + }) + return out, failure +} + +func ticketToken(index int) string { return fmt.Sprintf("ticket-%d", index) } diff --git a/go/pkg/basecamp/eventfeed/scenario_harness_test.go b/go/pkg/basecamp/eventfeed/scenario_harness_test.go new file mode 100644 index 0000000000..4c0e386435 --- /dev/null +++ b/go/pkg/basecamp/eventfeed/scenario_harness_test.go @@ -0,0 +1,626 @@ +// Tier-2 conformance: the scenario harness. +// +// The Go lane drives the REAL default WebSocket transport over an in-process +// loopback cable server (`httptest` + coder/websocket server-side), with only +// the Clock faked — the lane the family README's consumers table names. The +// mint and poll lanes are SEAM calls (the Layer-1 adapter over the generated +// CreateStreamTicket/PollEvents operations lands with Layer 1), so this +// harness scripts them at `TicketMinter`/`PollSource` with the family's +// parked-call semantics: a seam call blocks inside the seam until the driver +// reaches its matching expect step. +// +// Everything the connector reports — deliveries, saves, signals, handler +// invocations, observer notifications, state transitions, buffer occupancy — +// arrives on the consumer's goroutine (ranging Events IS the state machine), +// so the harness records a single serialized history and the driver asserts +// over it. No callback ever fails a test directly: violations are recorded and +// surfaced by the driver goroutine, which is the only one that reports. +package eventfeed_test + +import ( + "context" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "slices" + "strings" + "sync" + "time" + + "github.com/coder/websocket" + + "github.com/basecamp/basecamp-sdk/go/pkg/basecamp/eventfeed" + "github.com/basecamp/basecamp-sdk/go/pkg/basecamp/eventfeed/feedtest" +) + +// scenarioWatchdog bounds every rendezvous. Virtual time is frozen between +// directives, so this is the only wall clock in the driver. It is a var +// solely so the driver's own tests can shorten it while asserting that a +// rendezvous which can never be satisfied does time out; scenarios always run +// under the full window. +var scenarioWatchdog = 5 * time.Second + +// scenarioAccountID is the account every scenario's checkpoint identity binds +// to; scenarioNamespace is the fixed consumer namespace a configured store +// requires. +const ( + scenarioAccountID = "5951425" + scenarioNamespace = "conformance" +) + +// harness owns the scenario's servers, seams, and recorded history. One +// mutex guards everything: the connector's callbacks and the driver goroutine +// are the only writers, and every mutation broadcasts so rendezvous waiters +// wake. +type scenarioHarness struct { + mu sync.Mutex + changed chan struct{} + + clock *feedtest.Clock + + cableSrv *httptest.Server + apiSrv *httptest.Server + apiOrigin string + cableBase string + + // connectPlan is the scenario's expectConnect outcomes, in order: the Nth + // handshake is answered by the Nth plan entry (dials happen strictly in + // order, so the indices line up). + connectPlan []connectPlanEntry + + // Seam parking. + mintCalls int + pendingMints []*mintCall + pollCalls int + pendingPolls []*pollCall + + // Cable lane. + connects []*connectAttempt + peers []*cablePeer + + // Checkpoint store script. + storeLoad string + storeLoadOK bool + storeLoadErr error + saveScript []string + saveScripted bool + saveUsed int + + // Recorded history. + delivered []int64 + saves []saveRecord + savesTaken int + occupancy int + occupancyHistory []int + signals []eventfeed.Signal + signalsTaken int + invocations []invocation + gaps []int64 + gapsTaken int + saveFailures int + saveFailuresTaken int + positionRejected []string + posRejectedTaken int + invalidFrames int + invalidFramesTaken int + state string + terminal *eventfeed.TerminalError + iterDone bool + violations []string +} + +// saveRecord is one CheckpointStore.Save call plus the ordering witness the +// save-ordering assertions need: how many events had been delivered when the +// save was made. A save that arrives before the deliveries its script +// requires is exactly checkpoint-before-handoff. +type saveRecord struct { + position string + deliveredAt int +} + +type connectPlanEntry struct { + kind string // "accept" | "refuse" | "redirect" + target string +} + +type connectAttempt struct { + url string + kind string + peer *cablePeer +} + +// cablePeer is one accepted loopback cable connection: the server side of the +// socket the connector dialed. +type cablePeer struct { + conn *websocket.Conn + + frames []clientFrame + taken int + dead bool +} + +// clientFrame is one thing the connector sent: a text frame, or its +// client-initiated close. +type clientFrame struct { + kind string // "text" | "close" + data []byte + code int +} + +type mintCall struct{ release chan mintOutcome } + +type mintOutcome struct { + ticket eventfeed.StreamTicket + err error +} + +type pollCall struct { + cursor eventfeed.Cursor + filters eventfeed.Filters + release chan pollOutcome +} + +type pollOutcome struct { + page eventfeed.PollPage + err error +} + +// newScenarioHarness starts the loopback servers and returns a ready harness. +func newScenarioHarness() *scenarioHarness { + h := &scenarioHarness{ + changed: make(chan struct{}), + clock: feedtest.NewClock(), + } + h.cableSrv = httptest.NewServer(http.HandlerFunc(h.handleCable)) + h.apiSrv = httptest.NewServer(http.HandlerFunc(h.handleAPI)) + h.apiOrigin = h.apiSrv.URL + h.cableBase = "ws" + strings.TrimPrefix(h.cableSrv.URL, "http") + return h +} + +// close tears the harness down: live peers first (so hijacked handlers +// return), then the servers. +func (h *scenarioHarness) close() { + h.mu.Lock() + peers := slices.Clone(h.peers) + h.mu.Unlock() + for _, p := range peers { + _ = p.conn.CloseNow() + } + h.cableSrv.Close() + h.apiSrv.Close() +} + +// cableURL renders the nth cable URL — distinct per index, loopback ws:// per +// the SPEC §9 carve-out, ticket in the query string exactly as a mint's URL +// carries it. +func (h *scenarioHarness) cableURL(index int) string { + return fmt.Sprintf("%s/cable/%d?ticket=%s", h.cableBase, index, ticketToken(index)) +} + +// --- broadcast + rendezvous ---------------------------------------------- + +// notifyLocked wakes every rendezvous waiter. Callers hold h.mu. +func (h *scenarioHarness) notifyLocked() { + close(h.changed) + h.changed = make(chan struct{}) +} + +// violate records a harness-observed contract violation. Callbacks never fail +// the test directly — the driver goroutine surfaces this at its next +// rendezvous or at `finally`. +func (h *scenarioHarness) violate(format string, args ...any) { + h.mu.Lock() + defer h.mu.Unlock() + h.violations = append(h.violations, fmt.Sprintf(format, args...)) + h.notifyLocked() +} + +// await blocks until check reports satisfaction, fails immediately when check +// reports an unsatisfiable state, and gives up at the watchdog. check runs +// under h.mu and may consume harness state on the satisfying call. +func (h *scenarioHarness) await(what string, check func() (bool, string)) error { + deadline := time.Now().Add(scenarioWatchdog) + for { + h.mu.Lock() + violation := "" + if len(h.violations) > 0 { + violation = h.violations[0] + } + ok, bad := false, "" + if violation == "" { + ok, bad = check() + } + wake := h.changed + h.mu.Unlock() + switch { + case violation != "": + return fmt.Errorf("while waiting for %s: %s", what, violation) + case bad != "": + return fmt.Errorf("%s: %s", what, bad) + case ok: + return nil + } + remaining := time.Until(deadline) + if remaining <= 0 { + return fmt.Errorf("timed out after %s waiting for %s\n%s", scenarioWatchdog, what, h.describe()) + } + timer := time.NewTimer(remaining) + select { + case <-wake: + case <-timer.C: + } + timer.Stop() + } +} + +// describe renders the harness's current history for a timeout message. +func (h *scenarioHarness) describe() string { + h.mu.Lock() + defer h.mu.Unlock() + var b strings.Builder + fmt.Fprintf(&b, " state=%s delivered=%v buffered=%d\n", h.state, h.delivered, h.occupancy) + fmt.Fprintf(&b, " mintCalls=%d (%d parked) pollCalls=%d (%d parked) connects=%d\n", + h.mintCalls, len(h.pendingMints), h.pollCalls, len(h.pendingPolls), len(h.connects)) + positions := make([]string, len(h.saves)) + for i, rec := range h.saves { + positions[i] = rec.position + } + fmt.Fprintf(&b, " saves=%v invocations=%v gaps=%v\n", positions, h.invocations, h.gaps) + fmt.Fprintf(&b, " timers=%v terminal=%v iterationEnded=%t\n", h.clock.Outstanding(), h.terminal, h.iterDone) + return b.String() +} + +// --- seams --------------------------------------------------------------- + +// MintStreamTicket implements eventfeed.TicketMinter with the family's parked +// semantics: the call is counted and parked, and only the matching +// `expectMint` step releases its scripted response. +func (h *scenarioHarness) MintStreamTicket(ctx context.Context) (eventfeed.StreamTicket, error) { + call := &mintCall{release: make(chan mintOutcome, 1)} + h.mu.Lock() + h.mintCalls++ + h.pendingMints = append(h.pendingMints, call) + h.notifyLocked() + h.mu.Unlock() + select { + case out := <-call.release: + return out.ticket, out.err + case <-ctx.Done(): + return eventfeed.StreamTicket{}, ctx.Err() + } +} + +// Poll implements eventfeed.PollSource with the same parked semantics. +func (h *scenarioHarness) Poll(ctx context.Context, cursor eventfeed.Cursor, filters eventfeed.Filters) (eventfeed.PollPage, error) { + call := &pollCall{cursor: cursor, filters: filters, release: make(chan pollOutcome, 1)} + h.mu.Lock() + h.pollCalls++ + h.pendingPolls = append(h.pendingPolls, call) + h.notifyLocked() + h.mu.Unlock() + select { + case out := <-call.release: + return out.page, out.err + case <-ctx.Done(): + return eventfeed.PollPage{}, ctx.Err() + } +} + +// Load implements eventfeed.CheckpointStore's tri-state load. +func (h *scenarioHarness) Load(_ context.Context, _ eventfeed.CheckpointKey) (string, bool, error) { + h.mu.Lock() + defer h.mu.Unlock() + return h.storeLoad, h.storeLoadOK, h.storeLoadErr +} + +// Save implements eventfeed.CheckpointStore's save, recording the call with +// its ordering witness and returning the scripted outcome. A save beyond the +// script is a violation — the exact store-call script is what proves a +// subsequent save is attempted after a failure. +func (h *scenarioHarness) Save(_ context.Context, _ eventfeed.CheckpointKey, position string) error { + h.mu.Lock() + defer h.mu.Unlock() + h.saves = append(h.saves, saveRecord{position: position, deliveredAt: len(h.delivered)}) + var err error + if h.saveScripted { + if h.saveUsed >= len(h.saveScript) { + h.violations = append(h.violations, + fmt.Sprintf("save(%s) beyond the scripted store calls %v", position, h.saveScript)) + } else { + if h.saveScript[h.saveUsed] == "failed" { + err = errors.New("feedtest: scripted checkpoint save failure") + } + h.saveUsed++ + } + } + h.notifyLocked() + return err +} + +// --- cable loopback ------------------------------------------------------ + +// handleCable serves one dialed cable connection, recording the request-line +// URL verbatim first — even when the scenario scripts the dial to be refused. +func (h *scenarioHarness) handleCable(w http.ResponseWriter, r *http.Request) { + h.mu.Lock() + plan := connectPlanEntry{kind: "accept"} + if idx := len(h.connects); idx < len(h.connectPlan) { + plan = h.connectPlan[idx] + } + attempt := &connectAttempt{url: "ws://" + r.Host + r.RequestURI, kind: plan.kind} + h.connects = append(h.connects, attempt) + h.notifyLocked() + h.mu.Unlock() + + switch plan.kind { + case "refuse": + w.WriteHeader(http.StatusBadRequest) + return + case "redirect": + http.Redirect(w, r, plan.target, http.StatusFound) + return + } + conn, err := websocket.Accept(w, r, &websocket.AcceptOptions{ + Subprotocols: []string{"actioncable-v1-json"}, + }) + if err != nil { + h.violate("the cable upgrade failed: %v", err) + return + } + peer := &cablePeer{conn: conn} + h.mu.Lock() + attempt.peer = peer + h.peers = append(h.peers, peer) + h.notifyLocked() + h.mu.Unlock() + //nolint:contextcheck // deliberate: after the upgrade the connection's + // lifetime is scripted by the scenario, not bound to r.Context(). + h.readPeer(peer) +} + +// readPeer records everything the connector sends on one socket, in order: a +// text frame (the subscribe command), or its client-initiated close. +func (h *scenarioHarness) readPeer(peer *cablePeer) { + for { + typ, data, err := peer.conn.Read(context.Background()) + if err != nil { + var closeErr websocket.CloseError + h.mu.Lock() + if errors.As(err, &closeErr) { + peer.frames = append(peer.frames, clientFrame{kind: "close", code: int(closeErr.Code)}) + } + peer.dead = true + h.notifyLocked() + h.mu.Unlock() + return + } + if typ != websocket.MessageText { + h.violate("the connector sent a non-text cable frame") + continue + } + h.mu.Lock() + peer.frames = append(peer.frames, clientFrame{kind: "text", data: slices.Clone(data)}) + h.notifyLocked() + h.mu.Unlock() + } +} + +// handleAPI is the API origin's sentinel: the connector reaches the mint and +// poll lanes through seams, so ANY HTTP request arriving here is egress the +// scenario never scripted. +func (h *scenarioHarness) handleAPI(w http.ResponseWriter, r *http.Request) { + h.violate("an unscripted HTTP request reached the API origin: %s %s", r.Method, r.URL) + w.WriteHeader(http.StatusTeapot) +} + +// closePeer closes one socket server-side, with a close frame (contrast +// `sever`, which drops the TCP connection abruptly). +func (h *scenarioHarness) closePeer(peer *cablePeer, code int, reason string) error { + return peer.conn.Close(websocket.StatusCode(code), reason) +} + +// writeFrame sends one server frame to the peer. +func (h *scenarioHarness) writeFrame(peer *cablePeer, data []byte) error { + ctx, cancel := context.WithTimeout(context.Background(), scenarioWatchdog) + defer cancel() + return peer.conn.Write(ctx, websocket.MessageText, data) +} + +// --- recording ----------------------------------------------------------- + +func (h *scenarioHarness) recordDelivered(id int64) { + h.mu.Lock() + defer h.mu.Unlock() + h.delivered = append(h.delivered, id) + h.notifyLocked() +} + +func (h *scenarioHarness) recordTerminal(err error) { + h.mu.Lock() + defer h.mu.Unlock() + var terr *eventfeed.TerminalError + if !errors.As(err, &terr) { + h.violations = append(h.violations, fmt.Sprintf("the feed yielded a non-terminal error element: %v", err)) + } else { + h.terminal = terr + } + h.notifyLocked() +} + +func (h *scenarioHarness) recordIterDone() { + h.mu.Lock() + defer h.mu.Unlock() + h.iterDone = true + h.notifyLocked() +} + +func (h *scenarioHarness) recordState(state string) { + h.mu.Lock() + defer h.mu.Unlock() + h.state = state + h.notifyLocked() +} + +func (h *scenarioHarness) recordOccupancy(count int) { + h.mu.Lock() + defer h.mu.Unlock() + h.occupancy = count + h.occupancyHistory = append(h.occupancyHistory, count) + h.notifyLocked() +} + +func (h *scenarioHarness) recordSignal(signal eventfeed.Signal) { + h.mu.Lock() + defer h.mu.Unlock() + h.signals = append(h.signals, signal) + h.notifyLocked() +} + +func (h *scenarioHarness) recordInvocation(rec invocation) { + h.mu.Lock() + defer h.mu.Unlock() + h.invocations = append(h.invocations, rec) + h.notifyLocked() +} + +func (h *scenarioHarness) recordGap(epochAfterID int64) { + h.mu.Lock() + defer h.mu.Unlock() + h.gaps = append(h.gaps, epochAfterID) + h.notifyLocked() +} + +func (h *scenarioHarness) recordSaveFailed() { + h.mu.Lock() + defer h.mu.Unlock() + h.saveFailures++ + h.notifyLocked() +} + +func (h *scenarioHarness) recordPositionRejected(kind string) { + h.mu.Lock() + defer h.mu.Unlock() + h.positionRejected = append(h.positionRejected, kind) + h.notifyLocked() +} + +func (h *scenarioHarness) recordInvalidFrame() { + h.mu.Lock() + defer h.mu.Unlock() + h.invalidFrames++ + h.notifyLocked() +} + +// --- connector construction ---------------------------------------------- + +// newConnector builds the scenario's connector: real transport, virtual +// clock, parked seams, scripted store, recording handler and observer. +// +// Two mappings are worth stating outright. (1) A store is configured for +// EVERY scenario, because `finally.checkpoints` is an assertion about +// CheckpointStore.save calls and the Go connector makes none without one; +// `config.position` therefore enters as the store's Loaded outcome rather +// than through StartAtPosition, which §23 makes mutually exclusive with a +// store. Both resolve to the identical position-resume entry cursor, so the +// scenario means the same thing either way. (2) `config.signalDisposition` +// installs a handler ONLY when the fixture carries the key: an absent key is +// load-bearing (the default-terminal path), so with no dispositions at all no +// handler is registered, and a kind whose key is absent is answered Terminate +// WITHOUT recording an invocation — the no-handler behavior, which is what +// the per-signal default-terminal pins assert. +func (h *scenarioHarness) newConnector(cfg scenarioConfig) (*eventfeed.Connector, error) { + h.applyStoreScript(cfg) + + opts := []eventfeed.Option{ + eventfeed.WithClock(h.clock), + eventfeed.WithCheckpointStore(h), + eventfeed.WithConsumerNamespace(scenarioNamespace), + eventfeed.WithFilters(eventfeed.Filters{ + Types: cfg.Types, + Buckets: cfg.Buckets, + Creators: cfg.Creators, + }), + eventfeed.WithObserver(eventfeed.Observer{ + Gap: func(epochAfterID int64, _ string) { h.recordGap(epochAfterID) }, + CheckpointSaveFailed: func(error) { h.recordSaveFailed() }, + PositionRejected: func(kind eventfeed.PollErrorKind) { h.recordPositionRejected(kind.String()) }, + Disconnected: func(_ string, err error) { + if eventfeed.ExportIsInvalidFrameError(err) { + h.recordInvalidFrame() + } + }, + }), + } + if cfg.ConfirmationDeadlineMs > 0 { + opts = append(opts, eventfeed.WithConfirmationDeadline(millis(cfg.ConfirmationDeadlineMs))) + } + if cfg.RepairPollBaseMs > 0 { + opts = append(opts, eventfeed.WithRepairInterval(millis(cfg.RepairPollBaseMs))) + } + if cfg.LiveBufferCapacity > 0 { + opts = append(opts, eventfeed.WithLiveBufferCapacity(cfg.LiveBufferCapacity)) + } + if cfg.DedupeCapacity > 0 { + opts = append(opts, eventfeed.WithDedupeCapacity(cfg.DedupeCapacity)) + } + if len(cfg.SignalDisposition) > 0 { + opts = append(opts, eventfeed.WithSignalHandler(h.signalHandler(cfg.SignalDisposition))) + } + + conn, err := eventfeed.New(h.apiOrigin, scenarioAccountID, h, h, opts...) + if err != nil { + return nil, err + } + if cfg.StalenessMs > 0 { + conn.SetStaleAfter(millis(cfg.StalenessMs)) + } + conn.OnStateChanged(h.recordState) + conn.OnBufferOccupancy(h.recordOccupancy) + conn.OnSignal(h.recordSignal) + return conn, nil +} + +// applyStoreScript wires config.checkpointStore (or config.position) into the +// harness store. +func (h *scenarioHarness) applyStoreScript(cfg scenarioConfig) { + h.mu.Lock() + defer h.mu.Unlock() + switch { + case cfg.CheckpointStore == nil && cfg.Position != "": + h.storeLoad, h.storeLoadOK = cfg.Position, true + case cfg.CheckpointStore == nil: + // Missing: the entry is the bare present. + case cfg.CheckpointStore.Load == "failed": + h.storeLoadErr = errors.New("feedtest: scripted checkpoint load failure") + case strings.HasPrefix(cfg.CheckpointStore.Load, "loaded:"): + h.storeLoad, h.storeLoadOK = strings.TrimPrefix(cfg.CheckpointStore.Load, "loaded:"), true + } + if cfg.CheckpointStore != nil && len(cfg.CheckpointStore.Save) > 0 { + h.saveScript, h.saveScripted = cfg.CheckpointStore.Save, true + } +} + +// signalHandler builds the recording handler for the configured dispositions. +func (h *scenarioHarness) signalHandler(dispositions map[string]string) eventfeed.SignalHandler { + return func(signal eventfeed.Signal) eventfeed.Disposition { + kind := "feedGap" + if _, ok := signal.(eventfeed.BufferOverflow); ok { + kind = "bufferOverflow" + } + disposition, configured := dispositions[kind] + if !configured { + // No handler exists for this kind: answer exactly as the absent + // handler would, and record nothing. + return eventfeed.Terminate + } + h.recordInvocation(invocation{Kind: kind, Disposition: disposition}) + if disposition == "accept" { + return eventfeed.Accept + } + return eventfeed.Terminate + } +} + +func millis(ms int) time.Duration { return time.Duration(ms) * time.Millisecond } diff --git a/go/pkg/basecamp/eventfeed/scenario_selftest_test.go b/go/pkg/basecamp/eventfeed/scenario_selftest_test.go new file mode 100644 index 0000000000..5da9cc5881 --- /dev/null +++ b/go/pkg/basecamp/eventfeed/scenario_selftest_test.go @@ -0,0 +1,378 @@ +// Tier-2 conformance: the driver's own tests. +// +// A driver that cannot fail proves nothing. Every case here derives a HOSTILE +// scenario in memory — never a committed fixture; `conformance/event-feed/` is +// merged contract — and requires the driver to reject it. The mutation cases +// follow the family's own pin-probe discipline: each asserts the unmutated +// control PASSES and the single-delta mutant FAILS, so a case that stops +// discriminating fails loudly instead of going vacuous. +package eventfeed_test + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +// TestScenarioDriverRejectsMutatedFixtures is the driver's kill test: a +// one-field mutation of a real fixture must turn a passing scenario red. +func TestScenarioDriverRejectsMutatedFixtures(t *testing.T) { + for _, probe := range []struct { + name string + fixture string + path string + value any + // wants is a fragment the failure must name, so a mutant rejected for + // the wrong reason cannot pass as the pin firing. + wants string + }{ + { + name: "delivered set", + fixture: "01-happy-path-confirm-catchup-stream.json", + path: "finally.delivered.exact", + value: []any{101, 102, 103, 104}, + wants: "finally.delivered", + }, + { + name: "checkpoint ledger", + fixture: "01-happy-path-confirm-catchup-stream.json", + path: "finally.checkpoints.exact", + value: []any{"{{POS:1}}"}, + wants: "finally.checkpoints", + }, + { + name: "mint seam-call count", + fixture: "05-fresh-ticket-reconnect-after-ttl.json", + path: "finally.mintCount", + value: 1, + wants: "finally.mintCount", + }, + { + name: "outstanding timer set", + fixture: "04-terminal-rejection.json", + path: "finally.timers.exact", + value: map[string]any{"backoff": 1}, + wants: "finally.timers", + }, + { + name: "terminal reason", + fixture: "06-protocol-fatal-disconnect.json", + path: "finally.error.reason", + value: "subscription_rejected", + wants: "finally.error", + }, + { + name: "handler invocation record", + fixture: "24-overflow-handler-terminate.json", + path: "finally.handlerInvocations.exact", + value: []any{map[string]any{"kind": "bufferOverflow", "disposition": "accept"}}, + wants: "finally.handlerInvocations", + }, + { + name: "reconnect dials the previous mint's url", + fixture: "05-fresh-ticket-reconnect-after-ttl.json", + path: "steps.13.expectConnect.url", + value: "{{CABLE_URL:1}}", + wants: "a cable dial", + }, + { + name: "catch-up poll cursor", + fixture: "12-checkpoint-after-handoff.json", + path: "steps.5.expectPoll.query.position", + value: "{{POS:9}}", + wants: "poll query", + }, + { + name: "subscribe identifier params", + fixture: "01-happy-path-confirm-catchup-stream.json", + path: "steps.3.expectSubscribe.params.types", + value: "todo.created", + wants: "subscribe identifier", + }, + { + name: "signal dropped ids", + fixture: "21-overflow-default-terminal.json", + path: "steps.7.expectSignal.droppedIds", + value: []any{52}, + wants: "dropped ids", + }, + { + name: "buffered occupancy rendezvous", + fixture: "19-present-entry-buffered-lower-id.json", + path: "steps.6.expectBuffered.count", + value: 2, + wants: "live buffer occupancy", + }, + } { + t.Run(probe.name, func(t *testing.T) { + control := readFixture(t, probe.fixture) + if err := runScenarioBytes(control, probe.fixture); err != nil { + t.Fatalf("the control must pass, else the mutant proves nothing: %v", err) + } + mutant := mutateFixture(t, control, probe.path, probe.value) + err := underShortWatchdog(func() error { return runScenarioBytes(mutant, probe.fixture) }) + if err == nil { + t.Fatalf("the driver accepted %s = %v — a mutated expectation must turn the scenario red", probe.path, probe.value) + } + if !strings.Contains(err.Error(), probe.wants) { + t.Fatalf("the mutant failed for the wrong reason:\n got %v\n want a failure naming %q", err, probe.wants) + } + }) + } +} + +// TestScenarioDriverEnforcesDeliveryBeforeCheckpoint pins the driver's +// arrival-strict save rule — the teeth of checkpoint-after-handoff, which no +// end-state assertion can catch. Fixture 02's entry is position-resume, so its +// empty final page legitimately saves BEFORE the buffered event drains, and +// the fixture scripts exactly that order. Swapping its last two steps demands +// the delivery first; the save still arrives first, and the driver must say so +// rather than matching it against a later step. +func TestScenarioDriverEnforcesDeliveryBeforeCheckpoint(t *testing.T) { + const fixture = "02-confirmation-gating.json" + control := readFixture(t, fixture) + if err := runScenarioBytes(control, fixture); err != nil { + t.Fatalf("the control must pass, else the reordering proves nothing: %v", err) + } + + var doc map[string]any + if err := json.Unmarshal(control, &doc); err != nil { + t.Fatalf("decoding the control fixture: %v", err) + } + steps, ok := doc["steps"].([]any) + if !ok || len(steps) < 2 { + t.Fatalf("the control fixture has no steps to reorder") + } + last, penultimate := len(steps)-1, len(steps)-2 + if !stepIs(steps[penultimate], "expectCheckpoint") || !stepIs(steps[last], "expectDelivered") { + t.Fatalf("the control fixture no longer ends in expectCheckpoint, expectDelivered — re-derive this probe") + } + steps[penultimate], steps[last] = steps[last], steps[penultimate] + mutant, err := json.Marshal(doc) + if err != nil { + t.Fatalf("re-encoding the mutant: %v", err) + } + + err = underShortWatchdog(func() error { return runScenarioBytes(mutant, fixture) }) + if err == nil { + t.Fatal("a save that arrives before the deliveries the script requires must fail the scenario") + } + if !strings.Contains(err.Error(), "delivery strictly precedes") { + t.Fatalf("failed for the wrong reason:\n got %v\n want the save-ordering rule", err) + } +} + +func stepIs(step any, directive string) bool { + object, ok := step.(map[string]any) + if !ok { + return false + } + _, present := object[directive] + return present +} + +// TestScenarioDriverRejectsUnmodelledScripts pins the load-time half: an +// unknown directive, an unknown key, an unknown placeholder, or a construct +// the driver does not model must abort the fixture with a named error rather +// than being skipped. This is what keeps the driver honest as PR 4 adds +// fixtures and schema constructs. +func TestScenarioDriverRejectsUnmodelledScripts(t *testing.T) { + base := readFixture(t, "04-terminal-rejection.json") + for _, probe := range []struct { + name string + script string + wants string + }{ + { + name: "unknown directive", + script: `{"name":"x","description":"d","steps":[{"expectSomethingNew":{}}],"finally":{"state":"closed"}}`, + wants: "unknown step directive", + }, + { + name: "two directives in one step", + script: `{"name":"x","description":"d","steps":[{"sever":true,"advance":{"ms":1}}],"finally":{"state":"closed"}}`, + wants: "exactly one directive", + }, + { + name: "unknown key inside a directive", + script: `{"name":"x","description":"d","steps":[{"advance":{"ms":1,"jitter":2}}],"finally":{"state":"closed"}}`, + wants: "jitter", + }, + { + name: "unknown top-level key", + script: `{"name":"x","description":"d","only":true,"steps":[{"advance":{"ms":1}}],"finally":{"state":"closed"}}`, + wants: `unknown key "only"`, + }, + { + name: "unknown placeholder", + script: `{"name":"x","description":"d","steps":[{"expectConnect":{"url":"{{CABLE_HOST:1}}"}}],"finally":{"state":"closed"}}`, + wants: "unknown placeholder", + }, + { + name: "unknown timer kind", + script: `{"name":"x","description":"d","steps":[{"fireTimer":{"kind":"reconnect"}}],"finally":{"state":"closed"}}`, + wants: "unknown timer kind", + }, + { + name: "unknown terminal reason", + script: `{"name":"x","description":"d","steps":[{"expectError":{"reason":"exploded"}}],"finally":{"state":"terminal"}}`, + wants: "unknown terminal reason", + }, + { + name: "inverted fireTimer envelope", + script: `{"name":"x","description":"d","steps":[{"fireTimer":{"kind":"backoff","assertDelayMs":{"min":1000,"max":10}}}],"finally":{"state":"closed"}}`, + wants: "exceeds max", + }, + { + name: "droppedCount disagreeing with droppedIds", + script: `{"name":"x","description":"d","steps":[{"expectSignal":{"kind":"bufferOverflow","droppedIds":[1],"droppedCount":2}}],"finally":{"state":"closed"}}`, + wants: "disagrees with", + }, + { + name: "a push event missing a payload key", + script: `{"name":"x","description":"d","steps":[{"serve":{"frame":"message","event":{"id":1,"kind":"m","event_type":"t","action":"a","created_at":"2026-08-01T12:00:00Z","bucket_id":2,"creator_id":3,"recording_id":4}}}],"finally":{"state":"closed"}}`, + wants: "visible_to_clients", + }, + { + name: "a construct the driver does not model", + script: `{"name":"x","description":"d","config":{"backoffBaseMs":1000},"steps":[{"advance":{"ms":1}}],"finally":{"state":"closed"}}`, + wants: "not modeled", + }, + { + name: "a name disagreeing with the filename", + script: `{"name":"other-name","description":"d","steps":[{"advance":{"ms":1}}],"finally":{"state":"closed"}}`, + wants: "must match its filename", + }, + } { + t.Run(probe.name, func(t *testing.T) { + err := runScenarioBytes([]byte(probe.script), "x.json") + if err == nil { + t.Fatalf("the driver accepted a script it cannot model: %s", probe.script) + } + if !strings.Contains(err.Error(), probe.wants) { + t.Fatalf("rejected for the wrong reason:\n got %v\n want a failure naming %q", err, probe.wants) + } + }) + } + // The base fixture still passes: the cases above reject scripts, not + // everything. + if err := runScenarioBytes(base, "04-terminal-rejection.json"); err != nil { + t.Fatalf("the control fixture must still pass: %v", err) + } +} + +// TestScenarioDriverRejectsUnmatchedActions pins the strict interleave's +// residue half: a protocol action no expect step matched, and a timer +// directive with no such timer armed, each fail rather than passing silently. +func TestScenarioDriverRejectsUnmatchedActions(t *testing.T) { + t.Run("an unmatched mint seam call", func(t *testing.T) { + // The connector mints immediately; the script expects nothing and + // claims a terminal state, so the parked call is never matched. + script := `{"name":"x","description":"d","steps":[{"expectState":{"is":"minting"}}], + "finally":{"state":"minting","mintCount":0}}` + err := underShortWatchdog(func() error { return runScenarioBytes([]byte(script), "x.json") }) + if err == nil { + t.Fatal("a mint seam call no expect step matched must fail the scenario") + } + if !strings.Contains(err.Error(), "finally.mintCount") && !strings.Contains(err.Error(), "still parked") { + t.Fatalf("failed for the wrong reason: %v", err) + } + }) + + t.Run("a fireTimer with no such timer armed", func(t *testing.T) { + script := `{"name":"x","description":"d","steps":[{"fireTimer":{"kind":"poll-retry"}}], + "finally":{"state":"minting"}}` + err := underShortWatchdog(func() error { return runScenarioBytes([]byte(script), "x.json") }) + if err == nil { + t.Fatal("firing a timer that is not armed must fail the scenario, never pass vacuously") + } + if !strings.Contains(err.Error(), "poll-retry") { + t.Fatalf("failed for the wrong reason: %v", err) + } + }) +} + +// underShortWatchdog runs a scenario that is EXPECTED to fail under a short +// rendezvous window. A hostile scenario often fails by never satisfying a +// rendezvous, and waiting the full window for each would cost more than the +// whole conformance suite; every caller still pins the failure's reason, so a +// mutant rejected for the wrong reason cannot pass as the pin firing. +func underShortWatchdog(run func() error) error { + previous := scenarioWatchdog + scenarioWatchdog = 500 * time.Millisecond + defer func() { scenarioWatchdog = previous }() + return run() +} + +// readFixture loads one committed fixture verbatim. +func readFixture(t *testing.T, name string) []byte { + t.Helper() + raw, err := os.ReadFile(filepath.Join(filepath.Dir(scenarioFixtureGlob), name)) + if err != nil { + t.Fatalf("reading fixture %s: %v", name, err) + } + return raw +} + +// mutateFixture derives a single-delta mutant in memory: it walks a +// dot-separated path (numeric segments index `steps`) and replaces the value +// there. A path that does not resolve, or a mutation equal to the control's +// value, fails the probe as vacuous. +func mutateFixture(t *testing.T, raw []byte, path string, value any) []byte { + t.Helper() + var doc any + if err := json.Unmarshal(raw, &doc); err != nil { + t.Fatalf("decoding the control fixture: %v", err) + } + segments := strings.Split(path, ".") + cursor := doc + for _, segment := range segments[:len(segments)-1] { + cursor = descend(t, cursor, segment, path) + } + last := segments[len(segments)-1] + container, ok := cursor.(map[string]any) + if !ok { + t.Fatalf("path %s does not end in an object", path) + } + previous, present := container[last] + if !present { + t.Fatalf("path %s is absent from the control — a mutation must be a real delta", path) + } + if fmt.Sprint(previous) == fmt.Sprint(value) { + t.Fatalf("path %s already holds %v — a mutation equal to the control proves nothing", path, value) + } + container[last] = value + mutant, err := json.Marshal(doc) + if err != nil { + t.Fatalf("re-encoding the mutant: %v", err) + } + return mutant +} + +func descend(t *testing.T, cursor any, segment, path string) any { + t.Helper() + switch node := cursor.(type) { + case map[string]any: + next, ok := node[segment] + if !ok { + t.Fatalf("path %s: %q is absent", path, segment) + } + return next + case []any: + index := 0 + if _, err := fmt.Sscanf(segment, "%d", &index); err != nil { + t.Fatalf("path %s: %q is not an array index", path, segment) + } + if index >= len(node) { + t.Fatalf("path %s: index %d is out of range (%d entries)", path, index, len(node)) + } + return node[index] + default: + t.Fatalf("path %s: %q has no children", path, segment) + return nil + } +} From 9a8c96b65565c911ec16a81fbc6910dd9cef1b10 Mon Sep 17 00:00:00 2001 From: Jeremy Daer Date: Tue, 18 Aug 2026 11:13:11 -0700 Subject: [PATCH 02/51] Event feed: a poll page carrying no position is malformed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Daybreak blocker 2. The walk took page.Position on trust, and an empty one silently skipped history in two different ways depending on the entry class. Position-resume: acceptPosition("") sets l.position = "", and entryCursor selects on `l.position != ""` — so it does not preserve the old cursor, it falls through to the StartResume default, which is a BARE PRESENT ENTRY. The feed resumes at the server's head with everything between the real position and now skipped, and reports nothing. Present-class is worse. `held` uses "" as its sentinel for "the final entry was not present-class", so an empty position is not saved as empty — it collapses into the sentinel, `held != ""` skips acceptPosition AND saveCheckpoint outright, and caught_up is announced anyway. The entry position is DISCARDED, with the drain's deliveries already handed to the consumer and nothing durable recording them. Both are the same silent skip that a found-but-empty STORED position is already refused for at load (d379f2e11), arriving by the other door. The guard is the same shape and in the same spirit, and it also makes saveCheckpoint's stated contract true: it documents that "the live cursor is neither regressed nor blanked", which the assignment one line above its call site did. Terminal(poll_failed) is what §23 already classifies "an unexpected shape" from the poll lane as, and it is not retried — a server that answered without a position answers the same way to the same request. The refusal precedes delivery, the counter resets, and every mutation of the walk's state. That placement is the part worth pinning, so it is what the mutation check exercises: moving the guard to AFTER the delivery loop fails three of the four subtests on `delivered ids = [101], want []`, while the "is not retried" subtest still passes — the subtests discriminate different properties rather than all keying on one. Removing the guard entirely fails all four, but by watchdog timeout rather than on an assertion, and that is worth stating plainly: the un-fixed feed does not misbehave visibly, it proceeds into Streaming as though nothing happened. That IS the defect. --- go/pkg/basecamp/eventfeed/catchup.go | 38 +++++++ go/pkg/basecamp/eventfeed/catchup_test.go | 129 ++++++++++++++++++++++ go/pkg/basecamp/eventfeed/loop.go | 8 ++ 3 files changed, 175 insertions(+) diff --git a/go/pkg/basecamp/eventfeed/catchup.go b/go/pkg/basecamp/eventfeed/catchup.go index 3db2290311..be45f38cc2 100644 --- a/go/pkg/basecamp/eventfeed/catchup.go +++ b/go/pkg/basecamp/eventfeed/catchup.go @@ -184,6 +184,44 @@ func (l *loop) walk(at *attempt, cursor Cursor, presentClass bool) (out cycleOut } continue } + // A page carrying no position is a malformed poll response, and it is + // refused HERE — before any delivery, any counter reset, and any + // mutation of the walk's own state. + // + // Everything downstream treats a position as present. acceptPosition + // assigns it to l.position, and entryCursor selects on + // `l.position != ""`, so an empty one does not "keep the old cursor" — + // it falls through to the StartResume default, which is a BARE PRESENT + // ENTRY. The feed would then resume at the server's head, having + // skipped everything between the real position and now, and report + // nothing: the same silent history skip that a found-but-empty stored + // position is refused for at load (loadCheckpoint), arriving by the + // other door. + // + // The present-class branch fails differently and worse. `held` uses "" + // as its sentinel for "the final entry was not present-class", so an + // empty position does not get saved as empty — it collapses into that + // sentinel, the `held != ""` guard skips acceptPosition AND + // saveCheckpoint outright, and `caught_up` is still announced. The + // entry position is not blanked, it is DISCARDED, with the drain's + // deliveries already handed to the consumer and nothing durable + // recording that they happened. + // + // Refusing it is also what makes saveCheckpoint's stated contract true. + // It documents that "the live cursor is neither regressed nor blanked", + // and the assignment one line above its call site did exactly that. + // + // Terminal(poll_failed) is the classification §23 already gives "an + // unexpected shape" from the poll lane, and it is not retried: a server + // that answered without a position answers the same way to the same + // request, so a retry loop would spin. Nothing durable has moved. + if p.page.Position == "" { + l.disposeAttempt(at, nil) + return cycleOutcome{kind: outcomeTerminal, term: &TerminalError{ + Reason: ReasonPollFailed, + Msg: "the poll response carried no position; an empty position cannot be told apart from having none", + }}, "", true + } // A successful page resets both consecutive-failure counters: the // poll-retry index, and the shared connection-level authorization // counter — the ONLY thing that resets the latter (a confirmation diff --git a/go/pkg/basecamp/eventfeed/catchup_test.go b/go/pkg/basecamp/eventfeed/catchup_test.go index 529971f063..0e85bae967 100644 --- a/go/pkg/basecamp/eventfeed/catchup_test.go +++ b/go/pkg/basecamp/eventfeed/catchup_test.go @@ -1610,3 +1610,132 @@ func TestProtocolFatalBehindTheBlockedHandOffIsImmediate(t *testing.T) { assertProtocolFatalDrain(t, h, store, caughtUp) assertIDs(t, h.deliveredIDs(), 41) } + +// TestPollPageWithNoPositionIsMalformed is daybreak blocker 2. A page without +// a position is refused before delivery, before any counter reset, and before +// any mutation of the walk's state. +// +// Both entry classes are covered because they fail DIFFERENTLY, and the +// present-class one is the worse of the two — an assertion written only +// against the position-resume path would miss it entirely. +func TestPollPageWithNoPositionIsMalformed(t *testing.T) { + // Position-resume entry: acceptPosition("") would set l.position = "", + // and entryCursor selects on `l.position != ""` — so it does not keep the + // old cursor, it falls through to the StartResume default, which is a bare + // present entry. The feed would resume at the server's head with every + // event between the real position and now skipped, and no signal at all. + t.Run("position-resume entry", func(t *testing.T) { + store := feedtest.NewStore() + store.Stored("pos-0") + h := storedHarness(t, store) + h.minter.ScriptTicket(ticket(1)) + h.polls.ScriptPage(eventfeed.PollPage{ + Events: []eventfeed.Event{pollEvent(101)}, + Position: "", + }) + h.start() + + conn := h.driveToSubscribed() + conn.Serve(frameConfirm(noFilterIdentifier)) + h.join() + + _, terminal, _ := h.snapshot() + if terminal == nil || terminal.Reason != eventfeed.ReasonPollFailed { + t.Fatalf("terminal = %v, want reason %q", terminal, eventfeed.ReasonPollFailed) + } + // Nothing durable moved, and the page's events were never delivered: + // the refusal precedes both. + assertPositions(t, store.Saves()) + assertIDs(t, h.deliveredIDs()) + if !conn.Closed() { + t.Error("teardown must close the socket") + } + }) + + // Present-class entry, and the reason this subtest exists. `held` uses "" + // as its sentinel for "the final entry was not present-class", so an empty + // position does not get saved as empty — it collapses into that sentinel, + // `held != ""` skips acceptPosition AND saveCheckpoint outright, and + // caught_up is announced anyway. The entry position is DISCARDED, with the + // drain's deliveries already handed to the consumer. + t.Run("present-class entry", func(t *testing.T) { + store := feedtest.NewStore() + caughtUp := 0 + h := storedHarness(t, store, + eventfeed.WithStart(eventfeed.StartPresent()), + eventfeed.WithObserver(eventfeed.Observer{CaughtUp: func() { caughtUp++ }}), + ) + h.minter.ScriptTicket(ticket(1)) + h.polls.ScriptPage(eventfeed.PollPage{ + Events: []eventfeed.Event{pollEvent(101)}, + Position: "", + }) + h.start() + + conn := h.driveToSubscribed() + conn.Serve(frameConfirm(noFilterIdentifier)) + h.join() + + _, terminal, _ := h.snapshot() + if terminal == nil || terminal.Reason != eventfeed.ReasonPollFailed { + t.Fatalf("terminal = %v, want reason %q", terminal, eventfeed.ReasonPollFailed) + } + assertPositions(t, store.Saves()) + assertIDs(t, h.deliveredIDs()) + if caughtUp != 0 { + t.Errorf("caught_up announcements = %d, want 0 — the walk never completed", caughtUp) + } + }) + + // The refusal is not retried. A server that answered without a position + // answers the same way to the same request, so a poll-retry loop would + // spin: exactly one seam call is made. + t.Run("is not retried", func(t *testing.T) { + store := feedtest.NewStore() + store.Stored("pos-0") + h := storedHarness(t, store) + h.minter.ScriptTicket(ticket(1)) + h.polls.ScriptPage(eventfeed.PollPage{Position: ""}) + h.start() + + conn := h.driveToSubscribed() + conn.Serve(frameConfirm(noFilterIdentifier)) + h.join() + + if n := len(h.polls.Calls()); n != 1 { + t.Errorf("poll seam calls = %d, want 1 — a malformed shape must not be retried", n) + } + }) + + // A page on the SECOND hop of the walk is refused the same way, and the + // first page's save stands: the refusal is per-page, not a rollback. + t.Run("mid-walk page", func(t *testing.T) { + store := feedtest.NewStore() + store.Stored("pos-0") + h := storedHarness(t, store) + h.minter.ScriptTicket(ticket(1)) + next := testOrigin + "/999/events.json?after=101" + h.polls.ScriptPage(eventfeed.PollPage{ + Events: []eventfeed.Event{pollEvent(101)}, + Position: "pos-1", + Next: next, + }) + h.polls.ScriptPage(eventfeed.PollPage{ + Events: []eventfeed.Event{pollEvent(102)}, + Position: "", + }) + h.start() + + conn := h.driveToSubscribed() + conn.Serve(frameConfirm(noFilterIdentifier)) + h.join() + + _, terminal, _ := h.snapshot() + if terminal == nil || terminal.Reason != eventfeed.ReasonPollFailed { + t.Fatalf("terminal = %v, want reason %q", terminal, eventfeed.ReasonPollFailed) + } + // The first page completed normally; only the malformed one is refused. + assertPositions(t, store.Saves(), "pos-1") + assertIDs(t, h.deliveredIDs(), 101) + }) +} diff --git a/go/pkg/basecamp/eventfeed/loop.go b/go/pkg/basecamp/eventfeed/loop.go index e4ffd9beeb..2d482fd8c9 100644 --- a/go/pkg/basecamp/eventfeed/loop.go +++ b/go/pkg/basecamp/eventfeed/loop.go @@ -1138,6 +1138,14 @@ type reentryCursor struct { // it" is exactly this assignment, so a present-class re-entry stays latched // through its whole walk and drain (nothing durable has moved until the held // save) while a position-resume one is released by its first saved page. +// +// position is non-empty at both call sites, and must stay that way: the walk +// refuses a page that carries no position outright, and the held save is +// guarded on `held != ""`. An empty one assigned here does not preserve the +// old cursor — entryCursor selects on `l.position != ""`, so it falls through +// to a bare present entry and silently skips history. (`l.position = ""` IS +// legitimate, but only from reenterAtResetCursor, which means it deliberately: +// the server refused the cursor and there is nothing to resume from.) func (l *loop) acceptPosition(position string) { l.position = position l.reentry = nil From ad1b3c85f899ce87f33350f774e53e0fbe004625 Mon Sep 17 00:00:00 2001 From: Jeremy Daer Date: Tue, 18 Aug 2026 11:22:31 -0700 Subject: [PATCH 03/51] Event feed: staleness arms before Connected, and observers see origins only MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two independent items, both about what leaves the connector and when. measures silence on it. Observer callbacks are host code on the consumer's goroutine (a log write, a metrics emission, an error-tracker breadcrumb), and newStaleHolder reads the window's origin at construction — so whatever the callback spent was time the window never counted, and a peer that went silent immediately got that much extra grace. §23 says the window arms at socket open; now it does. Starting the pump ahead of the callback is the same trade and harmless: it only fills a bounded channel this goroutine drains later. The test observes the ordering from INSIDE the callback, which is the only place it is visible — asserting on the timer set afterwards passes either way, because the set is identical and only its origin moves. Restoring the old order fails it with `timers armed when Connected fired = [handshake-deadline]`. Blocker 7 — the redactor from the foundations PR is applied to both URL-bearing observer surfaces. Observer.Gap carried the server's resume URL whole, and Cursor.PageURL carries the same URL at a walk announcement: an accepted 410 latches it as reconnect state (deliberately, so a socket torn down before the resume page cannot re-select the cursor the server already refused), so the reconnect announces its walk carrying it. Redacting Gap alone would have left the identical URL leaving through a different callback one reconnect later. The handler still receives the URL WHOLE and the resume poll still follows it whole, and both tests assert that pairing — a test checking only the observer would be satisfied by redacting both, which would break the disposition. Reaching the CatchUpStarted leak took the one sequence that keeps the latch alive. Dropping the resume poll's socket does not: transition 21 observes that at the page boundary, AFTER the resume page is accepted and saved, which releases the latch and reconnects at pos-1. Written that way first, it proved nothing; PollUnauthorized is what rides the reconnect cycle with the latch intact. The first version also raced the second walk's announcement, which -race reported. Observer.Disconnected(err) was audited and deliberately NOT redacted. An error is opaque text, and stripping a credential out of arbitrary text means modelling the credential — the one thing §23's "opaque bearer" contract forbids, and the trap dialFailure's own comment documents three review rounds of. Every error the connector puts there is its own sentinel or a seam error, and the built-in transport draws its causes from a closed vocabulary. The audit did find a real gap, and it is a contract gap rather than a code one: DialError.Err is supplied by the CableTransport and concatenated into Error(), and CableTransport is a documented extension point whose seam stated no obligation about the URL. So the obligation is now stated where an implementer reads it — on Dial, ReadFrame and WriteFrame — and Observer.Disconnected's doc says plainly that err is unredacted and why. Stating it on the seam is the only place it can hold; a guard here could not see inside an opaque error. --- go/pkg/basecamp/eventfeed/catchup.go | 7 +- go/pkg/basecamp/eventfeed/loop.go | 18 +++- go/pkg/basecamp/eventfeed/loop_test.go | 40 +++++++++ go/pkg/basecamp/eventfeed/recovery.go | 11 ++- go/pkg/basecamp/eventfeed/recovery_test.go | 96 +++++++++++++++++++++- go/pkg/basecamp/eventfeed/seams.go | 29 ++++++- 6 files changed, 192 insertions(+), 9 deletions(-) diff --git a/go/pkg/basecamp/eventfeed/catchup.go b/go/pkg/basecamp/eventfeed/catchup.go index be45f38cc2..a69d06debc 100644 --- a/go/pkg/basecamp/eventfeed/catchup.go +++ b/go/pkg/basecamp/eventfeed/catchup.go @@ -34,7 +34,12 @@ func (l *loop) runCatchUp(h catchUpHandoff) cycleOutcome { l.hooks.catchUpEntered(h) } if l.cfg.observer.CatchUpStarted != nil { - l.cfg.observer.CatchUpStarted(h.entry) + // Cursor.PageURL is redacted to its origin: on a 410-accepted re-entry + // it IS the server-supplied resume URL, carrying whatever query the + // server put in it. Position and Since are server-issued opaque ids + // rather than URLs and pass through — an operator correlating a trace + // needs them. The walk itself uses the unredacted cursor. + l.cfg.observer.CatchUpStarted(redactCursor(l.cfg.origin, h.entry)) } // The consecutive-poll-failure index is per walk: a socket teardown ends // the previous walk, and from there the reconnect cycle's own counter diff --git a/go/pkg/basecamp/eventfeed/loop.go b/go/pkg/basecamp/eventfeed/loop.go index 2d482fd8c9..2eff4e8b57 100644 --- a/go/pkg/basecamp/eventfeed/loop.go +++ b/go/pkg/basecamp/eventfeed/loop.go @@ -772,10 +772,26 @@ func (l *loop) runCycle(delay time.Duration) cycleOutcome { // Transition 6 → AwaitingWelcome: staleness arms at socket open, the // pump starts, and the handshake deadline keeps running to `welcome`. + // + // The arming comes FIRST, before Observer.Connected, and the order is + // load-bearing rather than stylistic (#763). Observer callbacks are host + // code running on the consumer's goroutine — a log write, a metrics + // emission, an error-tracker breadcrumb — and this one sat between the + // socket opening and the window that measures silence on it. Whatever it + // spent became time the window did not count: newStaleHolder reads its + // origin at construction, so the window began when the callback returned, + // not when the socket opened, and a peer that went silent immediately was + // given that much extra grace before staleness could fire. §23 says the + // window arms at socket open, and this line is what that means. + // + // Starting the pump ahead of the callback is the same trade and harmless: + // the pump only fills a bounded channel that this goroutine drains later, + // so nothing is observed out of order — Connected is observability, not a + // gate. + at.lc = newLiveConn(at.ctx, conn, l.cfg.clock, l.cfg.staleAfter, l.hooks) if l.cfg.observer.Connected != nil { l.cfg.observer.Connected() } - at.lc = newLiveConn(at.ctx, conn, l.cfg.clock, l.cfg.staleAfter, l.hooks) return l.awaitConfirmation(at, hs) } diff --git a/go/pkg/basecamp/eventfeed/loop_test.go b/go/pkg/basecamp/eventfeed/loop_test.go index 31e7d654e4..385a00e52f 100644 --- a/go/pkg/basecamp/eventfeed/loop_test.go +++ b/go/pkg/basecamp/eventfeed/loop_test.go @@ -13,6 +13,7 @@ import ( "errors" "fmt" "runtime" + "slices" "strings" "sync" "testing" @@ -1757,3 +1758,42 @@ func TestConfirmRacingAnExpiredConfirmationDeadlineLapses(t *testing.T) { t.Fatalf("poll seam calls = %d, want 0 — the confirmation lapsed", got) } } + +// TestStalenessArmsBeforeConnectedObserver is #763. The staleness window must +// begin when the socket opens, not when a host's Connected callback returns. +// +// Observed from INSIDE the callback, which is the only place the ordering is +// visible: newStaleHolder reads the window's origin at construction, so if the +// callback runs first, everything it spends is time the window never counts, +// and a peer that went silent immediately gets that much extra grace before +// staleness can fire. Asserting on the timer set afterwards would pass either +// way — the set is the same, only its origin moves. +func TestStalenessArmsBeforeConnectedObserver(t *testing.T) { + var armedAtConnected []string + var connectedCalls int + // Declared ahead of the option so the callback can read the harness's own + // clock: the closure captures the variable, and by the time it fires the + // harness is assigned. + var h *harness + h = newHarness(t, eventfeed.WithObserver(eventfeed.Observer{ + Connected: func() { + connectedCalls++ + armedAtConnected = h.clock.Outstanding() + }, + })) + h.minter.ScriptTicket(ticket(1)) + h.polls.ScriptPage(eventfeed.PollPage{Position: "pos-1"}) + h.start() + + conn := h.driveToSubscribed() + conn.Serve(frameConfirm(noFilterIdentifier)) + h.awaitStreaming() + + if connectedCalls != 1 { + t.Fatalf("Connected fired %d time(s), want 1", connectedCalls) + } + if !slices.Contains(armedAtConnected, timerStaleness) { + t.Errorf("timers armed when Connected fired = %v, want the %q window already among them", + armedAtConnected, timerStaleness) + } +} diff --git a/go/pkg/basecamp/eventfeed/recovery.go b/go/pkg/basecamp/eventfeed/recovery.go index b5ecc508f3..fd37c3c3ae 100644 --- a/go/pkg/basecamp/eventfeed/recovery.go +++ b/go/pkg/basecamp/eventfeed/recovery.go @@ -45,7 +45,7 @@ func (l *loop) repairWalk(at *attempt) (cycleOutcome, bool) { cursor, presentClass := l.repairCursor() l.setState(stateCatchingUp) if l.cfg.observer.CatchUpStarted != nil { - l.cfg.observer.CatchUpStarted(cursor) + l.cfg.observer.CatchUpStarted(redactCursor(l.cfg.origin, cursor)) } return l.walkThenDrain(at, cursor, presentClass) } @@ -102,7 +102,14 @@ func (l *loop) recoverGone(at *attempt, pe *PollError) (walkStep, cycleOutcome, l.hooks.signalRaised(signal) } if l.cfg.observer.Gap != nil { - l.cfg.observer.Gap(pe.EpochAfterID, pe.ResumeURL) + // Redacted to its origin. The resume URL is server-supplied and + // followed with the caller's Authorization bearer, and this is the one + // place it leaves the connector for a destination it knows nothing + // about. The SignalHandler below still receives it WHOLE — the + // disposition is about which URL to follow, so redacting there would + // break the contract rather than protect it — and the resume poll + // follows the full URL after §8 validation. + l.cfg.observer.Gap(pe.EpochAfterID, redactURL(l.cfg.origin, pe.ResumeURL)) } if l.cfg.handler != nil && l.cfg.handler(signal) == Accept { if pe.ResumeURL == "" { diff --git a/go/pkg/basecamp/eventfeed/recovery_test.go b/go/pkg/basecamp/eventfeed/recovery_test.go index 77b9127598..cfe6801ddf 100644 --- a/go/pkg/basecamp/eventfeed/recovery_test.go +++ b/go/pkg/basecamp/eventfeed/recovery_test.go @@ -124,10 +124,29 @@ func TestFeedGapAcceptedResumesViaProvidedURL(t *testing.T) { conn.Serve(frameConfirm(noFilterIdentifier)) h.awaitStreaming() - if len(gaps) != 1 || gaps[0] != fmt.Sprintf("gap 100 %s", resume) { - t.Fatalf("Observer.Gap = %v, want exactly one firing carrying the epoch and resume URL", gaps) + // Observer.Gap carries the resume URL REDACTED to its origin. It fires + // unconditionally — observability is not disposition — so it is the one + // place a server-supplied, bearer-followed URL leaves the connector for a + // destination it knows nothing about: a log, an error tracker, a metrics + // label. + if len(gaps) != 1 || gaps[0] != fmt.Sprintf("gap 100 %s", testOrigin) { + t.Fatalf("Observer.Gap = %v, want exactly one firing carrying the epoch and the resume URL's ORIGIN", gaps) + } + if strings.Contains(gaps[0], "?") || strings.Contains(gaps[0], "since=now") { + t.Errorf("Observer.Gap = %q, which retains the resume URL's query", gaps[0]) } assertInvocations(t, ledger.invocations(), "feedGap/accept") + // The handler receives it WHOLE, and that pairing is the contract: the + // disposition is a decision about which URL to follow, so redacting here + // would break it rather than protect anything. A test that only checked + // the observer would be satisfied by redacting both. + gap, ok := ledger.first().(eventfeed.FeedGap) + if !ok { + t.Fatalf("first signal = %T, want eventfeed.FeedGap", ledger.first()) + } + if gap.ResumeURL != resume { + t.Errorf("FeedGap.ResumeURL = %q, want the full URL %q", gap.ResumeURL, resume) + } calls := h.polls.Calls() if len(calls) != 2 { t.Fatalf("poll seam calls = %d, want 2", len(calls)) @@ -758,3 +777,76 @@ func TestAcceptedGapResumeSurvivesAReconnect(t *testing.T) { assertInvocations(t, ledger.invocations(), "feedGap/accept") assertPositions(t, store.Saves(), "pos-1") } + +// TestCatchUpStartedRedactsTheLatchedResumeURL is the second URL-bearing +// observer surface, and reaching it takes the one sequence that puts a +// server-supplied URL into Cursor.PageURL at a walk announcement. +// +// Cursor.PageURL is normally connector-generated. But an ACCEPTED 410 latches +// the server's resume URL as reconnect state, deliberately — so that a socket +// torn down before the resume poll serves a page cannot re-select the cursor +// the server already refused. If that teardown happens, the reconnect +// announces its walk carrying the latched URL. Redacting Observer.Gap alone +// would have left the identical URL leaving through a different callback one +// reconnect later. +// +// The resume poll fails PollUnauthorized rather than having its socket dropped, +// and the difference is the point: a dropped socket is observed at the page +// boundary AFTER the resume page is accepted and saved (transition 21), which +// releases the latch — so that version of this test reconnects at pos-1 and +// proves nothing. Verified by writing it that way first. +func TestCatchUpStartedRedactsTheLatchedResumeURL(t *testing.T) { + store := feedtest.NewStore() + store.Stored("pos-0") + ledger := &signalLedger{give: eventfeed.Accept} + var cursors []eventfeed.Cursor + h := storedHarness(t, store, + eventfeed.WithSignalHandler(ledger.handler), + eventfeed.WithObserver(eventfeed.Observer{ + CatchUpStarted: func(c eventfeed.Cursor) { cursors = append(cursors, c) }, + })) + h.minter.ScriptTicket(ticket(1)) + h.minter.ScriptTicket(ticket(2)) + const secret = "resume-token-do-not-log" + resume := testOrigin + "/999/events.json?since=now&t=" + secret + h.polls.ScriptError(gonePoll(resume)) + h.polls.ScriptError(&eventfeed.PollError{Kind: eventfeed.PollUnauthorized}) + h.polls.ScriptPage(eventfeed.PollPage{Position: "pos-1"}) + h.start() + + conn := h.driveToSubscribed() + conn.Serve(frameConfirm(noFilterIdentifier)) + h.awaitTimer(timerBackoff) + + // The reconnect's entry cursor IS the latched resume URL, so this walk's + // announcement is the one that would carry it. + h.fireTimer(timerBackoff) + conn2 := h.driveToSubscribed() + conn2.Serve(frameConfirm(noFilterIdentifier)) + h.awaitStreaming() + + if len(cursors) != 2 { + t.Fatalf("CatchUpStarted fired %d time(s) (%+v), want the entry walk and the reconnect walk", len(cursors), cursors) + } + for i, c := range cursors { + if strings.Contains(c.PageURL, secret) { + t.Errorf("CatchUpStarted[%d].PageURL = %q, which leaks the resume URL's query", i, c.PageURL) + } + if strings.ContainsAny(c.PageURL, "?#") { + t.Errorf("CatchUpStarted[%d].PageURL = %q, which retains a query or fragment", i, c.PageURL) + } + } + // The reconnect announcement still names the origin — what an operator + // needs to know which host is being polled — and the poll that followed it + // still used the FULL URL. Redacting both would satisfy the loop above. + if got := cursors[1].PageURL; got != testOrigin { + t.Errorf("reconnect CatchUpStarted.PageURL = %q, want the origin %q", got, testOrigin) + } + calls := h.polls.Calls() + if len(calls) != 3 { + t.Fatalf("poll seam calls = %d (%+v), want 3", len(calls), calls) + } + if calls[2].Cursor.PageURL != resume { + t.Errorf("the resume poll followed %q, want the full URL %q", calls[2].Cursor.PageURL, resume) + } +} diff --git a/go/pkg/basecamp/eventfeed/seams.go b/go/pkg/basecamp/eventfeed/seams.go index a9c7b6a084..d5f55c9700 100644 --- a/go/pkg/basecamp/eventfeed/seams.go +++ b/go/pkg/basecamp/eventfeed/seams.go @@ -268,6 +268,15 @@ type CableTransport interface { // rejecting an over-limit message without materializing it; a // non-positive value is refused as a usage error before any I/O — there // is no unlimited mode. + // + // The returned error MUST NOT render wsURL or any part of its query + // string. The ticket rides in that query, and this error reaches + // Observer.Disconnected, which hosts log. An error is opaque text, so + // nothing downstream can redact it — see WebSocketTransport's dialFailure + // on why stripping a credential out of arbitrary text requires modelling + // the credential, which §23's "opaque bearer" contract forbids. Report + // the classification and a cause of your own choosing, never the peer's + // or a library's rendering of the URL. Dial(ctx context.Context, wsURL string, maxFrameBytes int64) (CableConn, error) } @@ -278,11 +287,14 @@ type CableConn interface { // of the seam: stock Action Cable discards the disconnect reason, and the // terminal/non-terminal distinction lives only in this raw frame. A peer // close surfaces as *CloseError; a message over the dial's maxFrameBytes - // surfaces as an error matching ErrFrameOversize. + // surfaces as an error matching ErrFrameOversize. Like Dial's, the + // returned error must not render the cable URL or its query string: it + // reaches Observer.Disconnected. ReadFrame(ctx context.Context) ([]byte, error) // WriteFrame sends one raw text frame. Close and ctx cancellation must // unblock an in-progress write; a write failure takes the current - // state's socket-failure path. + // state's socket-failure path, and must not render the cable URL or its + // query string: it reaches Observer.Disconnected. WriteFrame(ctx context.Context, data []byte) error // Close is idempotent, safe from any goroutine, and unblocks ReadFrame // and WriteFrame. @@ -460,7 +472,18 @@ type Observer struct { Connected func() // Confirmed fires on confirm_subscription. Confirmed func() - // Disconnected fires when a socket is torn down. + // Disconnected fires when a socket is torn down. reason is the + // frame-derived disconnect reason, bounded by the §9 cap; err is the + // failure that ended the socket. + // + // err is passed through UNREDACTED, deliberately: it is an error, i.e. + // opaque text, and stripping a credential out of arbitrary text requires + // modelling the credential — the one thing §23's "opaque bearer" contract + // rules out. Every error the connector puts here is either one of its own + // sentinels or a seam error, and Dial, ReadFrame and WriteFrame each carry + // the obligation not to render the cable URL. A custom transport that + // breaks that obligation leaks through this callback, which is why the + // obligation is stated on the seam rather than defended here. Disconnected func(reason string, err error) // CatchUpStarted fires when a poll walk begins. CatchUpStarted func(cursor Cursor) From 37e5598276eb93c17a9776a3e6ba519ac49a21ec Mon Sep 17 00:00:00 2001 From: Jeremy Daer Date: Tue, 18 Aug 2026 11:32:55 -0700 Subject: [PATCH 04/51] Event feed: order the one durable effect against Close, and narrow the promise MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Daybreak blocker 6. Close's doc claimed "no seam call (mint, dial, poll) and no delivery can BEGIN after Close has returned". That is not true and cannot be made true: the run goroutine checks its context at each dispatch point and then acts, so a Close landing between a check and the call it guards cannot stop the call from starting — only from starting on a live context. Closing that window would mean holding a lock across every seam call, i.e. across arbitrary host code, trading a benign race for a deadlock reachable from any callback. So the promise is narrowed to what holds: cancellation is visible before Close returns, and anything that begins afterwards begins on an already-cancelled context. For reads that is self-limiting — the call returns promptly, its result is discarded, the run takes the Closed edge. One effect is not self-limiting, and it is the one now ORDERED. A checkpoint save is the connector's only effect that outlives the process, and cancellation cannot stop it: a CheckpointStore is entitled to ignore ctx, and the built-in FileCheckpointStore documents that it does, deliberately, so a position already accepted is not dropped by a shutdown race. The cancelled context Close publishes is precisely the signal that store is specified not to act on. durableGate closes it. A save holds the gate across the store call; Close latches the gate, waiting for at most one in-flight write. Either a save commenced before Close (and Close waits for it, which is right — its position was accepted and delivered before Close, and abandoning a durable write half-done is worse) or it does not commence at all. The failure that motivates it is sequential, not adversarial: Close, then a second connector over the same store, then a save the first had not yet begun landing on a lineage the new run has already loaded. It is a second lock, not mu, so a host's write cannot hold up a concurrent Close caller's cheap serialization; and it is released BEFORE either observer callback, because Close from inside a callback is documented as supported and holding the gate across host code would deadlock Observer.Checkpoint against the very save that fired it. That hazard is real, not theoretical: holding the gate across the callbacks deadlocks the Checkpoint subtest, and Go's timeout names it. Third item: a checkpoint LOAD that failed because the run was cancelled is no longer Terminal(checkpoint_load). The load happens on the first iteration and before the first mint, which is exactly the window a prompt Close lands in, and diagnosing the consumer's store for the consumer's own shutdown is wrong — §23 ends a closed iterator with no error element. Classified on the CONTEXT, not the error's shape, because a store is under no obligation to wrap ctx.Err(). The first version of that test closed BEFORE the run started, which never reaches the store at all — Events takes the isClosed latch with zero wire attempts — so it passed with the classification deleted. It now parks the load in flight and lands Close on it, and the paired test keeps an UNCANCELLED load failure terminal, so the guard cannot be widened into swallowing the real edge. --- go/pkg/basecamp/eventfeed/connector.go | 118 ++++++- go/pkg/basecamp/eventfeed/connector_test.go | 322 ++++++++++++++++++++ go/pkg/basecamp/eventfeed/loop.go | 36 ++- 3 files changed, 459 insertions(+), 17 deletions(-) diff --git a/go/pkg/basecamp/eventfeed/connector.go b/go/pkg/basecamp/eventfeed/connector.go index ee0e66ac5e..b7033511dd 100644 --- a/go/pkg/basecamp/eventfeed/connector.go +++ b/go/pkg/basecamp/eventfeed/connector.go @@ -96,15 +96,18 @@ type config struct { // input, two consumers: it is CheckpointKey.Origin, and it is the // same-origin reference every continuation and resume URL is validated // against before an authenticated follow. - origin string - accountID string - minter TicketMinter - polls PollSource - filters Filters - start Start - transport CableTransport - clock Clock - store CheckpointStore + origin string + accountID string + minter TicketMinter + polls PollSource + filters Filters + start Start + transport CableTransport + clock Clock + store CheckpointStore + // durable serializes the commencement of a checkpoint save against Close. + // Always non-nil after New; see durableGate. + durable *durableGate consumerNamespace string confirmationDeadline time.Duration repairInterval time.Duration @@ -200,6 +203,60 @@ type Connector struct { cancelRun context.CancelFunc } +// durableGate serializes the COMMENCEMENT of a checkpoint save against Close. +// +// A save is the connector's only effect that outlives the process, and the +// failure this prevents is sequential rather than adversarial: a consumer calls +// Close, opens a second connector over the same checkpoint file, and a save the +// first connector had not yet commenced lands afterwards — on top of a lineage +// the new run has already loaded and may already have advanced. +// +// Cancellation alone cannot prevent it. A CheckpointStore is entitled to ignore +// ctx, and the built-in FileCheckpointStore documents that it does, deliberately: +// a position the connector already accepted must not be dropped because the +// run's context was cancelled between acceptance and write-through. So the +// cancelled context Close publishes is exactly the signal this store is +// specified not to act on. +// +// It is a lock of its own, not the Connector's mu, because a save must not hold +// mu across host store I/O — that would block every concurrent Close caller for +// as long as the host's write takes, and mu's whole job is that Close's callers +// are serialized cheaply. +// +// A save that has ALREADY commenced runs to completion and Close waits for it. +// That is the intended trade: its position was accepted and delivered before +// Close was called, and abandoning a durable write half-way is worse than +// finishing it. The wait is bounded by one store write. +type durableGate struct { + mu sync.Mutex + closed bool +} + +// begin claims the right to start one durable operation, reporting whether it +// was granted. The caller MUST call end when it returns. The gate is held for +// the whole operation, which is what makes the ordering against Close exact +// rather than a check-then-act. +func (g *durableGate) begin() bool { + g.mu.Lock() + if g.closed { + g.mu.Unlock() + return false + } + return true +} + +// end releases the gate after a granted begin. +func (g *durableGate) end() { g.mu.Unlock() } + +// close latches the gate, waiting for any operation already in flight. Called +// by Close, on the caller's goroutine, so that no save can COMMENCE after +// Close returns. +func (g *durableGate) close() { + g.mu.Lock() + g.closed = true + g.mu.Unlock() +} + // testHooks are unexported in-package observation points for the tier-2/3 // harnesses (rendezvous without wall-clock sleeps). Production code never // sets them. @@ -287,6 +344,7 @@ func New(origin, accountID string, minter TicketMinter, polls PollSource, opts . if cfg.clock == nil { cfg.clock = SystemClock() } + cfg.durable = &durableGate{} if err := validateConfig(&cfg); err != nil { return nil, err } @@ -443,12 +501,30 @@ func (c *Connector) Events(ctx context.Context) iter.Seq2[Event, error] { // itself. The iterator ends with no error element. // // Cancellation is SYNCHRONOUS with the return: an active run's context is -// cancelled here, on the caller's goroutine, so no seam call (mint, dial, -// poll) and no delivery can BEGIN after Close has returned. §23 makes -// close() a universal edge from every non-absorbing state, and delegating the -// cancel to an independently scheduled goroutine would leave the edge merely -// eventual — a Close taken inside Observer.Connecting would return, and the -// mint one statement later would still go out. +// cancelled here, on the caller's goroutine, so the cancellation is visible +// before Close returns, and any seam call (mint, dial, poll) or delivery that +// begins afterwards begins on an ALREADY-CANCELLED context. §23 makes close() +// a universal edge from every non-absorbing state, and delegating the cancel to +// an independently scheduled goroutine would leave the edge merely eventual — a +// Close taken inside Observer.Connecting would return, and the mint one +// statement later would still go out. +// +// What that does NOT promise, and used to: that no seam call can BEGIN after +// Close returns. Close is not the only goroutine involved. The run goroutine +// checks its context at each dispatch point and then acts, so a Close landing +// between one of those checks and the call it guards cannot stop the call from +// starting — only from starting on a live context. Closing that window would +// mean holding a lock across every seam call, i.e. across arbitrary host code, +// which trades a benign race for a deadlock reachable from any callback. +// +// The distinction matters for exactly one effect, and that one is ordered: +// a CHECKPOINT SAVE cannot commence after Close returns (durableGate). It is +// the only thing the connector does that outlives the process, and the only one +// where "began a moment late" is not self-limiting — a store may ignore the +// cancelled context, and the built-in one documents that it does. Everything +// else that can start in the window is a read whose context is already dead: +// it returns promptly, its result is discarded, and the run takes the Closed +// edge. // // What Close deliberately does NOT do is wait for the run goroutine to exit. // Close is callable from inside a consumer callback — an observer, a signal @@ -468,11 +544,21 @@ func (c *Connector) Events(ctx context.Context) iter.Seq2[Event, error] { // triggers runs on its own goroutine, so it cannot re-enter Close. func (c *Connector) Close() error { c.mu.Lock() - defer c.mu.Unlock() c.isClosed = true if c.cancelRun != nil { c.cancelRun() c.cancelRun = nil } + c.mu.Unlock() + // After the cancel, and outside mu: latch the durable gate, waiting for at + // most one checkpoint save already in flight. This is the one effect that + // outlives the process, so it is the one whose COMMENCEMENT is ordered + // against Close rather than merely cancelled — a CheckpointStore may + // ignore ctx, and the built-in one documents that it does. + // + // Outside mu so a host store's write cannot hold up a concurrent Close + // caller's cheap serialization, and after the cancel so the cancellation is + // already visible while this waits. + c.cfg.durable.close() return nil } diff --git a/go/pkg/basecamp/eventfeed/connector_test.go b/go/pkg/basecamp/eventfeed/connector_test.go index 49677c8d08..3ec585b4b2 100644 --- a/go/pkg/basecamp/eventfeed/connector_test.go +++ b/go/pkg/basecamp/eventfeed/connector_test.go @@ -620,3 +620,325 @@ func TestFoundButEmptyPositionIsALoadFailure(t *testing.T) { t.Errorf("dials = %d, want 0", got) } } + +// blockingStore is a CheckpointStore whose Save parks until released, so a +// test can hold one durable write in flight across a concurrent Close. Load is +// scripted separately. +type blockingStore struct { + entered chan struct{} + release chan struct{} + loadErr error + loadCtx chan context.Context + loadEntered chan struct{} + loadRelease chan struct{} + mu sync.Mutex + saves []string + attempts int +} + +func newBlockingStore() *blockingStore { + return &blockingStore{ + entered: make(chan struct{}, 8), + release: make(chan struct{}), + loadCtx: make(chan context.Context, 4), + } +} + +func (s *blockingStore) Load(ctx context.Context, _ eventfeed.CheckpointKey) (string, bool, error) { + select { + case s.loadCtx <- ctx: + default: + } + // Parkable, so a test can hold the load in flight and land a Close on it. + // That window is the only one the cancelled-load classification governs: + // closing BEFORE the run starts never reaches this function at all, since + // Events takes the isClosed latch with zero wire attempts. + if s.loadEntered != nil { + s.loadEntered <- struct{}{} + <-s.loadRelease + } + if s.loadErr != nil { + return "", false, s.loadErr + } + return "pos-0", true, nil +} + +func (s *blockingStore) Save(_ context.Context, _ eventfeed.CheckpointKey, position string) error { + s.mu.Lock() + s.attempts++ + s.mu.Unlock() + s.entered <- struct{}{} + <-s.release + s.mu.Lock() + s.saves = append(s.saves, position) + s.mu.Unlock() + return nil +} + +func (s *blockingStore) recorded() []string { + s.mu.Lock() + defer s.mu.Unlock() + return append([]string(nil), s.saves...) +} + +func (s *blockingStore) attemptCount() int { + s.mu.Lock() + defer s.mu.Unlock() + return s.attempts +} + +// TestCloseWaitsForAnInFlightCheckpointSave is the ordered half of blocker 6. +// A checkpoint save is the connector's only effect that outlives the process, +// so its COMMENCEMENT is ordered against Close: one already in flight runs to +// completion and Close waits for it, rather than Close returning while a +// durable write is still pending against a lineage a second connector may +// already have loaded. +// +// Cancellation cannot substitute. A CheckpointStore is entitled to ignore ctx — +// the built-in FileCheckpointStore documents that it does, so a position +// already accepted is not dropped by a shutdown race — so the cancelled context +// Close publishes is precisely the signal such a store is specified to ignore. +func TestCloseWaitsForAnInFlightCheckpointSave(t *testing.T) { + store := newBlockingStore() + h := newHarness(t, + eventfeed.WithCheckpointStore(store), + eventfeed.WithConsumerNamespace("agent")) + h.minter.ScriptTicket(ticket(1)) + h.polls.ScriptPage(eventfeed.PollPage{Events: []eventfeed.Event{pollEvent(101)}, Position: "pos-1"}) + h.start() + + sock := h.driveToSubscribed() + sock.Serve(frameConfirm(noFilterIdentifier)) + + // The save is now parked inside the store, on the run goroutine. + select { + case <-store.entered: + case <-time.After(watchdog): + t.Fatal("the checkpoint save never reached the store") + } + + closeReturned := make(chan struct{}) + go func() { _ = h.conn.Close(); close(closeReturned) }() + + // Close must NOT return while the write is pending. + select { + case <-closeReturned: + t.Fatal("Close returned while a checkpoint save was still in flight") + case <-time.After(150 * time.Millisecond): + } + + close(store.release) + select { + case <-closeReturned: + case <-time.After(watchdog): + t.Fatal("Close did not return after the save completed") + } + if got := store.recorded(); len(got) != 1 || got[0] != "pos-1" { + t.Errorf("recorded saves = %v, want [pos-1] — an in-flight write completes", got) + } +} + +// TestNoCheckpointSaveCommencesAfterClose is the other direction: once Close +// has returned, a save that had not begun does not begin. Without the gate the +// run goroutine's next accepted page would write through on a cancelled +// context, which a store is allowed to ignore. +func TestNoCheckpointSaveCommencesAfterClose(t *testing.T) { + store := newBlockingStore() + close(store.release) // saves complete immediately + h := newHarness(t, + eventfeed.WithCheckpointStore(store), + eventfeed.WithConsumerNamespace("agent")) + h.minter.ScriptTicket(ticket(1)) + h.start() + + // Close before any page is accepted, so no save has commenced. + if err := h.conn.Close(); err != nil { + t.Fatalf("Close() = %v", err) + } + h.join() + + if n := store.attemptCount(); n != 0 { + t.Errorf("store.Save was entered %d time(s) after Close returned, want 0", n) + } +} + +// TestCancelledCheckpointLoadIsNotAStoreFailure is the third item. The load +// happens on the first iteration and before the first mint, so it is exactly +// the window a prompt Close lands in. Reporting Terminal(checkpoint_load) for +// it would diagnose the consumer's store for the consumer's own shutdown — and +// §23 says Close ends the iterator with NO error element. +// +// The load is held IN FLIGHT and Close lands on it. Closing before the run +// starts proves nothing: Events takes the isClosed latch with zero wire +// attempts and never calls the store at all. Verified — written that way +// first, it passed with the classification deleted. +func TestCancelledCheckpointLoadIsNotAStoreFailure(t *testing.T) { + store := newBlockingStore() + store.loadEntered = make(chan struct{}, 1) + store.loadRelease = make(chan struct{}) + // The store reports its OWN error type, not a wrapped ctx.Err(), which is + // what a store is entitled to do and why the classification reads the + // context rather than the error's shape. + store.loadErr = errors.New("store: request aborted") + h := newHarness(t, + eventfeed.WithCheckpointStore(store), + eventfeed.WithConsumerNamespace("agent")) + h.minter.ScriptTicket(ticket(1)) + h.start() + + select { + case <-store.loadEntered: + case <-time.After(watchdog): + t.Fatal("the checkpoint load never reached the store") + } + if err := h.conn.Close(); err != nil { + t.Fatalf("Close() = %v", err) + } + close(store.loadRelease) + h.join() + + _, terminal, elements := h.snapshot() + if terminal != nil { + t.Errorf("terminal = %v, want none — a cancelled load is a shutdown, not a store failure", terminal) + } + if elements != 0 { + t.Errorf("iteration elements = %d, want 0", elements) + } + if got := h.minter.Calls(); got != 0 { + t.Errorf("mint seam calls = %d, want 0", got) + } +} + +// An UNCANCELLED load failure is still Terminal(checkpoint_load): the guard +// above must not swallow the real edge. Without this the previous test would be +// satisfied by deleting the classification entirely. +func TestUncancelledCheckpointLoadFailureIsStillTerminal(t *testing.T) { + store := feedtest.NewStore() + store.FailLoad(errors.New("disk on fire")) + h := newHarness(t, + eventfeed.WithCheckpointStore(store), + eventfeed.WithConsumerNamespace("agent")) + h.minter.ScriptTicket(ticket(1)) + h.start() + h.join() + + _, terminal, _ := h.snapshot() + if terminal == nil || terminal.Reason != eventfeed.ReasonCheckpointLoad { + t.Fatalf("terminal = %v, want reason %q", terminal, eventfeed.ReasonCheckpointLoad) + } + if got := h.minter.Calls(); got != 0 { + t.Errorf("mint seam calls = %d, want 0 (zero wire attempts)", got) + } +} + +// TestCloseFromEveryCallbackSiteDoesNotDeadlock covers the hazard the durable +// gate introduces. saveCheckpoint holds a lock across host store I/O, and Close +// waits for that lock — so if the gate were still held while a consumer callback +// ran, a Close taken from inside Observer.Checkpoint would deadlock against the +// very save that fired it. Close from inside a callback is documented as +// supported, so this is the case that has to be right, not merely likely. +// +// The release therefore happens before either observer callback. Each site below +// ends the iteration cleanly with no error element; a deadlock fails as the +// watchdog rather than hanging the package. +func TestCloseFromEveryCallbackSiteDoesNotDeadlock(t *testing.T) { + sites := []struct { + name string + observer func(closeFeed func()) eventfeed.Observer + failSave bool + }{ + {"Checkpoint", func(cl func()) eventfeed.Observer { + return eventfeed.Observer{Checkpoint: func(string) { cl() }} + }, false}, + {"CheckpointSaveFailed", func(cl func()) eventfeed.Observer { + return eventfeed.Observer{CheckpointSaveFailed: func(error) { cl() }} + }, true}, + {"PageDelivered", func(cl func()) eventfeed.Observer { + return eventfeed.Observer{PageDelivered: func(int, string) { cl() }} + }, false}, + {"CaughtUp", func(cl func()) eventfeed.Observer { + return eventfeed.Observer{CaughtUp: func() { cl() }} + }, false}, + {"CatchUpStarted", func(cl func()) eventfeed.Observer { + return eventfeed.Observer{CatchUpStarted: func(eventfeed.Cursor) { cl() }} + }, false}, + {"Confirmed", func(cl func()) eventfeed.Observer { + return eventfeed.Observer{Confirmed: func() { cl() }} + }, false}, + } + for _, site := range sites { + t.Run(site.name, func(t *testing.T) { + store := feedtest.NewStore() + store.Stored("pos-0") + if site.failSave { + store.FailNextSave(errors.New("store unavailable")) + } + var h *harness + h = newHarness(t, + eventfeed.WithCheckpointStore(store), + eventfeed.WithConsumerNamespace("agent"), + eventfeed.WithObserver(site.observer(func() { + if err := h.conn.Close(); err != nil { + t.Errorf("Close from %s: %v", site.name, err) + } + }))) + h.minter.ScriptTicket(ticket(1)) + h.polls.ScriptPage(eventfeed.PollPage{ + Events: []eventfeed.Event{pollEvent(101)}, + Position: "pos-1", + }) + h.start() + sock := h.driveToSubscribed() + sock.Serve(frameConfirm(noFilterIdentifier)) + // The watchdog inside join is what reports a deadlock as itself. + h.join() + + if _, terminal, _ := h.snapshot(); terminal != nil { + t.Errorf("terminal = %v, want none — Close ends the iterator with no error element", terminal) + } + }) + } +} + +// TestConcurrentCloseIsSerializedAndIdempotent drives Close from many +// goroutines at once, which is what the mutex across the cancel is for: a +// second Close must not observe a cleared cancelRun and return while the run +// context is still live. Under -race this also covers the durable gate being +// latched by whichever caller gets there. +func TestConcurrentCloseIsSerializedAndIdempotent(t *testing.T) { + store := feedtest.NewStore() + store.Stored("pos-0") + h := newHarness(t, + eventfeed.WithCheckpointStore(store), + eventfeed.WithConsumerNamespace("agent")) + h.minter.ScriptTicket(ticket(1)) + h.polls.ScriptPage(eventfeed.PollPage{Position: "pos-1"}) + h.start() + sock := h.driveToSubscribed() + sock.Serve(frameConfirm(noFilterIdentifier)) + + const closers = 16 + var wg sync.WaitGroup + start := make(chan struct{}) + errs := make(chan error, closers) + for range closers { + wg.Add(1) + go func() { + defer wg.Done() + <-start + if err := h.conn.Close(); err != nil { + errs <- err + } + }() + } + close(start) + wg.Wait() + close(errs) + for err := range errs { + t.Errorf("Close returned %v, want nil from every caller", err) + } + h.join() + if _, terminal, _ := h.snapshot(); terminal != nil { + t.Errorf("terminal = %v, want none", terminal) + } +} diff --git a/go/pkg/basecamp/eventfeed/loop.go b/go/pkg/basecamp/eventfeed/loop.go index 2eff4e8b57..a2497b7336 100644 --- a/go/pkg/basecamp/eventfeed/loop.go +++ b/go/pkg/basecamp/eventfeed/loop.go @@ -1273,6 +1273,26 @@ func (l *loop) loadCheckpoint() *TerminalError { } position, ok, err := l.cfg.store.Load(l.runCtx, l.checkpointKey()) if err != nil { + // A load that failed because the RUN was cancelled is not a store + // failure, and must not be reported as one. Close is a universal edge + // from every non-absorbing state and ends the iterator with no error + // element; surfacing Terminal(checkpoint_load) here would turn an + // ordinary shutdown — a Close racing the first iteration, which is the + // one iteration this load happens on — into a diagnosis of the + // consumer's own store. The run's next check takes the Closed edge. + // + // Checked on the CONTEXT rather than on the error's shape: a store is + // under no obligation to wrap ctx.Err(), and one that returns its own + // error type would otherwise be misclassified. + if l.runCtx.Err() != nil { + //nolint:nilerr // Deliberate: the error is real but it is not a + // STORE failure, it is this run being closed underneath a load that + // was already in flight. Returning it would give the consumer + // Terminal(checkpoint_load) for its own shutdown; §23 ends a closed + // iterator with no error element, and the run's next check takes + // the Closed edge. + return nil + } return &TerminalError{ Reason: ReasonCheckpointLoad, Msg: "the checkpoint store failed to load the stored position", @@ -1318,7 +1338,21 @@ func (l *loop) saveCheckpoint(position string) { if l.cfg.store == nil { return } - if err := l.cfg.store.Save(l.runCtx, l.checkpointKey(), position); err != nil { + // The store call runs under the durable gate, so a save either commenced + // before Close or does not commence at all (durableGate). A refused save is + // silent by design: Close abandons, and reporting a save that was declined + // BECAUSE the consumer closed would be reporting the consumer's own + // decision back to it as a store failure. + if !l.cfg.durable.begin() { + return + } + err := l.cfg.store.Save(l.runCtx, l.checkpointKey(), position) + // Released before either observer callback fires. Holding it across host + // code would deadlock the documented case of Close being called from + // inside a callback, and the gate's job is finished the moment the write + // is: it orders COMMENCEMENT, not notification. + l.cfg.durable.end() + if err != nil { if l.cfg.observer.CheckpointSaveFailed != nil { l.cfg.observer.CheckpointSaveFailed(err) } From f3a042585c53b7dbb4f9830afe2127ee61c79d23 Mon Sep 17 00:00:00 2001 From: Jeremy Daer Date: Tue, 18 Aug 2026 13:11:21 -0700 Subject: [PATCH 05/51] Event feed: an occupied deferral slot no longer blinds the drain's fatal scan MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #760, and the answer turned out to be removing a mechanism rather than adding one. drainScan returned the moment it found the deferral slot occupied, on the reasoning that "everything the pump has queued arrived behind this one, so the scan stops here either way". That is a claim about arrival ORDER, where §23's carve-out is a claim about which VERDICT governs: a raw invalid_event_stream_command observed during Draining is Terminal(protocol_fatal) with the drain not completed, the held position not saved and no caught_up. With any non-fatal outcome parked ahead of it the scan looked at nothing, the drain completed, the position saved and caught_up announced — all three of the things the carve-out exists to prevent. The budget that scan runs under is pumpDepth+1, sized in drain's own comment for the express purpose of reaching every frame the pump had already read. An occupied slot spent NONE of it. rev 15 selected a bounded deferral queue carved out of pumpDepth for this, and that is not needed. Reading the dispatch path, only ONE deferral is ever dispatched: dispatchDeferred sends it to pumpExited, dispatchDisconnect, or the invalid-frame teardown, and every one ends the cycle. A second entry could never be reached, so a queue would be a buffer sized for a delivery that cannot happen. The scan keeps the FIRST outcome — which is what preserves arrival order for the one that is reported — and discards the rest as it passes them. That removes the whole capacity question with it. Nothing new is retained, so the published memory bound ((pump depth + liveBufferCapacity) × MAX_FRAME_BYTES) is untouched, pumpDepth is untouched, and so is the depth at which the pump blocks — no fixture sweep, no §23 restatement, no ExportPumpDepth change. The new subtest needs both halves of the two around it and neither alone reaches the defect: deferring during the entry poll is what occupies the slot, and queuing the fatal frame MID-DRAIN is what puts it somewhere only the scan can find. Queued any earlier the ownership cut consumes it first and the carve-out fires without the scan being involved — which is how the first draft passed against un-fixed code. #758/#759, the bounded wait: no code change, and the reason is documented in place. The overshoot is real — a suspended firing re-arms a full window, so a deferral landing just after a frame waits close to two windows — but it cannot exceed two, because the post-select predicate runs on every wake and the first re-armed firing is necessarily past a deadline set one window after the deferral. A shorter re-armed window would be a staleness window that is not one, and a dedicated timer is the seventh kind §23 pins against, so the bound is stated rather than patched. I could not reproduce rev 15's missing-wake-source hang. Every path leaves one: a suspended firing always re-arms, and a latched authoritative expiry is reported by evaluate before any wait can consume its timer. The overshoot is what is real, and it is now published as the two-window worst case — carried to bc3 as an open §23 contract question rather than asserted as settled here. --- go/pkg/basecamp/eventfeed/catchup.go | 83 +++++++++++++++++++---- go/pkg/basecamp/eventfeed/catchup_test.go | 73 ++++++++++++++++++++ 2 files changed, 142 insertions(+), 14 deletions(-) diff --git a/go/pkg/basecamp/eventfeed/catchup.go b/go/pkg/basecamp/eventfeed/catchup.go index a69d06debc..c5f64a3fdc 100644 --- a/go/pkg/basecamp/eventfeed/catchup.go +++ b/go/pkg/basecamp/eventfeed/catchup.go @@ -428,6 +428,33 @@ func (l *loop) pollPage(at *attempt, cursor Cursor) pollAttempt { // used: SPEC.md §23 pins exactly six timer kinds AND every state's exact // timer set, both asserted by the cross-SDK fixtures, so a seventh kind is a // spec change across six SDKs rather than a fix to the Go reference. +// +// # The worst case is TWO staleness windows, and that is the published bound +// +// Borrowing the window means the wait wakes on the window's cadence, not the +// deadline's, so it can overshoot — and the size of the overshoot is worth +// stating rather than discovering. +// +// The window in flight was armed at the last frame receipt, at or before the +// deferral, so it fires at or before the deadline. If the pump is blocked at +// that moment the firing is not evidence (a full queue is a fast peer, not a +// dead one), evaluate re-arms, and the NEXT wake is a full window later — past +// the deadline by however much of the first window had already elapsed when +// the deferral happened. A deferral landing immediately after a frame +// therefore waits close to two windows in total: one for the in-flight window +// to close, one for the re-armed window to deliver the wake that lets the +// post-select predicate observe the lapse. +// +// It cannot exceed two. The post-select check runs on EVERY wake, and by the +// first re-armed firing the clock is necessarily past a deadline set one +// window after the deferral — so a second re-arm is never waited out, however +// long the pump stays blocked. +// +// This is a property of borrowing the timer, not a defect to patch: a shorter +// re-armed window would be a staleness window that is not one, and a dedicated +// one is the seventh kind above. It is carried to bc3 as an open §23 contract +// question (staleness × in-flight-poll deferral) rather than asserted as +// settled contract here. func (l *loop) awaitSupersededPoll(at *attempt, done <-chan pollResult, deadline time.Time) pollAttempt { for { staleTimer, staleGen := at.lc.stale.current() @@ -694,24 +721,32 @@ func (l *loop) drain(at *attempt) (cycleOutcome, bool) { // queue's own depth (see drain); an ordinary frame consuming it must never be // able to end the scan short of a fatal frame the pump had already queued. func (l *loop) drainScan(at *attempt, budget *int) (cycleOutcome, bool) { - if d := l.deferred; d != nil { - // A fatal frame already in the slot is dispatched now rather than - // after the save: the carve-out is about what governs, not about - // which queue the frame is sitting in. Everything the pump has - // queued arrived behind this one, so the scan stops here either way. - if !d.closed { - if f, ok := protocolFatalFrame(d.item); ok { - l.deferred = nil - return l.terminateProtocolFatal(at, f), true - } + // A fatal frame already in the slot is dispatched now rather than after + // the save: the carve-out is about what governs, not about which queue the + // frame is sitting in. + if d := l.deferred; d != nil && !d.closed { + if f, ok := protocolFatalFrame(d.item); ok { + l.deferred = nil + return l.terminateProtocolFatal(at, f), true } - return cycleOutcome{}, false } + // An occupied slot no longer ENDS the scan (#760). It used to, on the + // reasoning that "everything the pump has queued arrived behind this one, + // so the scan stops here either way" — which is a statement about arrival + // order, where the carve-out is a statement about which VERDICT governs. A + // raw invalid_event_stream_command the pump had already read went unseen + // whenever any non-fatal frame happened to be parked ahead of it; the drain + // then completed, the held entry position saved, and caught_up announced — + // the three things §23 says a fatal observed during Draining must prevent. + // + // The budget above is sized at pumpDepth+1 for the express purpose of + // reaching every frame the pump had already read. An occupied slot spent + // NONE of it. for *budget > 0 { select { case item, ok := <-at.lc.frames: if !ok { - l.deferred = &deferredFrame{closed: true} + l.deferForDrain(&deferredFrame{closed: true}) return cycleOutcome{}, false } *budget-- @@ -724,8 +759,7 @@ func (l *loop) drainScan(at *attempt, budget *int) (cycleOutcome, bool) { if f, ok := protocolFatalFrame(item); ok { return l.terminateProtocolFatal(at, f), true } - l.deferred = &deferredFrame{item: item} - return cycleOutcome{}, false + l.deferForDrain(&deferredFrame{item: item}) } default: return cycleOutcome{}, false @@ -734,6 +768,27 @@ func (l *loop) drainScan(at *attempt, budget *int) (cycleOutcome, bool) { return cycleOutcome{}, false } +// deferForDrain parks one socket outcome the scan passed over, keeping the +// FIRST and discarding the rest. +// +// Discarding is not a shortcut; it is what the dispatch path makes of them. +// Only one deferral is ever dispatched: dispatchDeferred sends it to +// pumpExited, to dispatchDisconnect, or to the invalid-frame teardown, and +// every one of those ENDS THE CYCLE. A second entry could never be reached, so +// a queue of them would be a buffer sized for a delivery that cannot happen. +// Keeping the first is what preserves arrival order for the one outcome that +// is actually reported. +// +// This is why the scan needs no queue and no share of pumpDepth: it retains +// exactly what the single slot always retained. The connector's published +// memory bound — (pump depth + liveBufferCapacity) × MAX_FRAME_BYTES — is +// untouched, and so is the depth at which the pump blocks. +func (l *loop) deferForDrain(d *deferredFrame) { + if l.deferred == nil { + l.deferred = d + } +} + // terminateProtocolFatal takes the raw disconnect's state-generic terminal // through the ordinary dispatch, so the frame is counted as handled and the // disconnect observation carries the server's reason verbatim. diff --git a/go/pkg/basecamp/eventfeed/catchup_test.go b/go/pkg/basecamp/eventfeed/catchup_test.go index 0e85bae967..181f1725cb 100644 --- a/go/pkg/basecamp/eventfeed/catchup_test.go +++ b/go/pkg/basecamp/eventfeed/catchup_test.go @@ -1097,6 +1097,79 @@ func TestProtocolFatalDuringDrainingIsImmediate(t *testing.T) { assertIDs(t, h.deliveredIDs()) }) + // #760: the fatal frame is queued while the deferral slot is ALREADY + // occupied by a non-fatal socket outcome. The scan used to return the + // moment it found the slot occupied — "everything the pump has queued + // arrived behind this one, so the scan stops here either way" — which is a + // claim about arrival order where the carve-out is a claim about which + // VERDICT governs. The drain then completed, the held position saved and + // caught_up announced: the three things §23 says a fatal observed during + // Draining must prevent. + // + // This needs BOTH halves of the two subtests around it, and neither alone + // reaches it. Deferring during the entry poll is what occupies the slot; + // queuing the fatal frame mid-drain is what puts it somewhere only the + // scan can find. Queued any earlier, the ownership cut consumes it first + // and the carve-out fires without the scan being involved at all — which + // is how the first draft of this test passed against the un-fixed code. + // + // A regression shows up as the watchdog rather than as a failed assertion + // on the save, and that IS the un-fixed behavior rather than a weak test: + // the drain completes, the held position saves, caught_up announces, and + // the recoverable disconnect is then dispatched — so the feed reconnects + // and parks in Backoff on a clock only the test advances. The iteration + // never ending is precisely the carve-out having failed. + t.Run("queued mid-drain behind an occupied deferral slot", func(t *testing.T) { + var caughtUp int + h, store := newDrainingHarness(t, &caughtUp) + h.pauseAfter = 1 + deferred := make(chan struct{}, 4) + h.conn.OnFrameDeferred(func() { deferred <- struct{}{} }) + handedOff := make(chan struct{}, 32) + h.conn.OnPumpHandedOff(func(bool) { + select { + case handedOff <- struct{}{}: + default: + } + }) + var conn *feedtest.Conn + h.polls.OnCall(func(feedtest.PollCall) { + // A reconnecting disconnect is recoverable, so it parks in the slot + // rather than ending the cycle — and it is still there when the + // drain begins, because the walk's dispatch points are the page + // boundary and the walk's END, and this page carries no `next`. + conn.Serve(frameDisconnect("server_restart", true)) + select { + case <-deferred: + case <-time.After(watchdog): + t.Error("the recoverable disconnect frame was never deferred") + } + }) + h.start() + conn = h.driveToSubscribed() + h.serveSettled(conn, frameMessage(noFilterIdentifier, 41)) + conn.Serve(frameConfirm(noFilterIdentifier)) + + // The consumer parks inside the drain's delivery of the retained event. + h.waitUntil("the drain parked mid-delivery", func() bool { return len(h.deliveredIDs()) == 1 }) + for len(handedOff) > 0 { + <-handedOff + } + // The fatal frame must be IN the queue before the drain's next scan: + // that receive is non-blocking by design. + conn.Serve(frameDisconnect("invalid_event_stream_command", false)) + select { + case <-handedOff: + case <-time.After(watchdog): + t.Error("the fatal frame never reached the pump's queue") + } + h.resume() + h.join() + + assertProtocolFatalDrain(t, h, store, caughtUp) + assertIDs(t, h.deliveredIDs(), 41) + }) + t.Run("queued while the drain is delivering", func(t *testing.T) { var caughtUp int h, store := newDrainingHarness(t, &caughtUp) From 2a257d9b74966eb282568c363dab86313f7ed020 Mon Sep 17 00:00:00 2001 From: Jeremy Daer Date: Tue, 18 Aug 2026 14:30:23 -0700 Subject: [PATCH 06/51] Event feed: close four semantic gaps daybreak found in B2-B5 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review of 4cff076 (B1 excluded, since uncommitted at the time). All four P1s reproduce; each fix is red-proven against the reported shape. P1 — Observer.Disconnected still leaked ticket text. Both arguments carry peer-controlled strings: a raw disconnect frame's reason, and a WebSocket close reason rendered through the error. Both were BOUNDED by §9's cap, which limits how much of a credential escapes rather than whether any does — the identical trap dialFailure documents three review rounds of. The cable server is exactly the party that knows the ticket: it was dialed with it. Both now go through closed vocabularies. observableDisconnectReason keeps the two reasons that change behavior and reports everything else as "other"; observableSocketError passes the connector's own sentinels and typed errors and degrades anything from a seam to a generic cause. CloseError.Error() renders only the code — an integer cannot carry a credential, and RFC 6455 codes are what an operator classifies on; Reason stays a readable FIELD. A canary planting a ticket in every peer-controlled teardown string found MORE than was reported: raw seam read errors leak too, which seam documentation cannot repair because the connector forwarded them verbatim. Four arms, all red before and green after. P1 — durableGate deadlocked reentrantly and blocked Close. It held the lock across CheckpointStore.Save while Close waited for it: a store whose Save calls Close self-deadlocks on the caller's own goroutine, and a merely stalled store blocked EVERY Close indefinitely — contradicting the one thing Close promises unconditionally. The two promises could not coexist, so the waiting one is dropped: the gate is claimed and released atomically, Close latches and returns, and a save that already claimed still completes. The guarantee is unchanged in substance — no save COMMENCES after Close returns — with commencing defined as claiming the gate, which takes no host code with it. The old test asserted Close WAITS and is replaced by one asserting it does not; the gate-holding variant deadlocks the new test at 40s. P1 — Close precedence, reopened by #763. Arming staleness before Connected also starts the pump before it, so a fatal frame can already be queued when a Connected callback calls Close, leaving two ready select cases. Reproduced: 25/50 rounds emitted a terminal element after Close returned. Fixed at the ONE exit (emitTerminal) rather than per-select — many selects, one exit, and a rule every future select must remember is what produced this. P1 — B2 discarded an earlier socket verdict. A deferred protocol-fatal followed by a positionless page took poll_failed, because disposal clears the deferral. The failed-poll branch already dispatches the deferral first, with a comment giving this exact reason; the new guard did not follow it. Now it does. Also fixed: TestNoCheckpointSaveCommencesAfterClose was vacuous (it closed before the run reached a page) and now closes from Observer.PageDelivered, the callback immediately preceding the save; the cancellation check after the checkpoint load covers every result rather than only the failure, since a found-empty result became terminal and a successful one let the run fire Connecting after Close; and deliver()'s stale "no delivery begins after Close returns" claim is corrected in place — it is a check-then-act, and the honest guarantee is the one Close states. Two of these touch foundations files that belong to #777 — CloseError in seams.go and its test. They stay here because the leak is only observable through the loop's observer path, which is this PR's, and the canary that proves it lives here. TestCloseError_Message is INVERTED, not adjusted: it required Error() to render the peer's reason, so it pinned the wrong contract. Verified: build, vet, -race, 22/22 fixtures, go-lint 0 issues, gosec 0 issues. --- go/pkg/basecamp/eventfeed/catchup.go | 28 +++- go/pkg/basecamp/eventfeed/catchup_test.go | 38 +++++ go/pkg/basecamp/eventfeed/connector.go | 64 ++++---- go/pkg/basecamp/eventfeed/connector_test.go | 143 ++++++++++++---- go/pkg/basecamp/eventfeed/errors_test.go | 31 +++- go/pkg/basecamp/eventfeed/loop.go | 173 +++++++++++++++----- go/pkg/basecamp/eventfeed/loop_test.go | 78 +++++++++ go/pkg/basecamp/eventfeed/seams.go | 8 +- 8 files changed, 450 insertions(+), 113 deletions(-) diff --git a/go/pkg/basecamp/eventfeed/catchup.go b/go/pkg/basecamp/eventfeed/catchup.go index c5f64a3fdc..07f6bb769d 100644 --- a/go/pkg/basecamp/eventfeed/catchup.go +++ b/go/pkg/basecamp/eventfeed/catchup.go @@ -221,6 +221,19 @@ func (l *loop) walk(at *attempt, cursor Cursor, presentClass bool) (out cycleOut // that answered without a position answers the same way to the same // request, so a retry loop would spin. Nothing durable has moved. if p.page.Position == "" { + // A socket outcome deferred while THIS call was in flight is + // dispatched BEFORE the malformed page is classified, for exactly + // the reason the failed-poll branch above gives: disposal discards + // the deferral, so classifying first loses the server's own + // verdict. Concretely, a raw invalid_event_stream_command parked + // during the call would be reported as poll_failed instead of + // Terminal(protocol_fatal). Transition 21's finish-the-page + // ordering does not apply here — it exists so an ACCEPTED page's + // deliveries and save are not stranded by the socket's death, and + // this page is being refused, so there is nothing to strand. + if out, done := l.dispatchDeferred(at); done { + return out, "", true + } l.disposeAttempt(at, nil) return cycleOutcome{kind: outcomeTerminal, term: &TerminalError{ Reason: ReasonPollFailed, @@ -877,9 +890,18 @@ func (l *loop) stream(at *attempt) cycleOutcome { // dispatch points, because Close is callable from the consumer's own loop // body and from any observer or handler callback — all of which run on this // goroutine, between one delivery and the next. Close cancels synchronously -// (Connector.Close), so this check is what makes "no delivery begins after -// Close returns" true mid-page and mid-drain, not merely at the next page -// boundary. It reports the same false every caller already routes onto the +// (Connector.Close), so this check is what narrows the window mid-page and +// mid-drain rather than only at the next page boundary. +// +// It does NOT make "no delivery begins after Close returns" true, and that +// claim used to be here. This is a check-then-act on a value another goroutine +// writes: a Close landing between the check and the yield still lets that yield +// happen. Closing the window would mean holding a lock across the consumer's +// own loop body — unbounded host code. What the connector guarantees instead is +// stated on Close: cancellation is visible before it returns, and the one +// effect that outlives the process (a checkpoint save) is ordered against it. +// +// It reports the same false every caller already routes onto the // Closed edge, and deliberately does NOT latch `stopped`: cancellation is not // a consumer break. func (l *loop) deliver(ev Event) bool { diff --git a/go/pkg/basecamp/eventfeed/catchup_test.go b/go/pkg/basecamp/eventfeed/catchup_test.go index 181f1725cb..4529d775b1 100644 --- a/go/pkg/basecamp/eventfeed/catchup_test.go +++ b/go/pkg/basecamp/eventfeed/catchup_test.go @@ -1812,3 +1812,41 @@ func TestPollPageWithNoPositionIsMalformed(t *testing.T) { assertIDs(t, h.deliveredIDs(), 101) }) } + +// TestDeferredFatalOutranksAMalformedPage is the precedence half of the +// empty-position guard. A socket outcome deferred while the call was in flight +// is the server's own verdict, and disposal discards the deferral — so +// classifying the malformed page first loses it, and a raw +// invalid_event_stream_command is reported as poll_failed. +// +// This mirrors the failed-poll branch, which already dispatches the deferral +// before classifying, for the same reason and with the same comment. +func TestDeferredFatalOutranksAMalformedPage(t *testing.T) { + store := feedtest.NewStore() + store.Stored("pos-0") + h := storedHarness(t, store) + h.minter.ScriptTicket(ticket(1)) + h.polls.ScriptPage(eventfeed.PollPage{Position: ""}) // malformed + deferred := make(chan struct{}, 4) + h.conn.OnFrameDeferred(func() { deferred <- struct{}{} }) + var conn *feedtest.Conn + h.polls.OnCall(func(feedtest.PollCall) { + conn.Serve(frameDisconnect("invalid_event_stream_command", false)) + select { + case <-deferred: + case <-time.After(watchdog): + t.Error("the fatal frame was never deferred") + } + }) + h.start() + conn = h.driveToSubscribed() + conn.Serve(frameConfirm(noFilterIdentifier)) + h.join() + + _, terminal, _ := h.snapshot() + if terminal == nil || terminal.Reason != eventfeed.ReasonProtocolFatal { + t.Fatalf("terminal = %v, want reason %q — the server's verdict outranks the malformed page", + terminal, eventfeed.ReasonProtocolFatal) + } + assertPositions(t, store.Saves()) +} diff --git a/go/pkg/basecamp/eventfeed/connector.go b/go/pkg/basecamp/eventfeed/connector.go index b7033511dd..429a023641 100644 --- a/go/pkg/basecamp/eventfeed/connector.go +++ b/go/pkg/basecamp/eventfeed/connector.go @@ -208,8 +208,8 @@ type Connector struct { // A save is the connector's only effect that outlives the process, and the // failure this prevents is sequential rather than adversarial: a consumer calls // Close, opens a second connector over the same checkpoint file, and a save the -// first connector had not yet commenced lands afterwards — on top of a lineage -// the new run has already loaded and may already have advanced. +// first connector had not yet begun lands afterwards — on top of a lineage the +// new run has already loaded and may already have advanced. // // Cancellation alone cannot prevent it. A CheckpointStore is entitled to ignore // ctx, and the built-in FileCheckpointStore documents that it does, deliberately: @@ -218,39 +218,40 @@ type Connector struct { // cancelled context Close publishes is exactly the signal this store is // specified not to act on. // -// It is a lock of its own, not the Connector's mu, because a save must not hold -// mu across host store I/O — that would block every concurrent Close caller for -// as long as the host's write takes, and mu's whole job is that Close's callers -// are serialized cheaply. +// # Close does not wait, and must not // -// A save that has ALREADY commenced runs to completion and Close waits for it. -// That is the intended trade: its position was accepted and delivered before -// Close was called, and abandoning a durable write half-way is worse than -// finishing it. The wait is bounded by one store write. +// The guarantee is that no save COMMENCES after Close returns, where commencing +// is claiming the gate — an operation that is atomic against Close and takes no +// host code with it. The lock is never held across CheckpointStore.Save. +// +// Holding it across the store call was the obvious spelling and is wrong twice +// over. A store whose Save calls Connector.Close self-deadlocks on the caller's +// own goroutine, and Close is documented as callable from anywhere; and a store +// that merely stalls blocks EVERY Close for as long as it stalls, which +// contradicts the one thing Close promises unconditionally — that cancellation +// is visible before it returns. +// +// A save that claimed the gate before Close therefore runs to completion after +// Close returns. That is the intended reading rather than a gap: its position +// was accepted and delivered before Close was called, and abandoning a durable +// write half-way is worse than finishing it. What cannot happen is a save the +// connector had not yet decided to make. type durableGate struct { mu sync.Mutex closed bool } // begin claims the right to start one durable operation, reporting whether it -// was granted. The caller MUST call end when it returns. The gate is held for -// the whole operation, which is what makes the ordering against Close exact -// rather than a check-then-act. +// was granted. The lock is released before returning: the claim is the +// serialized act, not the write. func (g *durableGate) begin() bool { g.mu.Lock() - if g.closed { - g.mu.Unlock() - return false - } - return true + defer g.mu.Unlock() + return !g.closed } -// end releases the gate after a granted begin. -func (g *durableGate) end() { g.mu.Unlock() } - -// close latches the gate, waiting for any operation already in flight. Called -// by Close, on the caller's goroutine, so that no save can COMMENCE after -// Close returns. +// close latches the gate. It never waits for an operation already in flight — +// see the type comment. func (g *durableGate) close() { g.mu.Lock() g.closed = true @@ -550,15 +551,12 @@ func (c *Connector) Close() error { c.cancelRun = nil } c.mu.Unlock() - // After the cancel, and outside mu: latch the durable gate, waiting for at - // most one checkpoint save already in flight. This is the one effect that - // outlives the process, so it is the one whose COMMENCEMENT is ordered - // against Close rather than merely cancelled — a CheckpointStore may - // ignore ctx, and the built-in one documents that it does. - // - // Outside mu so a host store's write cannot hold up a concurrent Close - // caller's cheap serialization, and after the cancel so the cancellation is - // already visible while this waits. + // After the cancel, and outside mu: latch the durable gate, so no + // checkpoint save COMMENCES from here on. This is the one effect that + // outlives the process, and the one cancellation cannot stop — a + // CheckpointStore may ignore ctx, and the built-in one documents that it + // does. Latching does not wait for a save already under way; see + // durableGate for why waiting is the wrong trade. c.cfg.durable.close() return nil } diff --git a/go/pkg/basecamp/eventfeed/connector_test.go b/go/pkg/basecamp/eventfeed/connector_test.go index 3ec585b4b2..6866a7786a 100644 --- a/go/pkg/basecamp/eventfeed/connector_test.go +++ b/go/pkg/basecamp/eventfeed/connector_test.go @@ -687,18 +687,20 @@ func (s *blockingStore) attemptCount() int { return s.attempts } -// TestCloseWaitsForAnInFlightCheckpointSave is the ordered half of blocker 6. -// A checkpoint save is the connector's only effect that outlives the process, -// so its COMMENCEMENT is ordered against Close: one already in flight runs to -// completion and Close waits for it, rather than Close returning while a -// durable write is still pending against a lineage a second connector may -// already have loaded. +// TestCloseDoesNotBlockOnAnInFlightCheckpointSave pins the half of the durable +// gate that is about Close's own promise. // -// Cancellation cannot substitute. A CheckpointStore is entitled to ignore ctx — -// the built-in FileCheckpointStore documents that it does, so a position -// already accepted is not dropped by a shutdown race — so the cancelled context -// Close publishes is precisely the signal such a store is specified to ignore. -func TestCloseWaitsForAnInFlightCheckpointSave(t *testing.T) { +// Holding the gate across CheckpointStore.Save was the obvious spelling and is +// wrong twice: a store whose Save calls Close self-deadlocks on the caller's +// goroutine (Close is documented as callable from anywhere), and a store that +// merely stalls blocks EVERY Close for as long as it stalls — contradicting the +// one thing Close promises unconditionally, that cancellation is visible before +// it returns. +// +// So Close latches and returns. The save that already claimed the gate still +// completes: its position was accepted and delivered before Close was called, +// and abandoning a durable write half-way is worse than finishing it. +func TestCloseDoesNotBlockOnAnInFlightCheckpointSave(t *testing.T) { store := newBlockingStore() h := newHarness(t, eventfeed.WithCheckpointStore(store), @@ -706,11 +708,10 @@ func TestCloseWaitsForAnInFlightCheckpointSave(t *testing.T) { h.minter.ScriptTicket(ticket(1)) h.polls.ScriptPage(eventfeed.PollPage{Events: []eventfeed.Event{pollEvent(101)}, Position: "pos-1"}) h.start() - sock := h.driveToSubscribed() sock.Serve(frameConfirm(noFilterIdentifier)) - // The save is now parked inside the store, on the run goroutine. + // The save is parked inside the store, on the run goroutine. select { case <-store.entered: case <-time.After(watchdog): @@ -719,47 +720,60 @@ func TestCloseWaitsForAnInFlightCheckpointSave(t *testing.T) { closeReturned := make(chan struct{}) go func() { _ = h.conn.Close(); close(closeReturned) }() - - // Close must NOT return while the write is pending. select { case <-closeReturned: - t.Fatal("Close returned while a checkpoint save was still in flight") - case <-time.After(150 * time.Millisecond): + case <-time.After(watchdog): + t.Fatal("Close blocked behind an in-flight checkpoint save; a stalled store must not hold up Close") } close(store.release) - select { - case <-closeReturned: - case <-time.After(watchdog): - t.Fatal("Close did not return after the save completed") + deadline := time.Now().Add(watchdog) + for len(store.recorded()) == 0 && time.Now().Before(deadline) { + time.Sleep(time.Millisecond) } if got := store.recorded(); len(got) != 1 || got[0] != "pos-1" { - t.Errorf("recorded saves = %v, want [pos-1] — an in-flight write completes", got) + t.Errorf("recorded saves = %v, want [pos-1] — a write that already commenced completes", got) } } // TestNoCheckpointSaveCommencesAfterClose is the other direction: once Close -// has returned, a save that had not begun does not begin. Without the gate the -// run goroutine's next accepted page would write through on a cancelled -// context, which a store is allowed to ignore. +// has returned, a save that had not begun does not begin. +// +// Close is taken from inside Observer.PageDelivered — the callback that fires +// on the consumer's goroutine immediately BEFORE the page's save — so the run +// is demonstrably at the save boundary when it lands. An earlier version closed +// before the feed ever reached a page and passed with the gate removed +// entirely, which is no test at all. func TestNoCheckpointSaveCommencesAfterClose(t *testing.T) { store := newBlockingStore() - close(store.release) // saves complete immediately - h := newHarness(t, + close(store.release) // any save that does commence completes immediately + var h *harness + h = newHarness(t, eventfeed.WithCheckpointStore(store), - eventfeed.WithConsumerNamespace("agent")) + eventfeed.WithConsumerNamespace("agent"), + eventfeed.WithObserver(eventfeed.Observer{ + PageDelivered: func(int, string) { + if err := h.conn.Close(); err != nil { + t.Errorf("Close: %v", err) + } + }, + })) h.minter.ScriptTicket(ticket(1)) + h.polls.ScriptPage(eventfeed.PollPage{ + Events: []eventfeed.Event{pollEvent(101)}, + Position: "pos-1", + }) h.start() - - // Close before any page is accepted, so no save has commenced. - if err := h.conn.Close(); err != nil { - t.Fatalf("Close() = %v", err) - } + sock := h.driveToSubscribed() + sock.Serve(frameConfirm(noFilterIdentifier)) h.join() if n := store.attemptCount(); n != 0 { t.Errorf("store.Save was entered %d time(s) after Close returned, want 0", n) } + if got := store.recorded(); len(got) != 0 { + t.Errorf("recorded saves = %v, want none", got) + } } // TestCancelledCheckpointLoadIsNotAStoreFailure is the third item. The load @@ -942,3 +956,66 @@ func TestConcurrentCloseIsSerializedAndIdempotent(t *testing.T) { t.Errorf("terminal = %v, want none", terminal) } } + +// TestCloseFromConnectedOutranksAQueuedFatalFrame is the precedence #763 +// reopened. Arming staleness before Observer.Connected also starts the frame +// pump before it, so by the time that callback runs the peer can already have +// queued a fatal frame. If the callback then calls Close, the state machine's +// next select has two ready cases — the frame and the cancellation — and Go +// picks freely. +// +// §23 makes close() a universal edge from every non-absorbing state and ends +// the iterator with NO error element, so the terminal must never win. The guard +// is central (emitTerminal), not per-select: there are many selects and one +// exit, and a rule every future select has to remember is the shape that +// produced this. +// +// Driven repeatedly because the underlying choice is random: with the guard the +// count is deterministically zero, and without it a terminal escapes within a +// handful of rounds. +func TestCloseFromConnectedOutranksAQueuedFatalFrame(t *testing.T) { + const rounds = 50 + terminals := 0 + for i := range rounds { + var h *harness + handedOff := make(chan struct{}, 8) + h = newHarness(t, eventfeed.WithObserver(eventfeed.Observer{ + Connected: func() { + // Connected now fires with the pump already running, so the + // peer's fatal frame can be queued before the consumer decides + // to stop — and then both are ready at the same select. + if sock := h.tr.LastConn(); sock != nil { + sock.Serve(frameDisconnect("invalid_event_stream_command", false)) + // Wait until the pump has actually QUEUED it, so the + // cancellation below genuinely races a ready frame rather + // than beating it to the channel. + select { + case <-handedOff: + case <-time.After(watchdog): + } + } + h.conn.Close() //nolint:errcheck // asserted via the element count + }, + })) + h.conn.OnPumpHandedOff(func(bool) { + select { + case handedOff <- struct{}{}: + default: + } + }) + h.minter.ScriptTicket(ticket(1)) + h.polls.ScriptPage(eventfeed.PollPage{Position: "pos-1"}) + h.start() + h.join() + if _, terminal, _ := h.snapshot(); terminal != nil { + terminals++ + if i == 0 || terminals == 1 { + t.Logf("round %d emitted %v after Close", i, terminal) + } + } + } + if terminals != 0 { + t.Errorf("%d/%d rounds emitted a terminal element after Close returned; the Closed edge must win", + terminals, rounds) + } +} diff --git a/go/pkg/basecamp/eventfeed/errors_test.go b/go/pkg/basecamp/eventfeed/errors_test.go index 23a43a2cb4..51867a751f 100644 --- a/go/pkg/basecamp/eventfeed/errors_test.go +++ b/go/pkg/basecamp/eventfeed/errors_test.go @@ -128,15 +128,36 @@ func TestSeamErrors_UnwrapReachesTheCause(t *testing.T) { } } -func TestCloseError_Message(t *testing.T) { - withReason := &CloseError{Code: 1008, Reason: "policy violation"} - if got := withReason.Error(); got != "cable connection closed by peer: code 1008 (peer reason withheld)" { - t.Errorf("Error() = %q", got) - } +// TestCloseError_MessageWithholdsThePeerReason inverts what this test asserted. +// It previously required Error() to RENDER Reason, bounded by §9's cap; that +// pinned the wrong contract, so nothing short of inverting it is honest. +// +// Reason is peer-supplied and this error reaches Observer.Disconnected, which +// hosts log. The cable server is precisely the party that knows the ticket — it +// was dialed with it — so a server echoing the URL it was dialed with puts the +// ticket in the host's logs, and §23 says the ticket is never logged. Bounding +// limits how much of a credential escapes, not whether any does. +func TestCloseError_MessageWithholdsThePeerReason(t *testing.T) { + const canary = "sekrit-ticket-value" + withReason := &CloseError{Code: 1008, Reason: "closing wss://x/cable?ticket=" + canary} + got := withReason.Error() + if got != "cable connection closed by peer: code 1008 (peer reason withheld)" { + t.Errorf("Error() = %q, want the fixed withheld marker", got) + } + if strings.Contains(got, canary) || strings.Contains(got, "ticket=") { + t.Errorf("Error() echoed peer text: %q", got) + } + // The code survives: an integer cannot carry a credential, and RFC 6455 + // close codes are what an operator classifies on. bare := &CloseError{Code: 1006} if got := bare.Error(); got != "cable connection closed by peer: code 1006" { t.Errorf("Error() = %q", got) } + // Reason remains a FIELD, so a host that has decided its server is + // trustworthy can read it deliberately. + if withReason.Reason == "" { + t.Error("Reason must remain readable on the struct") + } } // TestCloseError_RenderingWithholdsPeerReason pins the last rendering in the diff --git a/go/pkg/basecamp/eventfeed/loop.go b/go/pkg/basecamp/eventfeed/loop.go index a2497b7336..4898abe3e2 100644 --- a/go/pkg/basecamp/eventfeed/loop.go +++ b/go/pkg/basecamp/eventfeed/loop.go @@ -640,8 +640,7 @@ func (l *loop) run(yield func(Event, error) bool) { return } if terr := l.loadCheckpoint(); terr != nil { - l.setState(stateTerminal) - l.yield(Event{}, terr) + l.emitTerminal(terr) return } var delay time.Duration @@ -652,8 +651,7 @@ func (l *loop) run(yield func(Event, error) bool) { l.setState(stateClosed) return case outcomeTerminal: - l.setState(stateTerminal) - l.yield(Event{}, out.term) + l.emitTerminal(out.term) return case outcomeFailed: // The failed attempt is fully disposed, so the exact @@ -681,6 +679,30 @@ func (l *loop) run(yield func(Event, error) bool) { // confirmation. Every exit path has disposed the attempt (socket closed, // pump joined, timers stopped) before returning — except the confirmed // handoff, which transfers ownership to the catch-up continuation. +// emitTerminal is the ONE place a terminal element reaches the consumer, and +// it defers to Close. +// +// §23 makes close() a universal edge from every non-absorbing state and ends +// the iterator with NO error element, so a terminal must never outrank a Close +// the consumer has already taken. The race is real and is not hypothetical +// scheduling paranoia: the frame pump runs from socket open, so by the time a +// consumer callback calls Close there can already be a fatal frame queued, and +// the state machine's next select has two ready cases with no ordering between +// them. Go may take the frame. +// +// Checking here rather than at each select is deliberate. There are many +// selects and one exit, so the invariant belongs at the exit — a per-select +// check is a rule every future select has to remember, which is the shape that +// produced this defect in the first place. +func (l *loop) emitTerminal(term *TerminalError) { + if l.runCtx.Err() != nil { + l.setState(stateClosed) + return + } + l.setState(stateTerminal) + l.yield(Event{}, term) +} + func (l *loop) runCycle(delay time.Duration) cycleOutcome { at := &attempt{} at.ctx, at.cancel = context.WithCancel(l.runCtx) @@ -1068,7 +1090,7 @@ func (l *loop) handleFrame(at *attempt, deadline *Timer, item pumpItem) (cycleOu func (l *loop) dispatchDisconnect(at *attempt, deadline Timer, f frame) cycleOutcome { preWelcome := l.state == stateAwaitingWelcome l.disposeAttempt(at, deadline) - l.observeDisconnected(f.reason, nil) + l.observeDisconnected(observableDisconnectReason(f.reason), nil) switch f.reason { case disconnectReasonProtocolFatal: // Transition 13 — and the state-generic rule for AwaitingWelcome: @@ -1224,15 +1246,98 @@ func (l *loop) disposeAttempt(at *attempt, deadline Timer) { } } -// observeDisconnected reports one socket teardown. The reason is frame-derived -// (a raw disconnect frame's reason string, bounded only by the 1 MiB frame -// limit) and observers log it, so it is bounded by the same §9 cap the -// invalid-frame rendering uses — §23's Security Invariants apply it to frame -// contents generally, not just to errors. Dispatch reads f.reason itself, so -// the cap can never widen an unrecognized reason into a recognized one. +// observeDisconnected reports one socket teardown. Both arguments are reduced +// to closed vocabularies first: reason by observableDisconnectReason, err by +// observableSocketError. func (l *loop) observeDisconnected(reason string, err error) { if l.cfg.observer.Disconnected != nil { - l.cfg.observer.Disconnected(truncateErrorText(reason), err) + l.cfg.observer.Disconnected(truncateErrorText(reason), observableSocketError(err)) + } +} + +// errSocketFailed is the generic cause an unrecognized socket failure is +// reported as. +var errSocketFailed = errors.New("event feed socket failed") + +// observableSocketError reduces a teardown cause to what may be logged. +// +// The connector's OWN errors pass through: they are sentinels and typed values +// it constructs, so their text is its own. Anything else came out of a seam — +// CableConn.ReadFrame, WriteFrame, or a CableTransport's Dial — and a seam is +// host code wrapping a library. CableTransport is a documented extension point, +// and the seam contract requires those errors not to render the cable URL; this +// is what makes the connector not DEPEND on that requirement being met, because +// Observer.Disconnected is a logging surface and the ticket rides in the URL +// the peer was dialed with. +// +// Recognition is by TYPE and sentinel identity, never by matching text, so no +// arm can be widened by something a peer wrote. An unrecognized cause degrades +// to errSocketFailed: the direction of failure is "less diagnostic", never +// "leaks". The typed values the connector needs to ACT on are matched before +// this is ever called — dispatch reads them directly — so nothing here changes +// a verdict. +func observableSocketError(err error) error { + switch { + case err == nil: + return nil + case errors.Is(err, errStaleConnection), + errors.Is(err, errCableConnClosed), + errors.Is(err, context.Canceled), + errors.Is(err, context.DeadlineExceeded): + return err + } + var ce *CloseError + if errors.As(err, &ce) { + // Flat by construction and renders only its code — an integer cannot + // carry a credential. See CloseError.Error. + return ce + } + var ife *invalidFrameError + if errors.As(err, &ife) { + return ife + } + var de *DialError + if errors.As(err, &de) { + return de + } + var terr *TerminalError + if errors.As(err, &terr) { + return terr + } + return errSocketFailed +} + +// disconnectReasonOther is what an unrecognized disconnect reason is reported +// as. Deliberately not the peer's text, and deliberately not a truncation of +// it. +const disconnectReasonOther = "other" + +// observableDisconnectReason maps a raw disconnect reason onto the closed set +// the observer may see. +// +// The raw string is PEER-SUPPLIED and Observer.Disconnected is a logging +// surface, so forwarding it — even bounded — hands the peer a channel into the +// host's logs. §23 declares the ticket an "opaque bearer credential; never +// logged", and a peer that echoes the ticket-bearing URL it was dialed with +// puts it there. Truncation does not help: it bounds the length of a leak, not +// whether there is one. This is the identical trap dialFailure documents three +// review rounds of, and the same answer — a closed vocabulary — applies, for +// the same reason: to strip a credential out of arbitrary text you must MODEL +// it, and "opaque" is precisely the assumption that forbids modelling it. +// +// Nothing is lost that the observer could act on. Dispatch reads the raw reason +// itself, so the two reasons that CHANGE behavior are named exactly; everything +// else is, by construction, a reason this connector does not act on. An +// operator needing the verbatim string reads it from the server's own logs, +// where it did not have to cross a trust boundary to arrive. +func observableDisconnectReason(raw string) string { + switch raw { + case disconnectReasonUnauthorized, disconnectReasonProtocolFatal: + return raw + case "": + return "" + default: + return disconnectReasonOther } } @@ -1272,27 +1377,26 @@ func (l *loop) loadCheckpoint() *TerminalError { return nil } position, ok, err := l.cfg.store.Load(l.runCtx, l.checkpointKey()) + // One cancellation check, covering EVERY load result rather than only the + // failing one. The load runs on the first iteration and before the first + // mint, which is exactly the window a prompt Close lands in, and each + // result misbehaves differently if it is not checked: a failure becomes + // Terminal(checkpoint_load) — diagnosing the consumer's store for the + // consumer's own shutdown — a found-but-empty result becomes the same + // terminal, and a successful one lets the run walk on and fire + // Observer.Connecting after Close returned. §23 ends a closed iterator with + // no error element; the run's next check takes the Closed edge. + // + // Checked on the CONTEXT, not on the error's shape: a store is under no + // obligation to wrap ctx.Err(), and one returning its own error type would + // be misclassified. + if l.runCtx.Err() != nil { + //nolint:nilerr // Deliberate: any error here is real, but it is not a + // STORE failure — it is this run being closed underneath a load that + // was already in flight. + return nil + } if err != nil { - // A load that failed because the RUN was cancelled is not a store - // failure, and must not be reported as one. Close is a universal edge - // from every non-absorbing state and ends the iterator with no error - // element; surfacing Terminal(checkpoint_load) here would turn an - // ordinary shutdown — a Close racing the first iteration, which is the - // one iteration this load happens on — into a diagnosis of the - // consumer's own store. The run's next check takes the Closed edge. - // - // Checked on the CONTEXT rather than on the error's shape: a store is - // under no obligation to wrap ctx.Err(), and one that returns its own - // error type would otherwise be misclassified. - if l.runCtx.Err() != nil { - //nolint:nilerr // Deliberate: the error is real but it is not a - // STORE failure, it is this run being closed underneath a load that - // was already in flight. Returning it would give the consumer - // Terminal(checkpoint_load) for its own shutdown; §23 ends a closed - // iterator with no error element, and the run's next check takes - // the Closed edge. - return nil - } return &TerminalError{ Reason: ReasonCheckpointLoad, Msg: "the checkpoint store failed to load the stored position", @@ -1347,11 +1451,6 @@ func (l *loop) saveCheckpoint(position string) { return } err := l.cfg.store.Save(l.runCtx, l.checkpointKey(), position) - // Released before either observer callback fires. Holding it across host - // code would deadlock the documented case of Close being called from - // inside a callback, and the gate's job is finished the moment the write - // is: it orders COMMENCEMENT, not notification. - l.cfg.durable.end() if err != nil { if l.cfg.observer.CheckpointSaveFailed != nil { l.cfg.observer.CheckpointSaveFailed(err) diff --git a/go/pkg/basecamp/eventfeed/loop_test.go b/go/pkg/basecamp/eventfeed/loop_test.go index 385a00e52f..021bef1688 100644 --- a/go/pkg/basecamp/eventfeed/loop_test.go +++ b/go/pkg/basecamp/eventfeed/loop_test.go @@ -1797,3 +1797,81 @@ func TestStalenessArmsBeforeConnectedObserver(t *testing.T) { armedAtConnected, timerStaleness) } } + +// TestDisconnectObserverNeverEchoesPeerText is the canary for the credential +// boundary's logging half. +// +// Observer.Disconnected is a logging surface, and both of its arguments can be +// fed by the peer: a raw disconnect frame's reason, and a WebSocket close +// reason rendered through the error. §23 declares the ticket an "opaque bearer +// credential; never logged", and the cable server is exactly who knows it — it +// was dialed with it. A server that echoes the URL it was dialed with, whether +// maliciously or by logging its own request line into an error, puts the ticket +// in the host's logs. +// +// Truncation was the old defense and does not work: it bounds the length of a +// leak, not whether there is one. This asserts the property instead of the +// mechanism, so it stays true whatever the rendering becomes — the canary is +// planted in every peer-controlled string the teardown path can carry. +func TestDisconnectObserverNeverEchoesPeerText(t *testing.T) { + const canary = "sekrit-ticket-value" + peerText := "wss://28.cable.basecamp.com/cable?ticket=" + canary + + for _, tc := range []struct { + name string + //nolint:revive // the fixture drives one teardown shape per case + drive func(t *testing.T, h *harness, sock *feedtest.Conn) + }{ + {"raw disconnect frame reason", func(_ *testing.T, _ *harness, sock *feedtest.Conn) { + sock.Serve(frameDisconnect(peerText, false)) + }}, + {"peer close reason", func(_ *testing.T, _ *harness, sock *feedtest.Conn) { + sock.FailReads(&eventfeed.CloseError{Code: 1011, Reason: peerText}) + }}, + {"raw read error", func(_ *testing.T, _ *harness, sock *feedtest.Conn) { + sock.FailReads(errors.New("read tcp: " + peerText)) + }}, + {"invalid frame payload", func(_ *testing.T, _ *harness, sock *feedtest.Conn) { + sock.Serve([]byte(`{"type":"message","identifier":"` + peerText + `","message":{`)) + }}, + } { + t.Run(tc.name, func(t *testing.T) { + var mu sync.Mutex + var seen []string + h := newHarness(t, eventfeed.WithObserver(eventfeed.Observer{ + Disconnected: func(reason string, err error) { + mu.Lock() + defer mu.Unlock() + seen = append(seen, reason) + if err != nil { + // The whole chain, not just the top: Error() walks it. + seen = append(seen, err.Error()) + for e := err; e != nil; e = errors.Unwrap(e) { + seen = append(seen, e.Error()) + } + } + }, + })) + h.minter.ScriptTicket(ticket(1)) + h.polls.ScriptPage(eventfeed.PollPage{Position: "pos-1"}) + h.start() + sock := h.driveToSubscribed() + tc.drive(t, h, sock) + h.awaitTimer(timerBackoff) + + mu.Lock() + defer mu.Unlock() + if len(seen) == 0 { + t.Fatal("Observer.Disconnected never fired; the canary proves nothing") + } + for _, got := range seen { + if strings.Contains(got, canary) { + t.Errorf("Observer.Disconnected echoed the peer's ticket: %q", got) + } + if strings.Contains(got, "ticket=") { + t.Errorf("Observer.Disconnected echoed a ticket-bearing query: %q", got) + } + } + }) + } +} diff --git a/go/pkg/basecamp/eventfeed/seams.go b/go/pkg/basecamp/eventfeed/seams.go index d5f55c9700..270ebaded3 100644 --- a/go/pkg/basecamp/eventfeed/seams.go +++ b/go/pkg/basecamp/eventfeed/seams.go @@ -329,7 +329,9 @@ type CloseError struct { // comfortably, and §9's 500-byte cap bounds without redacting. So the // rendering withholds it behind a fixed marker, and with no unbounded input // left there is nothing to truncate. The type stays flat: no cause, nothing -// a chain walk recovers. +// a chain walk recovers. Reason remains a FIELD, so a host that has decided +// its cable server is trustworthy can read it deliberately — what changes is +// that the connector no longer puts it in front of every logger by default. func (e *CloseError) Error() string { if e.Reason != "" { return fmt.Sprintf("cable connection closed by peer: code %d (peer reason withheld)", e.Code) @@ -383,7 +385,9 @@ type DialError struct { // Err is the underlying cause. Never the dialed URL or an error that // renders it — the ticket rides in its query string, and url.Error // renders the full URL. The built-in transport stores only causes - // flattened to a closed vocabulary (dialFailure). + // flattened to a closed vocabulary (dialFailure); the connector treats + // every seam-returned value as untrusted regardless (it reads Kind and + // nothing else). Err error } From ba82047f8d16043be43ddd9d153ff9caf6fab7d7 Mon Sep 17 00:00:00 2001 From: Jeremy Daer Date: Tue, 18 Aug 2026 18:37:58 -0700 Subject: [PATCH 07/51] Event feed: three docs that outran the code they describe All three are documentation-accuracy fixes with no behaviour change; the verification is that each comment now matches the code beside it. Observer.Disconnected still promised that err is "passed through UNREDACTED". That stopped being true when the ticket-leak fix routed both arguments through closed vocabularies: observableDisconnectReason collapses every unrecognized peer reason to "other", and observableSocketError replaces every unrecognized error with errSocketFailed. The old text told a custom-transport author their diagnostics reach the observer, which is exactly backwards -- so the corollary is now stated outright rather than left to be discovered by an author whose error vanished. go/README.md claimed the connector "performs no wire I/O of its own" two sentences after describing the Action Cable subscription it dials. AGENTS.md and doc.go both name that dial as the one sanctioned non-HTTP wire act; doc.go's version of the sentence already carried the qualification and the README's did not. Scoped to HTTP API I/O, with the dial named. durableGate's opening paragraph promises to prevent a replacement connector overwriting a lineage, and "begun" there is the gate's term of art for claiming the gate -- defined twenty lines below the sentence that leans on it. A reviewer read it the other way, which is the evidence it needed saying. The residual window is real and narrowed rather than closed: claim and store call are not atomic, so a descheduled save can still land behind a replacement and move a checkpoint backward. Closing it needs Close to wait (a deadlock from any callback) or a fencing token on CheckpointStore (a six-SDK contract change), so it is stated here and tracked in #784. --- go/README.md | 12 ++++---- go/pkg/basecamp/eventfeed/connector.go | 22 +++++++++++++++ go/pkg/basecamp/eventfeed/seams.go | 39 ++++++++++++++++++-------- 3 files changed, 57 insertions(+), 16 deletions(-) diff --git a/go/README.md b/go/README.md index a246790525..a4fdb5121a 100644 --- a/go/README.md +++ b/go/README.md @@ -788,11 +788,13 @@ stream of events; the connector owns reconnection, backoff, staleness detection, the durable position. **Experimental: the Layer-1 seam adapters have not landed yet.** The connector performs -no wire I/O of its own — every HTTP exchange reaches the wire through a seam backed by -a generated operation — and the adapters that build those seams over the generated -`CreateStreamTicket` and `PollEvents` operations are still to come. Until they do, a -consumer must supply the `TicketMinter` and `PollSource` implementations itself, and the -exported surface may still change as they land. +no HTTP API I/O of its own: every HTTP exchange reaches the wire through a seam backed by +a generated operation. Its one direct wire act is the Action Cable dial above — the +connector connects verbatim to the URL a generated `CreateStreamTicket` call returned, +which is the sanctioned non-HTTP wire act. The adapters that build those seams over the +generated `CreateStreamTicket` and `PollEvents` operations are still to come. Until they +do, a consumer must supply the `TicketMinter` and `PollSource` implementations itself, and +the exported surface may still change as they land. ```go import ( diff --git a/go/pkg/basecamp/eventfeed/connector.go b/go/pkg/basecamp/eventfeed/connector.go index 429a023641..2458105411 100644 --- a/go/pkg/basecamp/eventfeed/connector.go +++ b/go/pkg/basecamp/eventfeed/connector.go @@ -236,6 +236,28 @@ type Connector struct { // was accepted and delivered before Close was called, and abandoning a durable // write half-way is worse than finishing it. What cannot happen is a save the // connector had not yet decided to make. +// +// # The residual, stated rather than left to be rediscovered +// +// "Begun" in the opening paragraph is this gate's term of art — claiming the +// gate — and the failure described there is prevented only under that reading. +// Say so plainly, because a reader who takes "begun" to mean the store call +// reads the opening as a stronger promise than the gate delivers. +// +// The claim and the store call are not atomic together: a save can be +// descheduled between them for as long as the scheduler likes. So the window +// in which a replacement connector's newer position can be overwritten is +// NARROWED, from [decision, write] to [claim, write], rather than closed. The +// overwrite moves the checkpoint BACKWARD, against §23's "checkpoints only +// move forward"; its cost is bounded replay on the next entry rather than +// skipped events, because the position that survives is the older one. +// +// Closing it needs one of two things this type cannot have: Close waiting on +// the store call (rejected above, and reachable as a deadlock from any +// callback), or a fencing token the store can use to refuse a write from a +// superseded run — a CheckpointStore contract change normative for all six +// SDKs. Tracked in #784 with a third option, declaring the residual +// in-contract, rather than decided here. type durableGate struct { mu sync.Mutex closed bool diff --git a/go/pkg/basecamp/eventfeed/seams.go b/go/pkg/basecamp/eventfeed/seams.go index 270ebaded3..164516da1d 100644 --- a/go/pkg/basecamp/eventfeed/seams.go +++ b/go/pkg/basecamp/eventfeed/seams.go @@ -476,18 +476,35 @@ type Observer struct { Connected func() // Confirmed fires on confirm_subscription. Confirmed func() - // Disconnected fires when a socket is torn down. reason is the - // frame-derived disconnect reason, bounded by the §9 cap; err is the - // failure that ended the socket. + // Disconnected fires when a socket is torn down. reason describes why; err + // is the failure that ended the socket. NEITHER is peer text: both are + // mapped onto closed vocabularies before they reach here. // - // err is passed through UNREDACTED, deliberately: it is an error, i.e. - // opaque text, and stripping a credential out of arbitrary text requires - // modelling the credential — the one thing §23's "opaque bearer" contract - // rules out. Every error the connector puts here is either one of its own - // sentinels or a seam error, and Dial, ReadFrame and WriteFrame each carry - // the obligation not to render the cable URL. A custom transport that - // breaks that obligation leaks through this callback, which is why the - // obligation is stated on the seam rather than defended here. + // reason goes through observableDisconnectReason: the two reasons dispatch + // actually acts on are named exactly, an absent reason stays empty, and + // every other peer-supplied string — however short — becomes "other". err + // goes through observableSocketError: the connector's own sentinels and + // typed errors are preserved (errStaleConnection, errCableConnClosed, the + // context sentinels, *CloseError, *DialError, *TerminalError, + // *invalidFrameError), and ANY other error is replaced by errSocketFailed. + // + // The normalization is deliberate, and it is the connector's own defense + // rather than a restatement of the seam's. §23 declares the ticket an + // opaque bearer credential that is never logged; this callback is a + // logging surface; and stripping a credential out of arbitrary text would + // require MODELLING the credential, which "opaque" is precisely the + // assumption that forbids. A closed vocabulary is the only answer that + // does not depend on the peer, or on a custom transport, behaving. + // + // Dial, ReadFrame and WriteFrame still carry the obligation not to render + // the cable URL, and that obligation is what keeps a PRESERVED typed error + // safe. What changed is the consequence of breaking it: a transport that + // renders the URL into an unrecognized error no longer leaks through this + // callback, because that error does not survive the mapping. + // + // The corollary for custom transports: diagnostics you attach to an error + // the connector does not recognize do NOT reach this callback. Report them + // through your own logging, not through this seam. Disconnected func(reason string, err error) // CatchUpStarted fires when a poll walk begins. CatchUpStarted func(cursor Cursor) From ec5f2eaec39e8308235eee69a25ed31204a88ac6 Mon Sep 17 00:00:00 2001 From: Jeremy Daer Date: Tue, 18 Aug 2026 18:58:52 -0700 Subject: [PATCH 08/51] Event feed: a wrapped sentinel is not a sentinel to log observableSocketError exists so the connector does not DEPEND on a custom transport honoring the seam's no-cable-URL obligation. For four of its arms it did depend on exactly that. Matching a sentinel is not the same as being one. errors.Is walks the chain and reports; it does not extract. So a transport returning fmt.Errorf("read %s: %w", cableURL, context.Canceled) matched the recognized arm, and returning the argument handed the wrapper -- ticket and all -- to Observer.Disconnected, which is a logging surface. The credential rides in the URL the peer was dialed with, so the one place text is most likely to be annotated is the one place the vocabulary was open. The typed arms below were already safe by a different mechanism: errors.As assigns the inner value, so returning it drops the wrapper. That asymmetry is why the two halves look different and must, and the test pins the As half too -- a refactor unifying both onto errors.Is would silently reopen this. Each arm now returns the canonical sentinel rather than its argument. errors.Is still matches for a consumer that branches on it; what does not survive is the wrapper's text. Red-proven with only the switch reverted: all four sentinel cases FAIL with the literal ticket in the message, while the wrapped-CloseError case PASSES against un-fixed code -- so the test is not vacuously red, and it discriminates the two mechanisms rather than asserting a blanket property. Also corrected, wrong in the same direction: Observer.Disconnected's godoc had just been rewritten to claim the vocabulary needs nothing of a custom transport, and the PR description still said the callback was "audited and deliberately not redacted". Both now describe what the code does. --- go/pkg/basecamp/eventfeed/export_test.go | 18 ++++++ go/pkg/basecamp/eventfeed/loop.go | 29 +++++++-- go/pkg/basecamp/eventfeed/loop_test.go | 75 ++++++++++++++++++++++++ go/pkg/basecamp/eventfeed/seams.go | 6 ++ 4 files changed, 123 insertions(+), 5 deletions(-) diff --git a/go/pkg/basecamp/eventfeed/export_test.go b/go/pkg/basecamp/eventfeed/export_test.go index dda9c9619a..2d85d85ab5 100644 --- a/go/pkg/basecamp/eventfeed/export_test.go +++ b/go/pkg/basecamp/eventfeed/export_test.go @@ -104,6 +104,24 @@ func ExportIsInvalidFrameError(err error) bool { return errors.As(err, &ife) } +// ExportObservableSocketError exposes the teardown-cause reduction that guards +// Observer.Disconnected's error argument. Tested directly rather than only +// through a socket because the leak it must prevent is in the SHAPE of the +// error handed to it — a seam error WRAPPING a sentinel — and driving every +// such shape through a live teardown proves less about the mapping than +// calling it does. +func ExportObservableSocketError(err error) error { return observableSocketError(err) } + +// ExportStaleConnectionErr returns the connector's staleness sentinel, for +// asserting that a wrapped one reduces to the sentinel itself rather than to +// its wrapper. A func rather than a var so the errname linter does not read a +// test-only bridge as a package sentinel needing the ErrXxx spelling. +func ExportStaleConnectionErr() error { return errStaleConnection } + +// ExportCableConnClosedErr returns the connector's socket-closed sentinel, +// exposed for the same reason as ExportStaleConnectionErr. +func ExportCableConnClosedErr() error { return errCableConnClosed } + // ExportSubscribeIdentifier exposes the exact EventsChannel subscription // identifier for frame construction in tests. func ExportSubscribeIdentifier(f Filters) string { return subscribeIdentifier(f) } diff --git a/go/pkg/basecamp/eventfeed/loop.go b/go/pkg/basecamp/eventfeed/loop.go index 4898abe3e2..9deebf11ad 100644 --- a/go/pkg/basecamp/eventfeed/loop.go +++ b/go/pkg/basecamp/eventfeed/loop.go @@ -1276,15 +1276,34 @@ var errSocketFailed = errors.New("event feed socket failed") // "leaks". The typed values the connector needs to ACT on are matched before // this is ever called — dispatch reads them directly — so nothing here changes // a verdict. +// +// # Every arm returns a value the CONNECTOR owns, never the argument +// +// Matching a sentinel is not the same as being one. errors.Is walks the chain, +// so a seam that returns fmt.Errorf("read %s: %w", cableURL, context.Canceled) +// MATCHES here while its text carries the ticket — and returning err would hand +// that wrapper to Observer.Disconnected verbatim, defeating the whole closed +// vocabulary for precisely the errors most likely to be wrapped. So each arm +// returns the canonical sentinel rather than the argument. errors.Is still +// matches for a consumer that checks, because the sentinel IS what it would +// have matched; what does not survive is the wrapper's text. +// +// The typed arms below are already safe for the same reason by a different +// mechanism: errors.As assigns the INNER typed value, so returning it drops any +// wrapper. That asymmetry — As extracts, Is only reports — is why the two +// halves of this function look different and must. func observableSocketError(err error) error { switch { case err == nil: return nil - case errors.Is(err, errStaleConnection), - errors.Is(err, errCableConnClosed), - errors.Is(err, context.Canceled), - errors.Is(err, context.DeadlineExceeded): - return err + case errors.Is(err, errStaleConnection): + return errStaleConnection + case errors.Is(err, errCableConnClosed): + return errCableConnClosed + case errors.Is(err, context.Canceled): + return context.Canceled + case errors.Is(err, context.DeadlineExceeded): + return context.DeadlineExceeded } var ce *CloseError if errors.As(err, &ce) { diff --git a/go/pkg/basecamp/eventfeed/loop_test.go b/go/pkg/basecamp/eventfeed/loop_test.go index 021bef1688..9d0ea45812 100644 --- a/go/pkg/basecamp/eventfeed/loop_test.go +++ b/go/pkg/basecamp/eventfeed/loop_test.go @@ -1813,6 +1813,81 @@ func TestStalenessArmsBeforeConnectedObserver(t *testing.T) { // leak, not whether there is one. This asserts the property instead of the // mechanism, so it stays true whatever the rendering becomes — the canary is // planted in every peer-controlled string the teardown path can carry. +// TestObservableSocketErrorDropsWrapperText pins the half of the closed +// vocabulary that errors.Is cannot enforce on its own. +// +// TestDisconnectObserverNeverEchoesPeerText covers the shapes a peer controls +// directly. This covers the one a conforming-looking SEAM controls: a transport +// that annotates its read failure with the URL it was reading — +// fmt.Errorf("read %s: %w", cableURL, context.Canceled) — produces an error that +// MATCHES a recognized sentinel while carrying a ticket in its text. Returning +// the argument on a match would forward that wrapper to Observer.Disconnected, +// so every arm must return the connector's own value instead. +// +// The typed arms are included to prove the same property holds through the +// other mechanism (errors.As extracting the inner value), because a future +// refactor that unified the two halves onto errors.Is would silently reopen +// exactly this hole. +func TestObservableSocketErrorDropsWrapperText(t *testing.T) { + const canary = "sekrit-ticket-value" + cableURL := "wss://28.cable.basecamp.com/cable?ticket=" + canary + + for _, tc := range []struct { + name string + err error + want error + }{ + {"wrapped context.Canceled", fmt.Errorf("read %s: %w", cableURL, context.Canceled), context.Canceled}, + { + "wrapped context.DeadlineExceeded", + fmt.Errorf("read %s: %w", cableURL, context.DeadlineExceeded), + context.DeadlineExceeded, + }, + { + "wrapped staleness sentinel", + fmt.Errorf("read %s: %w", cableURL, eventfeed.ExportStaleConnectionErr()), + eventfeed.ExportStaleConnectionErr(), + }, + { + "wrapped conn-closed sentinel", + fmt.Errorf("read %s: %w", cableURL, eventfeed.ExportCableConnClosedErr()), + eventfeed.ExportCableConnClosedErr(), + }, + { + "wrapped CloseError", + fmt.Errorf("read %s: %w", cableURL, &eventfeed.CloseError{Code: 1011, Reason: cableURL}), + nil, // typed arm: identity is not pinned, only the absence of the canary + }, + } { + t.Run(tc.name, func(t *testing.T) { + got := eventfeed.ExportObservableSocketError(tc.err) + if got == nil { + t.Fatal("observableSocketError returned nil for a non-nil cause") + } + // The whole chain, not just the top: Error() walks it, and so does + // any observer that logs %+v. + for e := got; e != nil; e = errors.Unwrap(e) { + if strings.Contains(e.Error(), canary) { + t.Errorf("reduced error echoed the ticket: %q", e.Error()) + } + if strings.Contains(e.Error(), "ticket=") { + t.Errorf("reduced error echoed a ticket-bearing query: %q", e.Error()) + } + } + // Recognition must survive the reduction, or a consumer that + // branches on the sentinel silently stops matching. + if tc.want != nil { + if got != tc.want { //nolint:errorlint // identity is the assertion + t.Errorf("reduced to %#v, want the canonical sentinel %#v", got, tc.want) + } + if !errors.Is(got, tc.want) { + t.Errorf("errors.Is no longer matches after reduction: %#v", got) + } + } + }) + } +} + func TestDisconnectObserverNeverEchoesPeerText(t *testing.T) { const canary = "sekrit-ticket-value" peerText := "wss://28.cable.basecamp.com/cable?ticket=" + canary diff --git a/go/pkg/basecamp/eventfeed/seams.go b/go/pkg/basecamp/eventfeed/seams.go index 164516da1d..1c69c89cf4 100644 --- a/go/pkg/basecamp/eventfeed/seams.go +++ b/go/pkg/basecamp/eventfeed/seams.go @@ -488,6 +488,12 @@ type Observer struct { // context sentinels, *CloseError, *DialError, *TerminalError, // *invalidFrameError), and ANY other error is replaced by errSocketFailed. // + // "Preserved" means reduced to the connector's OWN value, not passed + // along. A seam error that merely WRAPS a recognized sentinel — + // fmt.Errorf("read %s: %w", cableURL, context.Canceled) — reduces to the + // bare sentinel, because the wrapper's text is exactly where a cable URL + // would ride. errors.Is still matches for a consumer that checks. + // // The normalization is deliberate, and it is the connector's own defense // rather than a restatement of the seam's. §23 declares the ticket an // opaque bearer credential that is never logged; this callback is a From 929860cdb6a2832d1fd17775b00597fe7c90e1f4 Mon Sep 17 00:00:00 2001 From: Jeremy Daer Date: Tue, 18 Aug 2026 19:08:35 -0700 Subject: [PATCH 09/51] Event feed: Dial's error does not reach the callback its contract named CableTransport.Dial told implementers its returned error "reaches Observer.Disconnected, which hosts log". It does not, and cannot: a dial failure has no socket, so there is no teardown to report. DialPolicy becomes Terminal(invalid_cable_url); every other kind takes transition 7 to backoff, where the connector reports the classification and drops the cause. Left alone, that reads as a licence to relax -- the callback never fires, so why redact? The real exposure runs the other way and is stronger. A DialPolicy error is yielded as the ITERATION's terminal error: the consumer's own error value, which they cannot decline to receive, where a callback is something they may never have registered. So the no-credential obligation is not softened by the correction, it is what keeps a ticket out of the error a caller gets back from Events. ReadFrame's identically worded claim is left as it stands, because for a read error it is true. Fourth of these today -- prose that was accurate when written and falsified by a later, correct change. Noted on the PR as a pattern rather than filed: this package states unusually precise contracts in comments and nothing checks them, so each correct fix quietly ages the paragraph above it. --- go/pkg/basecamp/eventfeed/seams.go | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/go/pkg/basecamp/eventfeed/seams.go b/go/pkg/basecamp/eventfeed/seams.go index 1c69c89cf4..cfb871180e 100644 --- a/go/pkg/basecamp/eventfeed/seams.go +++ b/go/pkg/basecamp/eventfeed/seams.go @@ -270,13 +270,23 @@ type CableTransport interface { // is no unlimited mode. // // The returned error MUST NOT render wsURL or any part of its query - // string. The ticket rides in that query, and this error reaches - // Observer.Disconnected, which hosts log. An error is opaque text, so + // string. The ticket rides in that query, and an error is opaque text, so // nothing downstream can redact it — see WebSocketTransport's dialFailure // on why stripping a credential out of arbitrary text requires modelling // the credential, which §23's "opaque bearer" contract forbids. Report // the classification and a cause of your own choosing, never the peer's // or a library's rendering of the URL. + // + // Where it is exposed, stated exactly, because the obvious guess is wrong: + // a dial failure does NOT reach Observer.Disconnected. There is no socket + // yet, so there is no teardown to report. A DialPolicy failure becomes + // Terminal(invalid_cable_url) and is yielded as the iteration's terminal + // error — the consumer's own error value, which is a stronger exposure + // than a callback, not a weaker one. Every other kind takes transition 7 + // to backoff, where the connector reports the classification and drops the + // cause entirely. So the obligation above is not softened by the callback + // never firing; it is what keeps a credential out of the error a caller + // receives from Events. Dial(ctx context.Context, wsURL string, maxFrameBytes int64) (CableConn, error) } From ab0e33c61904afc820d8f10ddb801df743e71f6a Mon Sep 17 00:00:00 2001 From: Jeremy Daer Date: Tue, 18 Aug 2026 19:20:12 -0700 Subject: [PATCH 10/51] Event feed: the tier-2 driver could not fail the terminal contract it enforces MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit SPEC §23 gives the feed exactly one final error element, and that element ends iteration. The driver checked neither half. It recorded a terminal and kept ranging, overwriting `terminal` each time, so three non-conforming shapes passed every terminal fixture identically to a compliant connector: - yielding the expected terminal reason TWICE (the second overwrote the first with an equal value, so every later assertion still passed); - delivering an event AFTER the error element; - never ending iteration at all -- the deferred cancel ended the range afterwards, so nothing downstream could tell an iteration that ended by itself from one that had to be cancelled. None of the three is observable from an end-state assertion, which is why the gap survived: the fixtures in the tree pass either way. A test that cannot fail the contract it claims to enforce is worse than no test, because it is counted as coverage. The count and the post-terminal delivery are recorded as violations, which block every subsequent await, so a scenario fails at its next step rather than silently at the end. The first element is retained rather than the latest -- "exactly one final element" means the first one is the real one. Requiring the iteration to have ended is a driver-side await, bounded by the same watchdog as every other rendezvous, so a connector that stays open fails instead of hanging. Red-proven per case: with the three guards reverted, all three hostile shapes report "the driver accepted ..." and the compliant shape still passes, so the probes discriminate rather than failing on everything. All three were reachable; none had to be dropped as unreachable. Found by Copilot as a suppressed comment, which is the second time today that block carried the sharper finding. --- .../eventfeed/scenario_conformance_test.go | 18 ++++ .../eventfeed/scenario_harness_test.go | 31 +++++- .../eventfeed/scenario_selftest_test.go | 95 +++++++++++++++++++ 3 files changed, 143 insertions(+), 1 deletion(-) diff --git a/go/pkg/basecamp/eventfeed/scenario_conformance_test.go b/go/pkg/basecamp/eventfeed/scenario_conformance_test.go index 9864b8ff96..4a78d28518 100644 --- a/go/pkg/basecamp/eventfeed/scenario_conformance_test.go +++ b/go/pkg/basecamp/eventfeed/scenario_conformance_test.go @@ -583,6 +583,16 @@ func (d *driver) awaitTerminal(reason string) error { }) } +// awaitIterationEnded waits for the consumer's range over Events to return of +// its own accord — the observable half of "the error element ends iteration". +// Bounded by the same watchdog as every other await, so a connector that stays +// open fails here instead of hanging the scenario. +func (d *driver) awaitIterationEnded() error { + return d.h.await("the iteration to end on its final error element", func() (bool, string) { + return d.h.iterDone, "" + }) +} + func (d *driver) expectGap(epochAfterID int64) error { return d.h.await(fmt.Sprintf("Observer.gap(%d)", epochAfterID), func() (bool, string) { if d.h.gapsTaken >= len(d.h.gaps) { @@ -710,6 +720,14 @@ func (d *driver) runFinally() error { if err := d.awaitTerminal(fin.Error.Reason); err != nil { return fmt.Errorf("finally.error: %w", err) } + // The element is only the FINAL one if iteration actually ended on it. + // Without this, a connector that yields the right terminal and then + // stays open passes every terminal fixture: the deferred cancel ends + // the range afterwards, so nothing downstream can tell an iteration + // that ended by itself from one that had to be cancelled. + if err := d.awaitIterationEnded(); err != nil { + return fmt.Errorf("finally.error: %w", err) + } } else if terminal := d.h.terminalError(); terminal != nil { return fmt.Errorf("finally: the feed terminated, but the fixture states no error: %w", terminal) } diff --git a/go/pkg/basecamp/eventfeed/scenario_harness_test.go b/go/pkg/basecamp/eventfeed/scenario_harness_test.go index 4c0e386435..331c4ff992 100644 --- a/go/pkg/basecamp/eventfeed/scenario_harness_test.go +++ b/go/pkg/basecamp/eventfeed/scenario_harness_test.go @@ -106,6 +106,7 @@ type scenarioHarness struct { invalidFramesTaken int state string terminal *eventfeed.TerminalError + terminalCount int iterDone bool violations []string } @@ -433,10 +434,31 @@ func (h *scenarioHarness) writeFrame(peer *cablePeer, data []byte) error { func (h *scenarioHarness) recordDelivered(id int64) { h.mu.Lock() defer h.mu.Unlock() + // The error element is FINAL: nothing may be yielded after it. Recorded as + // a violation rather than dropped, so the delivered ledger still shows what + // arrived and the failure names the ordering rather than an exact-set + // mismatch that reads as a missing event. + if h.terminalCount > 0 { + h.violations = append(h.violations, fmt.Sprintf( + "the feed delivered event %d after its error element (%v); §23's final error element ends iteration", + id, h.terminal)) + } h.delivered = append(h.delivered, id) h.notifyLocked() } +// recordTerminal records one error element yielded by the iteration. +// +// §23 gives the feed EXACTLY ONE final error element, and it ends iteration. +// Both halves are enforced here rather than assumed, because neither is +// observable from an end-state assertion: keeping only the latest terminal — +// which this did — silently accepts a connector that yields the expected +// reason twice, since the second overwrites the first with an equal value and +// every later assertion still passes. +// +// The FIRST element is retained. A second is a violation naming both, and +// violations block every subsequent await, so the scenario fails at its next +// step rather than at the end. func (h *scenarioHarness) recordTerminal(err error) { h.mu.Lock() defer h.mu.Unlock() @@ -444,7 +466,14 @@ func (h *scenarioHarness) recordTerminal(err error) { if !errors.As(err, &terr) { h.violations = append(h.violations, fmt.Sprintf("the feed yielded a non-terminal error element: %v", err)) } else { - h.terminal = terr + h.terminalCount++ + if h.terminalCount > 1 { + h.violations = append(h.violations, fmt.Sprintf( + "the feed yielded %d error elements (%v after %v); §23 gives it exactly one, and it ends iteration", + h.terminalCount, terr, h.terminal)) + } else { + h.terminal = terr + } } h.notifyLocked() } diff --git a/go/pkg/basecamp/eventfeed/scenario_selftest_test.go b/go/pkg/basecamp/eventfeed/scenario_selftest_test.go index 5da9cc5881..55a33ee2dc 100644 --- a/go/pkg/basecamp/eventfeed/scenario_selftest_test.go +++ b/go/pkg/basecamp/eventfeed/scenario_selftest_test.go @@ -16,6 +16,8 @@ import ( "strings" "testing" "time" + + "github.com/basecamp/basecamp-sdk/go/pkg/basecamp/eventfeed" ) // TestScenarioDriverRejectsMutatedFixtures is the driver's kill test: a @@ -376,3 +378,96 @@ func descend(t *testing.T, cursor any, segment, path string) any { return nil } } + +// TestScenarioDriverEnforcesTheFinalErrorElement pins the three halves of +// §23's "exactly one final error element that ends iteration" that no +// end-state assertion can catch. +// +// The driver used to record a terminal and `continue` ranging, keeping only +// the LATEST element. So a connector that yielded the expected reason twice, +// or delivered an event after it, or never ended iteration at all, passed +// every terminal fixture identically to a compliant one — the defining +// property of a test that cannot fail the contract it enforces. +// +// Each case drives the harness's own record path, which is where the guard +// lives, and asserts through `await` — the same call every expect step makes, +// and the one that consults recorded violations. That is what ties these +// guards to scenario failure rather than to a flag nobody reads. +func TestScenarioDriverEnforcesTheFinalErrorElement(t *testing.T) { + terminal := func(reason eventfeed.TerminalReason) error { + return &eventfeed.TerminalError{Reason: reason, Msg: "scenario selftest"} + } + // A check that always passes: any failure returned by await is therefore + // the recorded violation, never the predicate. + passes := func() (bool, string) { return true, "" } + + t.Run("a second error element is rejected", func(t *testing.T) { + h := newScenarioHarness() + defer h.close() + h.recordTerminal(terminal(eventfeed.ReasonProtocolFatal)) + h.recordTerminal(terminal(eventfeed.ReasonProtocolFatal)) + + err := underShortWatchdog(func() error { return h.await("anything", passes) }) + if err == nil { + t.Fatal("the driver accepted two error elements; §23 gives the feed exactly one") + } + if !strings.Contains(err.Error(), "exactly one") { + t.Fatalf("failed for the wrong reason: %v", err) + } + }) + + t.Run("a delivery after the error element is rejected", func(t *testing.T) { + h := newScenarioHarness() + defer h.close() + h.recordTerminal(terminal(eventfeed.ReasonProtocolFatal)) + h.recordDelivered(42) + + err := underShortWatchdog(func() error { return h.await("anything", passes) }) + if err == nil { + t.Fatal("the driver accepted an event delivered after the final error element") + } + if !strings.Contains(err.Error(), "after its error element") { + t.Fatalf("failed for the wrong reason: %v", err) + } + }) + + t.Run("an iteration that never ends is rejected", func(t *testing.T) { + h := newScenarioHarness() + defer h.close() + const state = "terminal" + h.recordState(state) + h.recordTerminal(terminal(eventfeed.ReasonProtocolFatal)) + // Deliberately no recordIterDone: the connector yielded the right + // element and stayed open. + + d := &driver{h: h, sc: &scenario{Finally: scenarioFinally{ + State: state, + Error: &errorExpect{Reason: string(eventfeed.ReasonProtocolFatal)}, + }}} + err := underShortWatchdog(d.runFinally) + if err == nil { + t.Fatal("the driver accepted a terminal scenario whose iteration never ended") + } + if !strings.Contains(err.Error(), "end on its final error element") { + t.Fatalf("failed for the wrong reason: %v", err) + } + }) + + t.Run("the compliant shape still passes", func(t *testing.T) { + h := newScenarioHarness() + defer h.close() + const state = "terminal" + h.recordState(state) + h.recordDelivered(41) + h.recordTerminal(terminal(eventfeed.ReasonProtocolFatal)) + h.recordIterDone() + + d := &driver{h: h, sc: &scenario{Finally: scenarioFinally{ + State: state, + Error: &errorExpect{Reason: string(eventfeed.ReasonProtocolFatal)}, + }}} + if err := underShortWatchdog(d.runFinally); err != nil { + t.Fatalf("the compliant shape must pass, else the three probes above prove nothing: %v", err) + } + }) +} From 7380f3f7ca46ac5f423aad9d688a4d2c9568716c Mon Sep 17 00:00:00 2001 From: Jeremy Daer Date: Tue, 18 Aug 2026 22:36:11 -0700 Subject: [PATCH 11/51] Event feed: two of the "connector's own" errors are not the connector's to write MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `observableSocketError` exists so that `Observer.Disconnected` — a logging surface whose destination the connector knows nothing about — carries only values the connector authored. Recognition is by type, which is only as strong as the type's authorship, and two arms failed that test. `*DialError` and `*TerminalError` are both exported with exported fields, and both render free text: DialError its `Reason` plus `Err.Error()`, TerminalError its `Msg` plus `Err.Error()`. `CableTransport` is a documented extension point, so a `CableConn.ReadFrame` returning `&TerminalError{Msg: cableURL}` reached the callback verbatim — the ticket rides in that URL's query string, and §23 calls the ticket an opaque bearer credential that is never logged. Neither passes through now. The three arms that remain are safe by construction rather than by convention: the sentinels are package-level values whose text is fixed in this file, `*CloseError` renders an integer code alone, and `*invalidFrameError` renders one of two shape constants. Nothing is lost on the live path — dial failures never reach this function, and a terminated feed's reason and detail arrive through the iteration's terminal element, which is the semantic channel and keeps its SPEC-mandated wording. `seams.go`'s contract said the removed types were preserved; it now states the closed vocabulary and points at the terminal element for the rest. The canary arm that should have caught this was a false negative three times over, and each fault alone was enough. `{"type":"message",...}` is not a correlated broadcast — that shape is the TYPELESS one — so it never reached `decodeMessageEvent`. The payload was truncated JSON, so it failed the envelope unmarshal first and the payload was never read. And the canary sat in `identifier`, which nothing renders. It now serves a well-formed typeless broadcast under the connector's own identifier with the canary in `created_at`, the one field whose decoder quotes its input back. Four arms added for the errors a seam can author, at both seams that produce them: ReadFrame for the two types, and WriteFrame for the subscribe write — which had to be armed before welcome, since that is when the write happens. The assertions gained an identity check, because "the canary is absent" is satisfied by a callback firing with a nil error, which is not the guarantee. Mutation confirms the whole set: restoring the two arms fails nine cases, and three of them fail ONLY on the identity assertion. --- go/pkg/basecamp/eventfeed/export_test.go | 6 ++ go/pkg/basecamp/eventfeed/loop.go | 32 ++++-- go/pkg/basecamp/eventfeed/loop_test.go | 130 +++++++++++++++++++++-- go/pkg/basecamp/eventfeed/seams.go | 16 ++- 4 files changed, 162 insertions(+), 22 deletions(-) diff --git a/go/pkg/basecamp/eventfeed/export_test.go b/go/pkg/basecamp/eventfeed/export_test.go index 2d85d85ab5..9238953b41 100644 --- a/go/pkg/basecamp/eventfeed/export_test.go +++ b/go/pkg/basecamp/eventfeed/export_test.go @@ -122,6 +122,12 @@ func ExportStaleConnectionErr() error { return errStaleConnection } // exposed for the same reason as ExportStaleConnectionErr. func ExportCableConnClosedErr() error { return errCableConnClosed } +// ExportSocketFailedErr returns the generic cause an unrecognized teardown is +// reported as — what a seam-authored error must reduce to. Exposed so the +// reduction can be asserted by IDENTITY: an assertion that merely checks the +// canary is absent passes for any of several wrong answers, including nil. +func ExportSocketFailedErr() error { return errSocketFailed } + // ExportSubscribeIdentifier exposes the exact EventsChannel subscription // identifier for frame construction in tests. func ExportSubscribeIdentifier(f Filters) string { return subscribeIdentifier(f) } diff --git a/go/pkg/basecamp/eventfeed/loop.go b/go/pkg/basecamp/eventfeed/loop.go index 9deebf11ad..1dce5b1d2f 100644 --- a/go/pkg/basecamp/eventfeed/loop.go +++ b/go/pkg/basecamp/eventfeed/loop.go @@ -1277,6 +1277,28 @@ var errSocketFailed = errors.New("event feed socket failed") // this is ever called — dispatch reads them directly — so nothing here changes // a verdict. // +// # "The connector's own" means the connector is the only possible author +// +// Two types used to pass through that fail that test. *DialError and +// *TerminalError are both EXPORTED with exported fields, so a seam can +// construct one, and both render free text — DialError its Reason plus +// Err.Error(), TerminalError its Msg plus Err.Error(). A CableConn.ReadFrame +// returning &TerminalError{Msg: cableURL} therefore reached +// Observer.Disconnected verbatim: recognition by type is only as strong as the +// type's authorship, and neither of those types is authored solely here. +// +// So neither passes through any more. The three that remain are safe by +// construction rather than by convention: the sentinels are package-level +// values whose text is fixed in this file, *CloseError renders an integer code +// and nothing else, and *invalidFrameError renders one of two shape constants. +// +// Nothing is lost on the live path. Dial failures do not reach this function +// at all — they are dispatched before any teardown is observed — and a +// TerminalError's reason and text reach the consumer through the iteration's +// terminal element, which is the semantic channel and keeps its SPEC-mandated +// wording (§23's filter_invalid preserves the server's message). This callback +// is a log, not that channel. +// // # Every arm returns a value the CONNECTOR owns, never the argument // // Matching a sentinel is not the same as being one. errors.Is walks the chain, @@ -1313,16 +1335,10 @@ func observableSocketError(err error) error { } var ife *invalidFrameError if errors.As(err, &ife) { + // Flat by construction and renders only its shape, over a closed + // two-value vocabulary. See invalidFrameError. return ife } - var de *DialError - if errors.As(err, &de) { - return de - } - var terr *TerminalError - if errors.As(err, &terr) { - return terr - } return errSocketFailed } diff --git a/go/pkg/basecamp/eventfeed/loop_test.go b/go/pkg/basecamp/eventfeed/loop_test.go index 9d0ea45812..005c65cc4b 100644 --- a/go/pkg/basecamp/eventfeed/loop_test.go +++ b/go/pkg/basecamp/eventfeed/loop_test.go @@ -383,12 +383,20 @@ func frameDisconnect(reason string, reconnect bool) []byte { } func frameMessage(identifier string, id int64) []byte { + return frameMessageCreatedAt(identifier, id, "2026-08-01T12:00:00Z") +} + +// frameMessageCreatedAt builds a correlated broadcast with a caller-chosen +// created_at. Well-formed JSON with no "type" key and the connector's own +// identifier, so it reaches decodeMessageEvent — which is the point when +// createdAt is deliberately unparseable. +func frameMessageCreatedAt(identifier string, id int64, createdAt string) []byte { payload := map[string]any{ "id": id, "kind": "message", "event_type": "message.created", "action": "created", - "created_at": "2026-08-01T12:00:00Z", + "created_at": createdAt, "bucket_id": 2, "creator_id": 3, "recording_id": 900, @@ -1858,6 +1866,37 @@ func TestObservableSocketErrorDropsWrapperText(t *testing.T) { fmt.Errorf("read %s: %w", cableURL, &eventfeed.CloseError{Code: 1011, Reason: cableURL}), nil, // typed arm: identity is not pinned, only the absence of the canary }, + // The two types a SEAM can author. Both are exported with exported + // fields and both render free text — DialError its Reason plus + // Err.Error(), TerminalError its Msg plus Err.Error() — so recognizing + // them by type recognized nothing about who wrote them. A + // CableConn.ReadFrame returning either put its text on this callback + // verbatim. They must now reduce to the generic sentinel. + { + "seam-constructed DialError", + &eventfeed.DialError{Kind: eventfeed.DialPolicy, Reason: cableURL}, + eventfeed.ExportSocketFailedErr(), + }, + { + "seam-constructed DialError with a cause", + &eventfeed.DialError{Kind: eventfeed.DialTransient, Err: errors.New("dial " + cableURL)}, + eventfeed.ExportSocketFailedErr(), + }, + { + "seam-constructed TerminalError", + &eventfeed.TerminalError{Reason: eventfeed.ReasonPollFailed, Msg: cableURL}, + eventfeed.ExportSocketFailedErr(), + }, + { + "seam-constructed TerminalError with a cause", + &eventfeed.TerminalError{Reason: eventfeed.ReasonMintFailed, Err: errors.New("mint " + cableURL)}, + eventfeed.ExportSocketFailedErr(), + }, + { + "seam error wrapping a TerminalError", + fmt.Errorf("read: %w", &eventfeed.TerminalError{Reason: eventfeed.ReasonPollFailed, Msg: cableURL}), + eventfeed.ExportSocketFailedErr(), + }, } { t.Run(tc.name, func(t *testing.T) { got := eventfeed.ExportObservableSocketError(tc.err) @@ -1896,28 +1935,78 @@ func TestDisconnectObserverNeverEchoesPeerText(t *testing.T) { name string //nolint:revive // the fixture drives one teardown shape per case drive func(t *testing.T, h *harness, sock *feedtest.Conn) + // beforeWelcome runs drive against a dialed-but-pre-welcome socket, + // for the one failure that has to be armed before the subscribe write. + beforeWelcome bool + // wantErr, when set, is the exact error Disconnected must carry. + wantErr error }{ - {"raw disconnect frame reason", func(_ *testing.T, _ *harness, sock *feedtest.Conn) { + {name: "raw disconnect frame reason", drive: func(_ *testing.T, _ *harness, sock *feedtest.Conn) { sock.Serve(frameDisconnect(peerText, false)) }}, - {"peer close reason", func(_ *testing.T, _ *harness, sock *feedtest.Conn) { + {name: "peer close reason", drive: func(_ *testing.T, _ *harness, sock *feedtest.Conn) { sock.FailReads(&eventfeed.CloseError{Code: 1011, Reason: peerText}) }}, - {"raw read error", func(_ *testing.T, _ *harness, sock *feedtest.Conn) { - sock.FailReads(errors.New("read tcp: " + peerText)) - }}, - {"invalid frame payload", func(_ *testing.T, _ *harness, sock *feedtest.Conn) { - sock.Serve([]byte(`{"type":"message","identifier":"` + peerText + `","message":{`)) + { + name: "raw read error", + drive: func(_ *testing.T, _ *harness, sock *feedtest.Conn) { + sock.FailReads(errors.New("read tcp: " + peerText)) + }, + wantErr: eventfeed.ExportSocketFailedErr(), + }, + // The event-decode shape, which is the ONLY frame path where a decoder + // quotes peer bytes back (time.Time's UnmarshalJSON does, verbatim). + // Reaching it takes three things the predecessor of this case got + // wrong, each of which alone made the canary unobservable: NO "type" + // key, because a correlated broadcast is the typeless shape and + // `{"type":"message"}` is merely an unrecognized type; the connector's + // OWN identifier, because a frame that fails correlation is dropped + // before decode; and WELL-FORMED JSON, because truncated bytes fail + // the envelope unmarshal first and never reach the payload. The canary + // also has to sit in a field something renders — it sat in + // `identifier`, which nothing does — so it goes in created_at. + {name: "invalid frame payload", drive: func(_ *testing.T, _ *harness, sock *feedtest.Conn) { + sock.Serve(frameMessageCreatedAt(noFilterIdentifier, 107, peerText)) }}, + // A seam authoring one of the two exported error types. Both render + // free text and neither is the connector's to write, so both must + // reduce to the generic cause before they reach the callback. + { + name: "seam-authored TerminalError from ReadFrame", + drive: func(_ *testing.T, _ *harness, sock *feedtest.Conn) { + sock.FailReads(&eventfeed.TerminalError{Reason: eventfeed.ReasonPollFailed, Msg: peerText}) + }, + wantErr: eventfeed.ExportSocketFailedErr(), + }, + { + name: "seam-authored DialError from ReadFrame", + drive: func(_ *testing.T, _ *harness, sock *feedtest.Conn) { + sock.FailReads(&eventfeed.DialError{Kind: eventfeed.DialPolicy, Reason: peerText}) + }, + wantErr: eventfeed.ExportSocketFailedErr(), + }, + // The subscribe write is the one WriteFrame the connector makes on this + // path, and it happens ON welcome — so this case has to arm the failure + // BEFORE welcome is served, which is what beforeWelcome buys. + { + name: "seam-authored TerminalError from WriteFrame", + drive: func(_ *testing.T, _ *harness, sock *feedtest.Conn) { + sock.FailWrites(&eventfeed.TerminalError{Reason: eventfeed.ReasonMintFailed, Msg: peerText}) + }, + beforeWelcome: true, + wantErr: eventfeed.ExportSocketFailedErr(), + }, } { t.Run(tc.name, func(t *testing.T) { var mu sync.Mutex var seen []string + var sawErrs []error h := newHarness(t, eventfeed.WithObserver(eventfeed.Observer{ Disconnected: func(reason string, err error) { mu.Lock() defer mu.Unlock() seen = append(seen, reason) + sawErrs = append(sawErrs, err) if err != nil { // The whole chain, not just the top: Error() walks it. seen = append(seen, err.Error()) @@ -1930,8 +2019,15 @@ func TestDisconnectObserverNeverEchoesPeerText(t *testing.T) { h.minter.ScriptTicket(ticket(1)) h.polls.ScriptPage(eventfeed.PollPage{Position: "pos-1"}) h.start() - sock := h.driveToSubscribed() - tc.drive(t, h, sock) + var sock *feedtest.Conn + if tc.beforeWelcome { + sock = h.liveConn() + tc.drive(t, h, sock) + sock.Serve(frameWelcome()) + } else { + sock = h.driveToSubscribed() + tc.drive(t, h, sock) + } h.awaitTimer(timerBackoff) mu.Lock() @@ -1947,6 +2043,20 @@ func TestDisconnectObserverNeverEchoesPeerText(t *testing.T) { t.Errorf("Observer.Disconnected echoed a ticket-bearing query: %q", got) } } + // Absence of the canary is satisfied by a callback that fires with + // a nil err, which is not the guarantee. Where the reduction has a + // determinate answer, pin it by identity. + if tc.wantErr != nil { + found := false + for _, err := range sawErrs { + if err == tc.wantErr { //nolint:errorlint // identity is the assertion + found = true + } + } + if !found { + t.Errorf("Disconnected never carried the reduced cause %#v; saw %#v", tc.wantErr, sawErrs) + } + } }) } } diff --git a/go/pkg/basecamp/eventfeed/seams.go b/go/pkg/basecamp/eventfeed/seams.go index cfb871180e..9e4bfcff36 100644 --- a/go/pkg/basecamp/eventfeed/seams.go +++ b/go/pkg/basecamp/eventfeed/seams.go @@ -493,10 +493,13 @@ type Observer struct { // reason goes through observableDisconnectReason: the two reasons dispatch // actually acts on are named exactly, an absent reason stays empty, and // every other peer-supplied string — however short — becomes "other". err - // goes through observableSocketError: the connector's own sentinels and - // typed errors are preserved (errStaleConnection, errCableConnClosed, the - // context sentinels, *CloseError, *DialError, *TerminalError, - // *invalidFrameError), and ANY other error is replaced by errSocketFailed. + // goes through observableSocketError, whose vocabulary is CLOSED and + // short: the staleness and conn-closed sentinels, the two context + // sentinels, *CloseError (which renders its integer code alone), and + // *invalidFrameError (which renders one of two shape constants). ANY other + // error — including a *DialError or *TerminalError, both of which a seam + // can construct and both of which render free text — becomes + // errSocketFailed. // // "Preserved" means reduced to the connector's OWN value, not passed // along. A seam error that merely WRAPS a recognized sentinel — @@ -504,6 +507,11 @@ type Observer struct { // bare sentinel, because the wrapper's text is exactly where a cable URL // would ride. errors.Is still matches for a consumer that checks. // + // This is a LOGGING surface, and its vocabulary is chosen accordingly. A + // terminated feed's reason and detail reach you through the iteration's + // terminal *TerminalError, which is the semantic channel and is not + // reduced. + // // The normalization is deliberate, and it is the connector's own defense // rather than a restatement of the seam's. §23 declares the ticket an // opaque bearer credential that is never logged; this callback is a From b705ac186e9f4c224ec88541f38e9e1949109506 Mon Sep 17 00:00:00 2001 From: Jeremy Daer Date: Tue, 18 Aug 2026 22:49:06 -0700 Subject: [PATCH 12/51] Event feed: the fatal verdict was only probed for where the drain looks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #760 established that a protocol-fatal disconnect the pump has ALREADY READ must outrank whatever non-fatal outcome happens to be parked ahead of it, and fixed that inside the drain. The obligation is not the drain's. SPEC §23 says so twice — "terminal from EVERY socket-open state", and "a raw invalid_event_stream_command frame READ in AwaitingWelcome, CatchingUp, or Draining ... never Backoff. An explicitly non-retryable protocol rejection must not reconnect from any state." READ is the boundary the obligation attaches to. The walk has four points where it disposes of a deferred receive, and all four dispatched with the pump's queue never inspected. A `remote` disconnect in the slot — recoverable, arrived first — beat a fatal sitting right behind it: the cycle ended as an ordinary socket failure and the connector re-minted against a server that had already refused the command. Confirmed at every boundary before the fix, by mint count rather than by terminal reason alone, since a connector that reconnects once and then terminates on the re-issued frame still ends at protocol_fatal. `drainScan` becomes `fatalScan`, called at all four boundaries through `probeFatal` with a fresh pumpDepth+1 budget. That bound is the drain's, and for the drain's reason: the queue holds at most pumpDepth, the pump may be blocked in hand-off with one more, and the ownership cut's liveBufferCapacity is caller-configurable down to 1 — one queued ping under WithLiveBufferCapacity(1) spends an entire admission pass, which is what the page-boundary case reproduces. What the scan does with what it dequeues is what makes it safe mid-walk: live events are admitted to the buffer, which is what CatchingUp does with them anyway; the FIRST non-fatal socket outcome goes to the same single slot the caller is about to dispatch from, preserving arrival order for the outcome actually reported; and later ones are discarded, which is all that could happen to them, since every dispatch of a socket outcome ends the cycle. One case per boundary, and the mutation matrix is a clean diagonal: removing the probe at any one site fails exactly its own case and no other. Two of the four needed the case built for them — the page boundary only exposes its probe under a buffer capacity of 1 with a frame wedged ahead of the fatal, and the superseded boundary needs the call abandoned rather than returned. TestSupersededPollBoundSurvivesPumpBackPressure needed the harness's frame-handled drain that the blocked-hand-off test already carries: the notification send is blocking and a probe handles a whole queue in one pass, so the loop wedged in the test's own callback. That inline workaround is now a harness helper, since any test that fills the queue needs it. --- go/pkg/basecamp/eventfeed/catchup.go | 90 ++++++++- go/pkg/basecamp/eventfeed/catchup_test.go | 215 ++++++++++++++++++++-- go/pkg/basecamp/eventfeed/loop_test.go | 23 +++ 3 files changed, 307 insertions(+), 21 deletions(-) diff --git a/go/pkg/basecamp/eventfeed/catchup.go b/go/pkg/basecamp/eventfeed/catchup.go index 07f6bb769d..51f33eaa4b 100644 --- a/go/pkg/basecamp/eventfeed/catchup.go +++ b/go/pkg/basecamp/eventfeed/catchup.go @@ -120,6 +120,16 @@ func (l *loop) walk(at *attempt, cursor Cursor, presentClass bool) (out cycleOut // dispatched here, before anything else this iteration does: the page // that was in flight has been fully accepted, delivered and saved, so // this IS transition 21's page boundary. + // + // The probe runs FIRST, over every frame the pump has already read. + // Reaching them was previously left to the next poll's admission pass, + // which is bounded by the caller-configurable liveBufferCapacity — one + // queued ping under WithLiveBufferCapacity(1) spends the whole pass — + // and, more simply, never runs at all when the slot below ends the + // cycle first. + if out, done := l.probeFatal(at); done { + return out, "", true + } if out, done := l.dispatchDeferred(at); done { return out, "", true } @@ -148,7 +158,11 @@ func (l *loop) walk(at *attempt, cursor Cursor, presentClass bool) (out cycleOut // The socket's own outcome, deferred while the seam call was in // flight, outlived the staleness window with the call still // outstanding. It IS the disposition — transition 21 — and its - // teardown is what cancels the abandoned call. + // teardown is what cancels the abandoned call. The server's own + // verdict outranks it if the pump had already read one. + if out, done := l.probeFatal(at); done { + return out, "", true + } if out, done := l.dispatchDeferred(at); done { return out, "", true } @@ -172,6 +186,14 @@ func (l *loop) walk(at *attempt, cursor Cursor, presentClass bool) (out cycleOut // reported instead as authorization_failed or poll_failed, or // retried indefinitely behind poll-retry waits when the error is // transient. + // + // The probe extends that reasoning from the slot to the queue: + // recoverPoll's disposal discards everything the pump read, not + // just the parked frame, so a fatal sitting behind a recoverable + // one was replaced by the poll's own error just the same. + if out, done := l.probeFatal(at); done { + return out, "", true + } if out, done := l.dispatchDeferred(at); done { return out, "", true } @@ -230,7 +252,11 @@ func (l *loop) walk(at *attempt, cursor Cursor, presentClass bool) (out cycleOut // Terminal(protocol_fatal). Transition 21's finish-the-page // ordering does not apply here — it exists so an ACCEPTED page's // deliveries and save are not stranded by the socket's death, and - // this page is being refused, so there is nothing to strand. + // this page is being refused, so there is nothing to strand. The + // probe covers the queue for the same reason it covers the slot. + if out, done := l.probeFatal(at); done { + return out, "", true + } if out, done := l.dispatchDeferred(at); done { return out, "", true } @@ -685,7 +711,7 @@ func (l *loop) drain(at *attempt) (cycleOutcome, bool) { // Because nothing is held outside the buffer, an admission that overflows // evicts the oldest events still pending — including ones this drain has // not replayed yet. That is the same pre-cut loss condition the buffer - // always had, taking its drop-time dispatch in drainScan, before the held + // always had, taking its drop-time dispatch in fatalScan, before the held // save: the conjunctive invariant asks that every such loss be explicitly // accepted, not that a drain be exempt from the capacity it was // configured with. @@ -697,7 +723,7 @@ func (l *loop) drain(at *attempt) (cycleOutcome, bool) { // plus pumpDepth+1. budget := pumpDepth + 1 for { - if out, done := l.drainScan(at, &budget); done { + if out, done := l.fatalScan(at, &budget); done { return out, true } ev, ok := l.buffer.shift() @@ -711,16 +737,43 @@ func (l *loop) drain(at *attempt) (cycleOutcome, bool) { } } -// drainScan is Draining's protocol-fatal carve-out, and the ONLY reason the -// drain looks at the frame queue at all. SPEC.md §23: "a raw -// `invalid_event_stream_command` observed during Draining is -// Terminal(`protocol_fatal`) immediately — the drain is not completed, the +// fatalScan is the protocol-fatal carve-out's probe, and the ONLY reason the +// drain or the walk looks at the frame queue outside an ordinary admission. +// SPEC.md §23: "a raw `invalid_event_stream_command` observed during Draining +// is Terminal(`protocol_fatal`) immediately — the drain is not completed, the // held entry position is NOT saved, and no `caught_up` is announced; only // recoverable failures defer." Without the scan a fatal frame — one the walk // left deferred, or one the pump has already queued — is seen only in // Streaming, after the held save and the caught_up announcement have both // happened. // +// # Why the walk runs it too +// +// The obligation is not Draining's. §23 states it state-generically, twice: +// "the protocol-fatal disconnect is terminal from EVERY socket-open state", +// and "a raw invalid_event_stream_command frame READ in AwaitingWelcome, +// CatchingUp, or Draining — the pump runs throughout — is the same +// reason-level dispatch applied state-generically: Terminal(protocol_fatal), +// never Backoff. An explicitly non-retryable protocol rejection must not +// reconnect from any state." READ is the boundary the obligation attaches to. +// +// The walk has four points where it disposes of a deferred receive, and every +// one of them ENDS THE CYCLE with the pump's queue never inspected. So a +// `remote` disconnect parked in the slot — recoverable, arrived first — beat a +// fatal the pump had already read sitting right behind it: the cycle ended as +// an ordinary socket failure, the connector re-minted, and the fatal died with +// the socket. That is the identical hole #760 closed inside the drain, +// arriving through the other door, and it is why this is a shared helper +// rather than a second copy. +// +// The treatment of what it dequeues is what makes it safe to run mid-walk. +// Live events are admitted to the buffer, which is what CatchingUp does with +// them anyway; the FIRST non-fatal socket outcome is parked in the same single +// slot the caller is about to dispatch from, so arrival order for the reported +// outcome is preserved; and later ones are discarded, which is all that could +// ever happen to them (every dispatch of a socket outcome ends the cycle, so a +// second could never be reached). +// // Everything else keeps Draining's deferred consumption, which is §23's one // deliberate deferred-consumption case: a recoverable socket failure, a // non-fatal disconnect, an invalid frame or a closed pump is parked in the @@ -733,7 +786,7 @@ func (l *loop) drain(at *attempt) (cycleOutcome, bool) { // budget is the drain's shared dequeue allowance, sized at the scanned // queue's own depth (see drain); an ordinary frame consuming it must never be // able to end the scan short of a fatal frame the pump had already queued. -func (l *loop) drainScan(at *attempt, budget *int) (cycleOutcome, bool) { +func (l *loop) fatalScan(at *attempt, budget *int) (cycleOutcome, bool) { // A fatal frame already in the slot is dispatched now rather than after // the save: the carve-out is about what governs, not about which queue the // frame is sitting in. @@ -781,6 +834,25 @@ func (l *loop) drainScan(at *attempt, budget *int) (cycleOutcome, bool) { return cycleOutcome{}, false } +// probeFatal runs one fatalScan over everything the pump has already read, at +// a walk boundary that is about to dispose of a deferred receive. +// +// The budget is a fresh pumpDepth+1 per boundary, and for the same reason the +// drain's is: the queue holds at most pumpDepth items, while the pump — a +// single goroutine — may be blocked inside handOff with one more, and a budget +// of pumpDepth spends itself on that frame's predecessors. It is deliberately +// NOT the ownership cut's liveBufferCapacity, which is caller-configurable +// down to 1 and measures the state machine's buffer rather than the pump's +// queue. +// +// A fresh budget per boundary is bounded work, not an unbounded scan: the scan +// stops the moment the queue is momentarily empty, so a boundary reached with +// nothing queued costs one non-blocking receive. +func (l *loop) probeFatal(at *attempt) (cycleOutcome, bool) { + budget := pumpDepth + 1 + return l.fatalScan(at, &budget) +} + // deferForDrain parks one socket outcome the scan passed over, keeping the // FIRST and discarding the rest. // diff --git a/go/pkg/basecamp/eventfeed/catchup_test.go b/go/pkg/basecamp/eventfeed/catchup_test.go index 4529d775b1..01f65db271 100644 --- a/go/pkg/basecamp/eventfeed/catchup_test.go +++ b/go/pkg/basecamp/eventfeed/catchup_test.go @@ -12,7 +12,9 @@ import ( "errors" "fmt" "runtime" + "slices" "strings" + "sync" "testing" "time" @@ -1542,6 +1544,9 @@ func TestSupersededPollBoundSurvivesPumpBackPressure(t *testing.T) { default: } }) + // The probe at the superseded boundary handles the whole queue this test + // fills, in one pass; nothing here awaits handled frames. + h.drainHandled() var conn *feedtest.Conn h.polls.OnCall(func(feedtest.PollCall) { // The socket speaks its last word while the entry poll is @@ -1617,6 +1622,202 @@ func TestDeferredProtocolFatalOutranksAFailedPoll(t *testing.T) { assertPositions(t, store.Saves()) } +// TestProtocolFatalBehindADeferredRecoverableOutranksIt is the walk's version +// of the hole #760 closed inside the drain, and it is the same hole: a +// protocol-fatal verdict the pump has ALREADY READ loses to whatever +// non-fatal outcome happens to be parked ahead of it. +// +// SPEC §23 is state-generic about this and says so twice — "the protocol-fatal +// disconnect is terminal from EVERY socket-open state", and "a raw +// invalid_event_stream_command frame READ in AwaitingWelcome, CatchingUp, or +// Draining ... Terminal(protocol_fatal), never Backoff. An explicitly +// non-retryable protocol rejection must not reconnect from any state." Reading +// is the boundary the obligation attaches to, not dispatch order. +// +// So the deferral slot holds a `remote` disconnect — recoverable, arrived +// first — and the fatal sits in the pump's queue behind it. dispatchDeferred +// reported the `remote`, ended the cycle as a plain socket failure, and +// reconnected: a second mint, against a server that had already refused the +// command. The fatal died with the socket. +// +// The assertions are the ones the carve-out is about. The terminal reason, +// because reporting the recoverable verdict is the defect itself; and ZERO +// reconnects, because "never Backoff" is the half a reason assertion alone +// would not catch (a connector that reconnects once and then terminates on the +// re-issued frame still ends at protocol_fatal). +// +// There is one case per deferred-disposition boundary the walk has, because +// each is a separate hole: a probe removed at any ONE of them must fail +// exactly its own case, and a case that passes with every probe removed is +// pinning nothing. +func TestProtocolFatalBehindADeferredRecoverableOutranksIt(t *testing.T) { + for _, tc := range []struct { + name string + opts []eventfeed.Option + // arm scripts the poll outcome that decides WHICH deferred-disposition + // boundary the walk reaches with the slot occupied. + arm func(h *harness) + // wedge serves whatever must sit between the deferred recoverable and + // the fatal, for the boundaries where merely queueing the fatal is not + // enough to keep an ordinary admission pass from reaching it first. + wedge func(conn *feedtest.Conn) + // stalled marks the boundary reached by abandoning a stalled call + // rather than by the call returning. + stalled bool + // wantSaves is what may legitimately have been persisted BEFORE the + // fatal was observed. + wantSaves []string + }{ + // The ordinary page boundary. The page succeeds and its position + // saves; the walk then loops back to the top with the slot occupied. + // A ping is wedged ahead of the fatal and the buffer is sized to 1, + // because otherwise the entry cut's admission pass — bounded by + // liveBufferCapacity, which defaults large — dequeues the fatal on its + // own and the boundary's own probe is never what catches it. That + // bound is caller-configurable down to 1, which is the whole reason + // the probe carries pumpDepth+1 instead of borrowing it. + { + name: "page boundary", + opts: []eventfeed.Option{eventfeed.WithLiveBufferCapacity(1)}, + arm: func(h *harness) { + h.polls.ScriptPage(eventfeed.PollPage{Position: "pos-1", Next: testOrigin + "/999/events.json?after=1"}) + h.polls.ScriptPage(eventfeed.PollPage{Position: "pos-2"}) + }, + wedge: func(conn *feedtest.Conn) { conn.Serve(framePing()) }, + wantSaves: []string{"pos-1"}, + }, + // The superseded boundary: the call is abandoned with the deferral + // still parked, and its teardown is what cancels the call. + { + name: "superseded boundary", + arm: func(h *harness) { h.polls.StallNext() }, + stalled: true, + }, + // The poll-error boundary: recoverPoll's terminal and re-entry + // branches dispose the attempt, discarding everything the pump read. + { + name: "poll error boundary", + arm: func(h *harness) { + h.polls.ScriptError(&eventfeed.PollError{Kind: eventfeed.PollUnauthorized}) + }, + }, + // The malformed-page boundary: a page with no position is refused, + // and the refusal is classified after the deferral is dispatched. + { + name: "malformed page boundary", + arm: func(h *harness) { h.polls.ScriptPage(eventfeed.PollPage{Position: ""}) }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + store := feedtest.NewStore() + store.Stored("pos-0") + h := storedHarness(t, store, tc.opts...) + h.minter.ScriptTicket(ticket(1)) + // The second mint is scripted to fail TERMINALLY. A reconnecting + // connector therefore ends at Terminal(mint_failed) having made + // two mint calls, and a correct one ends at + // Terminal(protocol_fatal) having made one — so both assertions + // discriminate, and neither outcome strands the run on a socket + // the test never serves. + h.minter.ScriptError(&eventfeed.MintError{Kind: eventfeed.MintUnrecoverable}) + tc.arm(h) + h.drainHandled() + + deferred := make(chan struct{}, 4) + h.conn.OnFrameDeferred(func() { deferred <- struct{}{} }) + queued := make(chan struct{}, eventfeed.ExportPumpDepth) + h.conn.OnPumpHandedOff(func(bool) { + select { + case queued <- struct{}{}: + default: + } + }) + var conn *feedtest.Conn + var once sync.Once + h.polls.OnCall(func(feedtest.PollCall) { + once.Do(func() { + // Recoverable FIRST, so it takes the single deferral slot. + conn.Serve(frameDisconnect("remote", true)) + select { + case <-deferred: + case <-time.After(watchdog): + t.Error("the recoverable disconnect was never deferred") + return + } + if tc.wedge != nil { + tc.wedge(conn) + } + // Fatal LAST, and only once the pump has handed it off: + // the guarantee is about a frame the pump has READ, so a + // test that raced the read would prove nothing. + drain(queued) + conn.Serve(frameDisconnect("invalid_event_stream_command", false)) + select { + case <-queued: + case <-time.After(watchdog): + t.Error("the fatal frame never reached the hand-off queue") + } + if tc.stalled { + // Lapse the superseded-poll bound so the walk + // abandons the call it is still inside. + h.clock.Advance(staleAfter) + } + }) + }) + h.start() + conn = h.driveToSubscribed() + conn.Serve(frameConfirm(noFilterIdentifier)) + h.awaitEndOrReconnect() + h.join() + + _, terminal, _ := h.snapshot() + if terminal == nil || terminal.Reason != eventfeed.ReasonProtocolFatal { + t.Fatalf("terminal = %v, want reason %q — the server's verdict was already read", terminal, eventfeed.ReasonProtocolFatal) + } + if got := h.minter.Calls(); got != 1 { + t.Fatalf("mint calls = %d, want 1: a protocol-fatal rejection must not reconnect from any state", got) + } + assertPositions(t, store.Saves(), tc.wantSaves...) + }) + } +} + +// drain empties a buffered signal channel without blocking. +func drain(ch chan struct{}) { + for { + select { + case <-ch: + default: + return + } + } +} + +// awaitEndOrReconnect returns once the run has ended, or once a reconnect has +// armed the backoff timer — firing it in that case so the run proceeds to its +// scripted terminal mint failure instead of parking on a virtual clock nothing +// advances. Without it a run that wrongly reconnects hangs, and a hang reports +// as "the watchdog expired" rather than as the defect under test. +func (h *harness) awaitEndOrReconnect() { + h.t.Helper() + deadline := time.Now().Add(watchdog) + for { + select { + case <-h.done: + return + default: + } + if slices.Contains(h.clock.Outstanding(), timerBackoff) { + h.fireTimer(timerBackoff) + return + } + if time.Now().After(deadline) { + h.t.Fatal("the run neither ended nor reconnected within the watchdog") + } + time.Sleep(time.Millisecond) + } +} + // TestProtocolFatalBehindTheBlockedHandOffIsImmediate extends the carve-out's // scan bound to the frame the pump has already READ but not yet handed off. // @@ -1656,18 +1857,8 @@ func TestProtocolFatalBehindTheBlockedHandOffIsImmediate(t *testing.T) { h.waitUntil("the drain parked mid-delivery", func() bool { return len(h.deliveredIDs()) == 1 }) // The scan handles a whole queue's worth of frames in one pass, which is // more than the harness's frame-handled channel holds; nothing waits on - // it from here, so it is drained for the rest of the test. - drained := make(chan struct{}) - go func() { - for { - select { - case <-h.handled: - case <-drained: - return - } - } - }() - t.Cleanup(func() { close(drained) }) + // it from here. + h.drainHandled() for i := 0; i < eventfeed.ExportPumpDepth; i++ { conn.Serve(framePing()) } diff --git a/go/pkg/basecamp/eventfeed/loop_test.go b/go/pkg/basecamp/eventfeed/loop_test.go index 005c65cc4b..6ba0b4befd 100644 --- a/go/pkg/basecamp/eventfeed/loop_test.go +++ b/go/pkg/basecamp/eventfeed/loop_test.go @@ -268,6 +268,29 @@ func (h *harness) serveSettled(conn *feedtest.Conn, frames ...[]byte) { h.waitUntil("pump took the served frames", func() bool { return conn.Pending() == 0 }) } +// drainHandled consumes frame-handled notifications for the rest of the test. +// +// The notification send is BLOCKING, and the protocol-fatal probe handles a +// whole queue's worth of frames in a single pass — far more than the channel +// buffers. A test that fills the pump's queue and does not itself await +// handled frames must drain them, or the loop wedges inside the probe on a +// send nobody is receiving. That is a property of the harness's notification, +// not of the connector: the wedge shows up as the state machine parked in +// newHarness's OnFrameHandled callback. +func (h *harness) drainHandled() { + done := make(chan struct{}) + go func() { + for { + select { + case <-h.handled: + case <-done: + return + } + } + }() + h.t.Cleanup(func() { close(done) }) +} + // awaitFrameHandled consumes handled-frame notifications until the given // kind is seen. func (h *harness) awaitFrameHandled(kind string) { From f925a4bf8f792b60e4dccf6c755828468b58e496 Mon Sep 17 00:00:00 2001 From: Jeremy Daer Date: Tue, 18 Aug 2026 23:03:23 -0700 Subject: [PATCH 13/51] Event feed: the gate paid a real cost for a partial version of a free guarantee MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `durableGate` is deleted, and `Connector.Wait` replaces what it was reaching for. The gate refused a checkpoint save once Close had latched. Its own doc states the residual honestly: claiming the gate and writing are not atomic together, so the window in which a replacement connector's newer position gets overwritten is NARROWED, from [decision, write] to [claim, write], not closed. Weigh that against what it cost, which the test pinning it spelled out without naming it as a cost. Close taken from Observer.PageDelivered — a callback that fires immediately after the page's events reach the consumer and immediately before its position is written — meant that position was never persisted. The next run re-delivers those events with no record they ever arrived. So: a bounded-replay harm avoided some of the time, paid for with a bounded-replay harm caused every time. And it prevented at most about one save anyway, since runCtx is cancelled synchronously and the walk's dispatch points unwind almost immediately. The guarantee it wanted is free. The iterator returning IS quiescence — the run has exited, so no save can be in flight, by construction rather than by a window narrow enough to be unlikely. `Wait` exposes that to a caller who closes from a goroutine that does not own the range loop; a caller that owns the loop already has it. The contract is now "await termination — or Wait — before opening a second connector over the same store", which drops the term of art ("commence") that two reviewers independently read as a stronger promise than it was. Closes #784. The second half is Close precedence, which `emitTerminal` had established for the terminal element and nothing else. Every lifecycle announcement kept firing after a Close, because the nearest runCtx check sat AFTER the callback — placed to catch a Close taken from INSIDE it, which is a different Close from one taken earlier. Five checks now, each with a case that only it can satisfy: before `connecting`, `catch_up_started` on both the post-confirmation and repair paths, `caught_up`, the ownership cut, and socketCheck. Two of those are about announcements the walk makes on the way past. The ownership cut and socketCheck both DISPATCH what they dequeue, so a disconnect frame queued when the consumer closes was announced through Observer.Disconnected — a teardown report for a socket the consumer had already stopped caring about. Ordering within the page boundary is deliberate and runs the other way from the rest: the page's save happens BEFORE the check, its announcements after. A check placed before the save would reintroduce exactly the dropped save the gate was deleted for. Finish the page, then observe the close. Getting the mutation matrix to a clean diagonal took three corrections worth recording. The `caught_up` check and the page boundary's masked each other under a position-resume entry, so the case that isolates `caught_up` had to use a PRESENT-class entry, where the held save fires after the walk has returned. The socketCheck case was driving the ownership cut instead — the default harness resolves to present-class — and claimed in its name to be about the other. And its hand-off wait was satisfied by a stale token from the handshake, so Close landed before the frame was ever queued and the code under test had nothing to find. One check has no test and says so: the repair path's, which backstops a select in `stream` where `runCtx.Done()` and `repair.C()` can both be ready and Go chooses. That is the same race `emitTerminal` documents, and it cannot be driven deterministically from outside. --- go/pkg/basecamp/eventfeed/catchup.go | 40 ++ go/pkg/basecamp/eventfeed/connector.go | 163 +++---- go/pkg/basecamp/eventfeed/connector_test.go | 479 ++++++++++++++++++-- go/pkg/basecamp/eventfeed/loop.go | 23 +- go/pkg/basecamp/eventfeed/recovery.go | 6 + 5 files changed, 555 insertions(+), 156 deletions(-) diff --git a/go/pkg/basecamp/eventfeed/catchup.go b/go/pkg/basecamp/eventfeed/catchup.go index 51f33eaa4b..a07b4b431e 100644 --- a/go/pkg/basecamp/eventfeed/catchup.go +++ b/go/pkg/basecamp/eventfeed/catchup.go @@ -33,6 +33,11 @@ func (l *loop) runCatchUp(h catchUpHandoff) cycleOutcome { if l.hooks.catchUpEntered != nil { l.hooks.catchUpEntered(h) } + // Close outranks the announcement: no walk is about to start. + if l.runCtx.Err() != nil { + l.disposeAttempt(h.at, nil) + return cycleOutcome{kind: outcomeClosed} + } if l.cfg.observer.CatchUpStarted != nil { // Cursor.PageURL is redacted to its origin: on a 410-accepted re-entry // it IS the server-supplied resume URL, carrying whatever query the @@ -81,6 +86,17 @@ func (l *loop) walkThenDrain(at *attempt, cursor Cursor, presentClass bool) (cyc l.acceptPosition(held) l.saveCheckpoint(held) } + // Close outranks the announcement, and the ordering with the held save + // above is deliberate in BOTH directions. The save comes first because its + // events were already delivered, and dropping the write because the + // consumer closed a moment later silently re-delivers them. `caught_up` + // comes after the check because it is an announcement about a feed that is + // now Closed — it would be reporting a steady state the connector is not + // about to enter. + if l.runCtx.Err() != nil { + l.disposeAttempt(at, nil) + return cycleOutcome{kind: outcomeClosed}, true + } if l.cfg.observer.CaughtUp != nil { l.cfg.observer.CaughtUp() } @@ -299,6 +315,15 @@ func (l *loop) walk(at *attempt, cursor Cursor, presentClass bool) (out cycleOut // delivered before anything durable moves. held = page.Position if !cutTaken { + // The cut's admission pass DISPATCHES what it dequeues, so it + // is a second place a page boundary can announce a teardown + // the consumer has already closed past. It is also pure + // preparation for the held save, which a closing walk will + // never reach — so on Close there is nothing to prepare. + if l.runCtx.Err() != nil { + l.disposeAttempt(at, nil) + return cycleOutcome{kind: outcomeClosed}, "", true + } if out, done := l.ownershipCut(at); done { return out, "", true } @@ -309,6 +334,21 @@ func (l *loop) walk(at *attempt, cursor Cursor, presentClass bool) (out cycleOut l.saveCheckpoint(page.Position) } + // The page is FINISHED — delivered, and its position accepted and + // saved — so a Close taken from inside PageDelivered, or from any + // other callback this page ran, ends the walk here rather than + // following `next` into another poll. + // + // After the save, never before it. A check between PageDelivered and + // the save would drop the write for events the consumer has already + // received, which the next run then re-delivers with no record they + // arrived. That is the same trade the deleted durable gate was making, + // and it is the wrong one in both places: finish the page, then + // observe the close. + if l.runCtx.Err() != nil { + l.disposeAttempt(at, nil) + return cycleOutcome{kind: outcomeClosed}, "", true + } if page.Next == "" { return cycleOutcome{}, held, false } diff --git a/go/pkg/basecamp/eventfeed/connector.go b/go/pkg/basecamp/eventfeed/connector.go index 2458105411..fd17ae4640 100644 --- a/go/pkg/basecamp/eventfeed/connector.go +++ b/go/pkg/basecamp/eventfeed/connector.go @@ -96,18 +96,15 @@ type config struct { // input, two consumers: it is CheckpointKey.Origin, and it is the // same-origin reference every continuation and resume URL is validated // against before an authenticated follow. - origin string - accountID string - minter TicketMinter - polls PollSource - filters Filters - start Start - transport CableTransport - clock Clock - store CheckpointStore - // durable serializes the commencement of a checkpoint save against Close. - // Always non-nil after New; see durableGate. - durable *durableGate + origin string + accountID string + minter TicketMinter + polls PollSource + filters Filters + start Start + transport CableTransport + clock Clock + store CheckpointStore consumerNamespace string confirmationDeadline time.Duration repairInterval time.Duration @@ -201,83 +198,10 @@ type Connector struct { // cancelRun cancels the active run's context. Registered by Events for // exactly the span of one iteration, nil otherwise. cancelRun context.CancelFunc -} - -// durableGate serializes the COMMENCEMENT of a checkpoint save against Close. -// -// A save is the connector's only effect that outlives the process, and the -// failure this prevents is sequential rather than adversarial: a consumer calls -// Close, opens a second connector over the same checkpoint file, and a save the -// first connector had not yet begun lands afterwards — on top of a lineage the -// new run has already loaded and may already have advanced. -// -// Cancellation alone cannot prevent it. A CheckpointStore is entitled to ignore -// ctx, and the built-in FileCheckpointStore documents that it does, deliberately: -// a position the connector already accepted must not be dropped because the -// run's context was cancelled between acceptance and write-through. So the -// cancelled context Close publishes is exactly the signal this store is -// specified not to act on. -// -// # Close does not wait, and must not -// -// The guarantee is that no save COMMENCES after Close returns, where commencing -// is claiming the gate — an operation that is atomic against Close and takes no -// host code with it. The lock is never held across CheckpointStore.Save. -// -// Holding it across the store call was the obvious spelling and is wrong twice -// over. A store whose Save calls Connector.Close self-deadlocks on the caller's -// own goroutine, and Close is documented as callable from anywhere; and a store -// that merely stalls blocks EVERY Close for as long as it stalls, which -// contradicts the one thing Close promises unconditionally — that cancellation -// is visible before it returns. -// -// A save that claimed the gate before Close therefore runs to completion after -// Close returns. That is the intended reading rather than a gap: its position -// was accepted and delivered before Close was called, and abandoning a durable -// write half-way is worse than finishing it. What cannot happen is a save the -// connector had not yet decided to make. -// -// # The residual, stated rather than left to be rediscovered -// -// "Begun" in the opening paragraph is this gate's term of art — claiming the -// gate — and the failure described there is prevented only under that reading. -// Say so plainly, because a reader who takes "begun" to mean the store call -// reads the opening as a stronger promise than the gate delivers. -// -// The claim and the store call are not atomic together: a save can be -// descheduled between them for as long as the scheduler likes. So the window -// in which a replacement connector's newer position can be overwritten is -// NARROWED, from [decision, write] to [claim, write], rather than closed. The -// overwrite moves the checkpoint BACKWARD, against §23's "checkpoints only -// move forward"; its cost is bounded replay on the next entry rather than -// skipped events, because the position that survives is the older one. -// -// Closing it needs one of two things this type cannot have: Close waiting on -// the store call (rejected above, and reachable as a deadlock from any -// callback), or a fencing token the store can use to refuse a write from a -// superseded run — a CheckpointStore contract change normative for all six -// SDKs. Tracked in #784 with a third option, declaring the residual -// in-contract, rather than decided here. -type durableGate struct { - mu sync.Mutex - closed bool -} - -// begin claims the right to start one durable operation, reporting whether it -// was granted. The lock is released before returning: the claim is the -// serialized act, not the write. -func (g *durableGate) begin() bool { - g.mu.Lock() - defer g.mu.Unlock() - return !g.closed -} - -// close latches the gate. It never waits for an operation already in flight — -// see the type comment. -func (g *durableGate) close() { - g.mu.Lock() - g.closed = true - g.mu.Unlock() + // runDone is closed when the iteration function returns. Published by + // Events at the start of a run and never cleared, so Wait either finds no + // run at all or a channel that is closed exactly when the run has exited. + runDone chan struct{} } // testHooks are unexported in-package observation points for the tier-2/3 @@ -367,7 +291,6 @@ func New(origin, accountID string, minter TicketMinter, polls PollSource, opts . if cfg.clock == nil { cfg.clock = SystemClock() } - cfg.durable = &durableGate{} if err := validateConfig(&cfg); err != nil { return nil, err } @@ -488,12 +411,19 @@ func (c *Connector) Events(ctx context.Context) iter.Seq2[Event, error] { } runCtx, cancel := context.WithCancel(ctx) defer cancel() + // Published under the same lock as the cancel func, and CLOSED rather + // than cleared on exit: clearing it would leave a window where Wait + // reads nil — and returns at once — while this function has not + // finished unwinding, which is the one instant Wait exists to cover. + done := make(chan struct{}) + defer close(done) // Register this run's cancellation under the same lock Close latches // under, so the two cannot straddle each other: a Close that already // latched cancels here, and one that latches later finds the func and // cancels it itself. Either way runCtx is done before Close returns — // there is no window in which the run proceeds past a returned Close. c.mu.Lock() + c.runDone = done if c.isClosed { // Closed before the first iteration: end cleanly with zero wire // attempts, deterministically. @@ -540,14 +470,23 @@ func (c *Connector) Events(ctx context.Context) iter.Seq2[Event, error] { // mean holding a lock across every seam call, i.e. across arbitrary host code, // which trades a benign race for a deadlock reachable from any callback. // -// The distinction matters for exactly one effect, and that one is ordered: -// a CHECKPOINT SAVE cannot commence after Close returns (durableGate). It is -// the only thing the connector does that outlives the process, and the only one -// where "began a moment late" is not self-limiting — a store may ignore the -// cancelled context, and the built-in one documents that it does. Everything -// else that can start in the window is a read whose context is already dead: -// it returns promptly, its result is discarded, and the run takes the Closed -// edge. +// A CHECKPOINT SAVE is the one effect where that matters, because it is the +// only one that outlives the process: a store may ignore the cancelled context, +// and the built-in one documents that it deliberately does. So a save decided +// just before Close can still be written just after. That is intended. The +// position was accepted and its events were delivered before Close was called, +// and the store is what tells the next run where those deliveries stopped — +// dropping the write would silently re-deliver them. Everything else that can +// start in the window is a read whose context is already dead: it returns +// promptly, its result is discarded, and the run takes the Closed edge. +// +// # Ordering a second connector over the same store is the CONSUMER's to do +// +// Close does not order it, and no gate inside Close can. Wait does: the run +// goroutine has exited, so no save can be in flight, by construction rather +// than by a window narrow enough to be unlikely. Await the iterator's +// termination — or Wait — before opening a second connector over the same +// checkpoint store. // // What Close deliberately does NOT do is wait for the run goroutine to exit. // Close is callable from inside a consumer callback — an observer, a signal @@ -573,12 +512,26 @@ func (c *Connector) Close() error { c.cancelRun = nil } c.mu.Unlock() - // After the cancel, and outside mu: latch the durable gate, so no - // checkpoint save COMMENCES from here on. This is the one effect that - // outlives the process, and the one cancellation cannot stop — a - // CheckpointStore may ignore ctx, and the built-in one documents that it - // does. Latching does not wait for a save already under way; see - // durableGate for why waiting is the wrong trade. - c.cfg.durable.close() return nil } + +// Wait blocks until the run goroutine has exited, and returns immediately when +// no run is active. It is the quiescence point Close deliberately is not. +// +// Use it before opening a second connector over the same checkpoint store from +// a goroutine that does not own the range loop. A consumer that DOES own the +// loop needs nothing extra: the iterator returning is the same guarantee — the +// run goroutine has exited by then, so no save can be in flight. +// +// It is not callable from a consumer callback. Every callback — an observer, a +// signal handler, the loop body — runs ON the run goroutine, so waiting for +// that goroutine from inside one waits for itself. Close is the call that is +// safe from anywhere; this is the one that is safe from anywhere ELSE. +func (c *Connector) Wait() { + c.mu.Lock() + done := c.runDone + c.mu.Unlock() + if done != nil { + <-done + } +} diff --git a/go/pkg/basecamp/eventfeed/connector_test.go b/go/pkg/basecamp/eventfeed/connector_test.go index 6866a7786a..01b8189920 100644 --- a/go/pkg/basecamp/eventfeed/connector_test.go +++ b/go/pkg/basecamp/eventfeed/connector_test.go @@ -633,7 +633,6 @@ type blockingStore struct { loadRelease chan struct{} mu sync.Mutex saves []string - attempts int } func newBlockingStore() *blockingStore { @@ -664,9 +663,6 @@ func (s *blockingStore) Load(ctx context.Context, _ eventfeed.CheckpointKey) (st } func (s *blockingStore) Save(_ context.Context, _ eventfeed.CheckpointKey, position string) error { - s.mu.Lock() - s.attempts++ - s.mu.Unlock() s.entered <- struct{}{} <-s.release s.mu.Lock() @@ -681,25 +677,18 @@ func (s *blockingStore) recorded() []string { return append([]string(nil), s.saves...) } -func (s *blockingStore) attemptCount() int { - s.mu.Lock() - defer s.mu.Unlock() - return s.attempts -} - -// TestCloseDoesNotBlockOnAnInFlightCheckpointSave pins the half of the durable -// gate that is about Close's own promise. +// TestCloseDoesNotBlockOnAnInFlightCheckpointSave pins Close's own promise. // -// Holding the gate across CheckpointStore.Save was the obvious spelling and is -// wrong twice: a store whose Save calls Close self-deadlocks on the caller's -// goroutine (Close is documented as callable from anywhere), and a store that -// merely stalls blocks EVERY Close for as long as it stalls — contradicting the -// one thing Close promises unconditionally, that cancellation is visible before -// it returns. +// Waiting on CheckpointStore.Save was the obvious spelling and is wrong twice: +// a store whose Save calls Close self-deadlocks on the caller's goroutine +// (Close is documented as callable from anywhere), and a store that merely +// stalls blocks EVERY Close for as long as it stalls — contradicting the one +// thing Close promises unconditionally, that cancellation is visible before it +// returns. // -// So Close latches and returns. The save that already claimed the gate still -// completes: its position was accepted and delivered before Close was called, -// and abandoning a durable write half-way is worse than finishing it. +// So Close cancels and returns. The in-flight save still completes: its +// position was accepted and delivered before Close was called, and abandoning a +// durable write half-way is worse than finishing it. func TestCloseDoesNotBlockOnAnInFlightCheckpointSave(t *testing.T) { store := newBlockingStore() h := newHarness(t, @@ -736,17 +725,25 @@ func TestCloseDoesNotBlockOnAnInFlightCheckpointSave(t *testing.T) { } } -// TestNoCheckpointSaveCommencesAfterClose is the other direction: once Close -// has returned, a save that had not begun does not begin. +// TestAnAcceptedPositionSavesEvenWhenCloseLandsFirst is the other direction, +// and it is the inverse of what this test used to assert. // -// Close is taken from inside Observer.PageDelivered — the callback that fires -// on the consumer's goroutine immediately BEFORE the page's save — so the run -// is demonstrably at the save boundary when it lands. An earlier version closed -// before the feed ever reached a page and passed with the gate removed -// entirely, which is no test at all. -func TestNoCheckpointSaveCommencesAfterClose(t *testing.T) { +// It used to pin a gate that REFUSED a save once Close had returned. Close is +// taken from inside Observer.PageDelivered, which fires immediately after the +// page's events have been handed to the consumer and immediately before the +// page's position is written — so what the gate actually bought here was a +// silent dropped save for events the consumer had already received. The next +// run then re-delivers them, having no record they ever arrived. +// +// That was the whole of the gate's cost, and against it stood a guarantee it +// did not deliver: the overwrite window it existed to close was narrowed, not +// closed (claiming the gate and writing are not atomic together), and the harm +// it narrowed — a checkpoint moving backward — is bounded replay, the same +// class as the drop it caused. Quiescence is what actually closes it, and the +// iterator returning IS quiescence. See Connector.Wait. +func TestAnAcceptedPositionSavesEvenWhenCloseLandsFirst(t *testing.T) { store := newBlockingStore() - close(store.release) // any save that does commence completes immediately + close(store.release) // saves complete immediately var h *harness h = newHarness(t, eventfeed.WithCheckpointStore(store), @@ -768,12 +765,91 @@ func TestNoCheckpointSaveCommencesAfterClose(t *testing.T) { sock.Serve(frameConfirm(noFilterIdentifier)) h.join() - if n := store.attemptCount(); n != 0 { - t.Errorf("store.Save was entered %d time(s) after Close returned, want 0", n) + if got := store.recorded(); len(got) != 1 || got[0] != "pos-1" { + t.Errorf("recorded saves = %v, want [pos-1] — event 101 was delivered, so its position must be durable", got) } - if got := store.recorded(); len(got) != 0 { - t.Errorf("recorded saves = %v, want none", got) +} + +// TestWaitReturnsOnlyAfterTheRunHasExited pins the guarantee that replaced the +// durable gate, and pins it against the thing it must not be confused with. +// +// Close cancels and returns while the run is still unwinding — deliberately, +// since it is callable from inside a callback that runs ON the run's goroutine. +// So Close alone cannot order a second connector over the same store. Wait can, +// because it reports the run's exit, at which point no save can be in flight by +// construction rather than by a window narrow enough to be unlikely. +// +// The save is held INSIDE the store so the two are distinguishable: a Wait that +// merely mirrored Close would return here, with the write still pending. +func TestWaitReturnsOnlyAfterTheRunHasExited(t *testing.T) { + store := newBlockingStore() + h := newHarness(t, + eventfeed.WithCheckpointStore(store), + eventfeed.WithConsumerNamespace("agent")) + h.minter.ScriptTicket(ticket(1)) + h.polls.ScriptPage(eventfeed.PollPage{Events: []eventfeed.Event{pollEvent(101)}, Position: "pos-1"}) + h.start() + sock := h.driveToSubscribed() + sock.Serve(frameConfirm(noFilterIdentifier)) + + // The run goroutine is parked inside CheckpointStore.Save. + select { + case <-store.entered: + case <-time.After(watchdog): + t.Fatal("the checkpoint save never reached the store") } + if err := h.conn.Close(); err != nil { + t.Fatalf("Close: %v", err) + } + + waited := make(chan struct{}) + go func() { h.conn.Wait(); close(waited) }() + select { + case <-waited: + t.Fatal("Wait returned with a checkpoint save still in flight; it must report the run's EXIT, not Close's return") + case <-time.After(50 * time.Millisecond): + } + + close(store.release) + select { + case <-waited: + case <-time.After(watchdog): + t.Fatal("Wait did not return after the run exited") + } + // Quiescence is the point: by the time Wait returns, the write a second + // connector would race is already durable. + if got := store.recorded(); len(got) != 1 || got[0] != "pos-1" { + t.Errorf("recorded saves = %v, want [pos-1] before Wait returns", got) + } + h.join() +} + +// TestWaitWithNoActiveRunReturnsImmediately covers the two shapes that are not +// a running feed: never consumed, and already finished. Neither may block. +func TestWaitWithNoActiveRunReturnsImmediately(t *testing.T) { + t.Run("never consumed", func(t *testing.T) { + h := newHarness(t) + done := make(chan struct{}) + go func() { h.conn.Wait(); close(done) }() + select { + case <-done: + case <-time.After(watchdog): + t.Fatal("Wait blocked on a connector that was never consumed") + } + }) + t.Run("run already finished", func(t *testing.T) { + h := newHarness(t) + h.minter.ScriptError(&eventfeed.MintError{Kind: eventfeed.MintUnrecoverable}) + h.start() + h.join() + done := make(chan struct{}) + go func() { h.conn.Wait(); close(done) }() + select { + case <-done: + case <-time.After(watchdog): + t.Fatal("Wait blocked after the run had already exited") + } + }) } // TestCancelledCheckpointLoadIsNotAStoreFailure is the third item. The load @@ -845,16 +921,333 @@ func TestUncancelledCheckpointLoadFailureIsStillTerminal(t *testing.T) { } } -// TestCloseFromEveryCallbackSiteDoesNotDeadlock covers the hazard the durable -// gate introduces. saveCheckpoint holds a lock across host store I/O, and Close -// waits for that lock — so if the gate were still held while a consumer callback -// ran, a Close taken from inside Observer.Checkpoint would deadlock against the -// very save that fired it. Close from inside a callback is documented as -// supported, so this is the case that has to be right, not merely likely. +// hookedStore is a CheckpointStore that runs a hook inside Load — the one +// callback site that is not an Observer, and the only deterministic way to +// take a Close between the run starting and its first Observer.Connecting. +type hookedStore struct { + onLoad func() + mu sync.Mutex + saves []string +} + +func (s *hookedStore) Load(context.Context, eventfeed.CheckpointKey) (string, bool, error) { + if s.onLoad != nil { + s.onLoad() + } + return "pos-0", true, nil +} + +func (s *hookedStore) Save(_ context.Context, _ eventfeed.CheckpointKey, position string) error { + s.mu.Lock() + defer s.mu.Unlock() + s.saves = append(s.saves, position) + return nil +} + +func (s *hookedStore) recorded() []string { + s.mu.Lock() + defer s.mu.Unlock() + return append([]string(nil), s.saves...) +} + +// TestCloseOutranksTheNextLifecycleAnnouncement extends to the observer what +// emitTerminal already does for the terminal element: a Close the consumer has +// already taken outranks what the connector says next. +// +// emitTerminal made exactly ONE exit Close-ordered. Every lifecycle +// announcement kept firing, because the nearest runCtx check sat AFTER the +// callback rather than before it — a check placed to catch a Close taken from +// INSIDE the callback, which is a different Close from one taken earlier. So a +// consumer that closed during `confirmed` was still told `catch_up_started`, +// and one that closed during `catch_up_started` was still told a page had been +// delivered. +// +// Each case closes from one callback and names the announcement that must not +// follow it. The pairing is the assertion: "the feed stopped" is satisfied by +// almost any behavior, where "this specific next thing was not announced" is +// satisfied only by the ordering. +func TestCloseOutranksTheNextLifecycleAnnouncement(t *testing.T) { + for _, tc := range []struct { + name string + // observer wires the CLOSING callback and the one that must not fire + // after it; forbidden records the latter. + observer func(closeFeed func(), forbidden func(string)) eventfeed.Observer + // notAfter names the announcement under test, for the failure message. + notAfter string + // presentAt enters present-class, which HOLDS its position to the end + // of the walk instead of saving per page. + presentAt bool + }{ + { + name: "Connecting after a Close taken during the checkpoint load", + notAfter: "connecting", + observer: func(_ func(), forbidden func(string)) eventfeed.Observer { + return eventfeed.Observer{ + Connecting: func(int, time.Duration) { forbidden("connecting") }, + } + }, + }, + { + name: "CatchUpStarted after a Close taken during Confirmed", + notAfter: "catch_up_started", + observer: func(cl func(), forbidden func(string)) eventfeed.Observer { + return eventfeed.Observer{ + Confirmed: func() { cl() }, + CatchUpStarted: func(eventfeed.Cursor) { forbidden("catch_up_started") }, + } + }, + }, + { + name: "PageDelivered after a Close taken during CatchUpStarted", + notAfter: "page_delivered", + observer: func(cl func(), forbidden func(string)) eventfeed.Observer { + return eventfeed.Observer{ + CatchUpStarted: func(eventfeed.Cursor) { cl() }, + PageDelivered: func(int, string) { forbidden("page_delivered") }, + } + }, + }, + { + name: "CaughtUp after a Close taken during PageDelivered", + notAfter: "caught_up", + observer: func(cl func(), forbidden func(string)) eventfeed.Observer { + return eventfeed.Observer{ + PageDelivered: func(int, string) { cl() }, + CaughtUp: func() { forbidden("caught_up") }, + } + }, + }, + { + name: "CaughtUp after a Close taken during Checkpoint", + notAfter: "caught_up", + observer: func(cl func(), forbidden func(string)) eventfeed.Observer { + return eventfeed.Observer{ + Checkpoint: func(string) { cl() }, + CaughtUp: func() { forbidden("caught_up") }, + } + }, + }, + // The case above cannot isolate the check before `caught_up`: on a + // position-resume entry the save fires per page, so the walk's own + // boundary check ends things first and either check alone would pass + // it. A PRESENT-class entry HOLDS its position to the end, so its + // Checkpoint callback fires after the walk has already returned and + // after the drain — leaving the check before `caught_up` as the only + // thing standing between that Close and the announcement. + { + name: "CaughtUp after a Close taken during the HELD save", + notAfter: "caught_up", + presentAt: true, + observer: func(cl func(), forbidden func(string)) eventfeed.Observer { + return eventfeed.Observer{ + Checkpoint: func(string) { cl() }, + CaughtUp: func() { forbidden("caught_up") }, + } + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + var mu sync.Mutex + var seen []string + forbidden := func(name string) { + mu.Lock() + defer mu.Unlock() + seen = append(seen, name) + } + var h *harness + closeFeed := func() { + if err := h.conn.Close(); err != nil { + t.Errorf("Close: %v", err) + } + } + store := &hookedStore{} + if tc.notAfter == "connecting" { + // The load runs before the first mint and before the first + // Connecting announcement, so it is the deterministic way to + // take a Close in that window. Every other case closes from an + // Observer callback. + store.onLoad = func() { closeFeed() } + } + opts := []eventfeed.Option{ + eventfeed.WithCheckpointStore(store), + eventfeed.WithConsumerNamespace("agent"), + eventfeed.WithObserver(tc.observer(closeFeed, forbidden)), + } + if tc.presentAt { + opts = append(opts, eventfeed.WithStart(eventfeed.StartPresent())) + } + h = newHarness(t, opts...) + h.minter.ScriptTicket(ticket(1)) + h.polls.ScriptPage(eventfeed.PollPage{ + Events: []eventfeed.Event{pollEvent(101)}, + Position: "pos-1", + }) + h.start() + if tc.notAfter != "connecting" { + sock := h.driveToSubscribed() + sock.Serve(frameConfirm(noFilterIdentifier)) + } + h.join() + + mu.Lock() + defer mu.Unlock() + if len(seen) != 0 { + t.Errorf("%q fired after Close; §23 makes close() a universal edge from every non-absorbing state", tc.notAfter) + } + if _, terminal, _ := h.snapshot(); terminal != nil { + t.Errorf("terminal = %v, want none — Close ends the iterator with no error element", terminal) + } + }) + } +} + +// TestCloseDuringPageDeliveredFinishesThePageAndStops is the other half of the +// ordering above, and the half that must NOT be a suppression. +// +// A Close taken from Observer.PageDelivered ends the walk — no second poll, +// whatever `next` says. But the page it interrupted is finished first: its +// events were already handed to the consumer, so its position is written. The +// check sits after the save for exactly that reason, and a check placed before +// it would reintroduce the silent dropped save the durable gate was deleted +// for. +func TestCloseDuringPageDeliveredFinishesThePageAndStops(t *testing.T) { + store := &hookedStore{} + var h *harness + h = newHarness(t, + eventfeed.WithCheckpointStore(store), + eventfeed.WithConsumerNamespace("agent"), + eventfeed.WithObserver(eventfeed.Observer{ + PageDelivered: func(int, string) { + if err := h.conn.Close(); err != nil { + t.Errorf("Close: %v", err) + } + }, + })) + h.minter.ScriptTicket(ticket(1)) + h.polls.ScriptPage(eventfeed.PollPage{ + Events: []eventfeed.Event{pollEvent(101)}, + Position: "pos-1", + Next: testOrigin + "/999/events.json?after=101", + }) + h.polls.ScriptPage(eventfeed.PollPage{Events: []eventfeed.Event{pollEvent(102)}, Position: "pos-2"}) + h.start() + sock := h.driveToSubscribed() + sock.Serve(frameConfirm(noFilterIdentifier)) + h.join() + + if got := store.recorded(); len(got) != 1 || got[0] != "pos-1" { + t.Errorf("recorded saves = %v, want [pos-1] — event 101 was delivered, so its position must be durable", got) + } + if got := h.polls.Calls(); len(got) != 1 { + t.Errorf("poll seam calls = %d, want 1 — Close must stop the walk before it follows `next`", len(got)) + } + assertIDs(t, h.deliveredIDs(), 101) +} + +// TestCloseDuringPageDeliveredSilencesThePageBoundary is what the page +// boundary's own checks buy that the walk's top-of-loop check does not. +// +// The top-of-loop check is later, and between it and PageDelivered sit two +// passes that DISPATCH what they dequeue: the ownership cut on a present-class +// entry, and socketCheck before following `next`. With a disconnect frame +// queued when the consumer closes, either announces `disconnected` — a +// teardown report for a socket the consumer had already stopped caring about, +// after the universal Closed edge was taken. +// +// Both entry classes, because they reach different passes and a single case +// would leave one of the two checks unexercised. That is not hypothetical: the +// first version of this test used the default harness, which resolves to +// PRESENT-class, so it drove only the ownership cut while claiming to be about +// socketCheck. +func TestCloseDuringPageDeliveredSilencesThePageBoundary(t *testing.T) { + for _, tc := range []struct { + name string + // stored gives the entry a position, which makes it position-resume + // and routes the page through socketCheck rather than the cut. + stored bool + }{ + {name: "present-class entry, at the ownership cut"}, + {name: "position-resume entry, at socketCheck", stored: true}, + } { + t.Run(tc.name, func(t *testing.T) { + var mu sync.Mutex + var disconnects int + var h *harness + var conn *feedtest.Conn + queued := make(chan struct{}, 8) + store := feedtest.NewStore() + if tc.stored { + store.Stored("pos-0") + } + h = newHarness(t, + eventfeed.WithCheckpointStore(store), + eventfeed.WithConsumerNamespace("agent"), + eventfeed.WithObserver(eventfeed.Observer{ + PageDelivered: func(int, string) { + // Served HERE, not during the poll: a frame arriving + // while the call is in flight is parked in the + // deferral slot, which neither pass looks at. This one + // goes into the pump's QUEUE, which is what they drain. + // + // The drain first is load-bearing: welcome and confirm + // have already handed off, so an undrained channel + // satisfies the wait below on a STALE token, and Close + // then lands before the disconnect is queued at all — + // leaving nothing for the code under test to find. + drain(queued) + conn.Serve(frameDisconnect("remote", true)) + select { + case <-queued: + case <-time.After(watchdog): + t.Error("the disconnect frame never reached the hand-off queue") + return + } + if err := h.conn.Close(); err != nil { + t.Errorf("Close: %v", err) + } + }, + Disconnected: func(string, error) { + mu.Lock() + defer mu.Unlock() + disconnects++ + }, + })) + h.conn.OnPumpHandedOff(func(bool) { + select { + case queued <- struct{}{}: + default: + } + }) + h.minter.ScriptTicket(ticket(1)) + h.polls.ScriptPage(eventfeed.PollPage{ + Events: []eventfeed.Event{pollEvent(101)}, + Position: "pos-1", + Next: testOrigin + "/999/events.json?after=101", + }) + h.polls.ScriptPage(eventfeed.PollPage{Position: "pos-2"}) + h.start() + conn = h.driveToSubscribed() + conn.Serve(frameConfirm(noFilterIdentifier)) + h.join() + + mu.Lock() + defer mu.Unlock() + if disconnects != 0 { + t.Errorf("Observer.Disconnected fired %d time(s) after Close; the page boundary must end the walk before its dispatching passes run", disconnects) + } + }) + } +} + +// TestCloseFromEveryCallbackSiteDoesNotDeadlock pins that Close is callable +// from inside a consumer callback, which is documented as supported and is the +// case that has to be right rather than merely likely. Every callback runs ON +// the run goroutine, and Close takes the connector's own mutex, so a Close that +// waited for anything the run goroutine still owed would deadlock against the +// very callback that made it. // -// The release therefore happens before either observer callback. Each site below -// ends the iteration cleanly with no error element; a deadlock fails as the -// watchdog rather than hanging the package. +// Each site below ends the iteration cleanly with no error element; a deadlock +// fails as the watchdog rather than hanging the package. func TestCloseFromEveryCallbackSiteDoesNotDeadlock(t *testing.T) { sites := []struct { name string diff --git a/go/pkg/basecamp/eventfeed/loop.go b/go/pkg/basecamp/eventfeed/loop.go index 1dce5b1d2f..9a202b6229 100644 --- a/go/pkg/basecamp/eventfeed/loop.go +++ b/go/pkg/basecamp/eventfeed/loop.go @@ -712,6 +712,14 @@ func (l *loop) runCycle(delay time.Duration) cycleOutcome { // invariant: Minting's set is {}); mint cancellation rides the attempt // context. l.setState(stateMinting) + // Close outranks the announcement, not just the act. A Connecting callback + // after Close tells the host a connect attempt is beginning when §23's + // universal Closed edge has already ruled one out — and the check that used + // to be the only one here sat AFTER the callback, so it caught a Close + // taken INSIDE the callback and missed every Close taken before it. + if l.runCtx.Err() != nil { + return cycleOutcome{kind: outcomeClosed} + } if l.cfg.observer.Connecting != nil { l.cfg.observer.Connecting(l.failedCycles+1, delay) } @@ -1477,14 +1485,13 @@ func (l *loop) saveCheckpoint(position string) { if l.cfg.store == nil { return } - // The store call runs under the durable gate, so a save either commenced - // before Close or does not commence at all (durableGate). A refused save is - // silent by design: Close abandons, and reporting a save that was declined - // BECAUSE the consumer closed would be reporting the consumer's own - // decision back to it as a store failure. - if !l.cfg.durable.begin() { - return - } + // A position that reached here was ACCEPTED, and its events were delivered + // to the consumer before it was. It is written unconditionally, including + // after Close: the store is what tells the next run where those deliveries + // stopped, and dropping the write because the consumer closed a moment + // earlier silently re-delivers them. Ordering a second connector against + // this one is the consumer's to arrange, and Connector.Wait is what it + // arranges with. err := l.cfg.store.Save(l.runCtx, l.checkpointKey(), position) if err != nil { if l.cfg.observer.CheckpointSaveFailed != nil { diff --git a/go/pkg/basecamp/eventfeed/recovery.go b/go/pkg/basecamp/eventfeed/recovery.go index fd37c3c3ae..1d3fdb7910 100644 --- a/go/pkg/basecamp/eventfeed/recovery.go +++ b/go/pkg/basecamp/eventfeed/recovery.go @@ -44,6 +44,12 @@ import ( func (l *loop) repairWalk(at *attempt) (cycleOutcome, bool) { cursor, presentClass := l.repairCursor() l.setState(stateCatchingUp) + // Close outranks the announcement, on the repair path for the same reason + // as on the post-confirmation one: no walk is about to start. + if l.runCtx.Err() != nil { + l.disposeAttempt(at, nil) + return cycleOutcome{kind: outcomeClosed}, true + } if l.cfg.observer.CatchUpStarted != nil { l.cfg.observer.CatchUpStarted(redactCursor(l.cfg.origin, cursor)) } From 484ed587ee0313f1b9586db40b95cef195802dde Mon Sep 17 00:00:00 2001 From: Jeremy Daer Date: Tue, 18 Aug 2026 23:04:19 -0700 Subject: [PATCH 14/51] =?UTF-8?q?Event=20feed:=20the=20consumer=20surface?= =?UTF-8?q?=20gained=20a=20method,=20so=20=C2=A723=20gained=20its=20contra?= =?UTF-8?q?ct?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `wait()` is a public act with a cross-SDK obligation, not a Go convenience, so §23's Consumer Surface states it rather than leaving the other five SDKs to infer it from the Go source. What the section said about `close()` was true and incomplete in the same breath: it abandons, it is callable from anywhere — and it silently implied that closing was enough to hand a checkpoint store to a replacement connector. It is not, and no gate inside `close()` can make it so. So the section now says both halves: a save decided just before the close may be written just after, deliberately, because its events were already delivered; and `wait()` is the quiescence point, with the iteration terminating being the same guarantee for a consumer that owns the loop. The naming note says what has to exist rather than what has to be added. A language whose streaming idiom already exposes the run's completion — a Kotlin Job, a Swift Task — has `wait()` already; the requirement is that observing the run's exit be possible WITHOUT `close()`, since `close()` is the one call that cannot wait. --- SPEC.md | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/SPEC.md b/SPEC.md index 7aaaa7ab86..f339a5d99d 100644 --- a/SPEC.md +++ b/SPEC.md @@ -3224,7 +3224,18 @@ is shared: which keeps hosts and tests deterministic. - **`close()`** is idempotent and callable from any context: it abandons, never drains. Undelivered buffered events are abandoned — the next run re-serves from the last **usable** - checkpoint (see the exclusion under Entry Boundary). + checkpoint (see the exclusion under Entry Boundary). Cancellation is visible before it + returns; it does **not** wait for the run to unwind, because every consumer callback runs + on the run's own execution context and waiting there would deadlock on the caller. +- **`close()` does not order a second connector over the same checkpoint store, and cannot.** + A save decided just before the close may still be written just after — a store is entitled + to ignore the cancelled context, and the reference `FileCheckpointStore` does. That is + intended: the position was accepted and its events delivered before the close, so dropping + the write would silently re-deliver them. **`wait()`** is the quiescence point — it blocks + until the run has exited, after which no save can be in flight. A consumer that owns the + iteration needs nothing extra, since the iteration terminating is the same guarantee. + Await termination — or `wait()` — before opening a second connector over the same store. + `wait()` is not callable from a consumer callback, for the reason `close()` does not wait. - A consumer break takes the identical teardown path; the in-flight page's checkpoint is **not** saved. - All Observer callbacks fire on the consumer's execution context, never concurrently with a @@ -4093,6 +4104,13 @@ ships off. | Signal handler (none ⇒ default-terminal) | `WithSignalHandler` | `signalHandler?` | `signal_handler` | `signalHandler` | `signal_handler` | | Observer (none) | `WithObserver` | `observer?` | `observer` | `observer` | `observer` | +The two lifecycle methods take each language's native spelling of the same two acts: Go +`Close()` / `Wait()`, TypeScript `close()` / `wait()`, Python and Ruby `close` / `wait`, +Kotlin `close()` / `join()`, Swift `close()` / `wait()`. Where the language's streaming +idiom already exposes the run's completion — a Kotlin `Job`, a Swift `Task` — that handle +IS `wait()` and no second method is added; what must exist is a way to observe the run's +exit that is not `close()`. + The Observer is a struct of optional callbacks in the `httptrace.ClientTrace` style — extensible without breaking implementers: `connecting(attempt, delay)`, `connected()`, `confirmed()`, `disconnected(reason, error)`, `catch_up_started(cursor)`, From 187b58c599520e9064dc9bfaa150eed2a941091c Mon Sep 17 00:00:00 2001 From: Jeremy Daer Date: Tue, 18 Aug 2026 23:17:10 -0700 Subject: [PATCH 15/51] Conformance: the driver contract claims a strictness the Go driver does not have MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Copilot's two suppressed comments on this PR are right. The README states, normatively and for all six drivers, that checkpoint saves and outbound frames are ARRIVAL-strict — "observed while the current step is anything other than their matching expect step, the scenario fails immediately." The Go reference driver appends both to queues and lets the matching expect step consume whatever is there, with no record of which step was active on arrival and no field to record it against. That is the order-strict rule, which is what the parked classes get. The rule is not weakened to match the implementation, because the rule is normative for five drivers not yet written and this is one implementation behind it. What the README now carries is the gap itself, sized: early-SAVE ordering is not at risk — every save records the delivery count it arrived at, and fixture 12 fails a save that precedes its page's deliveries — so the residue is an outbound frame written earlier than its step, concretely a subscribe command sent at dial rather than on welcome. Not patched here because the fix is not the queue check it looks like. The rule's own second sentence is the hard part: an action that becomes legal the instant a rendezvous is satisfied must match the NEXT step rather than fail against the stale one, so the handoff between adjacent steps has to be atomic against the harness mutex. A naive active-step guard fails exactly there and would turn every rendezvous-adjacent action into a false failure. Filed as #792 with the red proof it needs: a connector that subscribes at dial passes all 23 fixtures today. --- conformance/event-feed/README.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/conformance/event-feed/README.md b/conformance/event-feed/README.md index 0629c17c46..18725c9541 100644 --- a/conformance/event-feed/README.md +++ b/conformance/event-feed/README.md @@ -45,6 +45,23 @@ The strict-match rule splits by action class; every driver implements exactly th unmatched when the script ends fails the scenario, and the seam-call counts in `finally` count it either way. +**The Go reference driver does not implement the arrival-strict half yet**, and +saying so here is the point: it queues saves and outbound frames and matches them +to their expect steps in ORDER, so an action that arrived during an earlier, +non-matching step still satisfies its expectation. The rule above is the contract +every driver is held to; this is a gap in one implementation, not a weakening of +it. The hard part is the atomic handoff the rule already names — an action that +becomes legal the instant a rendezvous is satisfied must match the next step +rather than fail against the stale one — which is why it is a tracked change +rather than a queue check. + +What that gap does NOT cover for is early-save ordering, which has its own +witness rather than relying on arrival strictness: every `save` records the +delivery count at the moment it arrived, and fixture 12 fails a save that +precedes its page's deliveries. The residue is narrower than the rule — an +outbound frame written earlier than its step, most concretely a subscribe +command sent before `welcome`. Tracked in #792. + ## Validation `schema.json` is the contract. `make event-feed-fixtures-check` validates the schema From 9e24666296f2be2ca7e50325a07744ca53dd82cb Mon Sep 17 00:00:00 2001 From: Jeremy Daer Date: Wed, 19 Aug 2026 12:52:50 -0700 Subject: [PATCH 16/51] Event feed: three leaks and a race the last pass reasoned past MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Four defects from daybreak's review, each one a place where an argument I made correctly in one spot was not carried to the next. **A seam-authored DialPolicy verdict reached the terminal element.** The previous commit established that *DialError and *TerminalError are exported, seam-constructible, and render free text, and used that to strip them from Observer.Disconnected — then exempted the terminal channel on the grounds that "terminal elements keep their SPEC-mandated text". That is true of the one case §23 actually mandates, filter_invalid, and false as a blanket. A DialPolicy error comes out of CableTransport.Dial, a documented extension point, and its Reason and cause went verbatim into the terminal's Msg and chain. The KIND is still read; the text is now the connector's own and no cause is retained. The connector's own pre-check keeps its text and is not the same case: its reasons are written in transport.go, and its two interpolations are a URL scheme and a port, which url.Parse constrains to alphanumerics and digits. **CloseError passed through with its peer Reason intact.** The comment said "renders only its code — an integer cannot carry a credential", which is true of Error() and irrelevant to the struct: Reason is exported and held the peer's string, so %+v, a structured logger, or ce.Reason put it on the surface the closed vocabulary exists to protect. Rendering is not the only way out of a struct. It is rebuilt code-only now. **The accepted-page save was handed a cancelled context.** Deleting the durable gate was supposed to make an accepted position durable even when Close lands at the save boundary. It passed l.runCtx, which Close cancels synchronously, so a store that HONORS its context — which CheckpointStore permits and says nothing against — returns ctx.Err() and the position is lost, exactly as under the gate. Both stores this package ships ignore ctx, which is why every test passed: the guarantee was tested only against implementations that could not fail it. context.WithoutCancel keeps the run's values and drops the cancellation. The trade is named in the code: a store that blocks forever now blocks the run's exit rather than being released by Close, which is the lesser harm, and the narrower one, since a store may already ignore ctx entirely. **The fatal scan could sample inside the pump's read window.** The carve-out is stated over "every frame the pump had ALREADY READ", and pumpDepth+1 delivers that only for a frame blocked inside the hand-off with a full queue. Between ReadFrame returning and the send starting, the frame is invisible: the scan sees an empty queue, completes the drain, saves the held position and announces caught_up with the fatal sitting in the pump's stack. Red-proven with a new pumpRead hook that parks the pump in exactly that window — checkpoint ledger [pos-1] where the carve-out requires none. Peeking at what the pump holds would move the race one instruction earlier rather than close it, so the scan's completion condition changed instead: an empty queue ends the scan only when the pump also holds nothing it has read. That second half is an atomic flag covering a stretch with no I/O and no blocking except the hand-off, which the scan's own dequeue releases, so the spin terminates and the budget bounds it regardless. Mutation: all four die, each on its own test and no other. The fourth needed its mutant rewritten — removing the guard alone left `runtime` unused, and a vet failure is not a kill. --- go/pkg/basecamp/eventfeed/catchup.go | 20 ++++- go/pkg/basecamp/eventfeed/catchup_test.go | 82 ++++++++++++++++++++ go/pkg/basecamp/eventfeed/connector.go | 6 ++ go/pkg/basecamp/eventfeed/connector_test.go | 73 ++++++++++++++++++ go/pkg/basecamp/eventfeed/export_test.go | 6 ++ go/pkg/basecamp/eventfeed/loop.go | 68 +++++++++++++++-- go/pkg/basecamp/eventfeed/loop_test.go | 85 +++++++++++++++++++++ 7 files changed, 333 insertions(+), 7 deletions(-) diff --git a/go/pkg/basecamp/eventfeed/catchup.go b/go/pkg/basecamp/eventfeed/catchup.go index a07b4b431e..57fde7de05 100644 --- a/go/pkg/basecamp/eventfeed/catchup.go +++ b/go/pkg/basecamp/eventfeed/catchup.go @@ -3,6 +3,7 @@ package eventfeed import ( "errors" "fmt" + "runtime" "time" ) @@ -868,7 +869,24 @@ func (l *loop) fatalScan(at *attempt, budget *int) (cycleOutcome, bool) { l.deferForDrain(&deferredFrame{item: item}) } default: - return cycleOutcome{}, false + // An empty queue is proof there is nothing to find ONLY if the + // pump also holds nothing it has already read. The two are not + // the same instant: the pump sets `holding` when ReadFrame + // returns and clears it when the item is in the queue, and a scan + // sampling between those points sees an empty queue with the + // frame — possibly the fatal one — sitting in the pump's own + // stack. The carve-out is stated over what the pump has READ, so + // the scan waits out that window rather than reading it as quiet. + // + // It terminates. The interval `holding` covers contains no I/O + // and no blocking except the hand-off itself, and a hand-off can + // only block on a FULL queue — which is not this branch, since a + // full queue is dequeued above and each dequeue releases it. The + // budget bounds the loop either way. + if !at.lc.holding.Load() { + return cycleOutcome{}, false + } + runtime.Gosched() } } return cycleOutcome{}, false diff --git a/go/pkg/basecamp/eventfeed/catchup_test.go b/go/pkg/basecamp/eventfeed/catchup_test.go index 01f65db271..f07841f102 100644 --- a/go/pkg/basecamp/eventfeed/catchup_test.go +++ b/go/pkg/basecamp/eventfeed/catchup_test.go @@ -15,6 +15,7 @@ import ( "slices" "strings" "sync" + "sync/atomic" "testing" "time" @@ -2041,3 +2042,84 @@ func TestDeferredFatalOutranksAMalformedPage(t *testing.T) { } assertPositions(t, store.Saves()) } + +// TestProtocolFatalReadButNotYetHandedOffIsImmediate closes the last gap in +// the carve-out's stated boundary. +// +// The guarantee is over "every frame the pump had ALREADY READ", and the +// pumpDepth+1 budget delivers that only for a frame blocked INSIDE the +// hand-off with a full queue — Go moves a blocked sender's value into the +// buffer on the first receive, so the budget's +1 reaches it. It says nothing +// about the window between ReadFrame returning and the send starting. A scan +// sampling inside that window sees an empty queue, concludes there is nothing +// to find, and completes the drain: the held entry position saves and +// caught_up announces, with the fatal frame sitting in the pump's own stack. +// +// The window is short but it is not theoretical — it is a preemption point +// like any other, and the drain's scan runs on every replay iteration, so the +// last iteration is one scheduling decision away from missing it. +// +// It cannot be closed by peeking at what the pump holds: the pump would still +// have to publish it, leaving the same race one instruction earlier. So the +// scan's completion condition changes instead, from "the queue is momentarily +// empty" to "the queue is empty AND the pump holds nothing it has read". The +// second half is an atomic flag the pump sets before the hand-off and clears +// after it, and the scan spins on it — bounded, because the flag covers a +// stretch of code with no I/O and no blocking except the hand-off itself, +// which the scan's own dequeue releases. +func TestProtocolFatalReadButNotYetHandedOffIsImmediate(t *testing.T) { + var caughtUp int + store := feedtest.NewStore() + h := storedHarness(t, store, eventfeed.WithObserver(eventfeed.Observer{ + CaughtUp: func() { caughtUp++ }, + })) + h.minter.ScriptTicket(ticket(1)) + h.polls.ScriptPage(eventfeed.PollPage{Position: "pos-1"}) + h.drainHandled() + + // The pump parks in the read→hand-off window on the fatal frame, and only + // on it: parking every read would stall the handshake. + parked := make(chan struct{}, 1) + release := make(chan struct{}) + var armed atomic.Bool + h.conn.OnPumpRead(func() { + if !armed.Load() { + return + } + armed.Store(false) + select { + case parked <- struct{}{}: + default: + } + <-release + }) + h.pauseAfter = 1 + h.start() + + conn := h.driveToSubscribed() + h.serveSettled(conn, frameMessage(noFilterIdentifier, 41)) + conn.Serve(frameConfirm(noFilterIdentifier)) + + // The consumer parks inside the drain's delivery of the retained event, so + // the drain is mid-flight when the fatal frame is read. + h.waitUntil("the drain parked mid-delivery", func() bool { return len(h.deliveredIDs()) == 1 }) + armed.Store(true) + conn.Serve(frameDisconnect("invalid_event_stream_command", false)) + select { + case <-parked: + case <-time.After(watchdog): + t.Fatal("the pump never parked between reading the fatal frame and handing it off") + } + + // Resume the drain with the fatal frame READ and unqueued. The scan must + // not treat the empty queue as proof there is nothing to find. + h.resume() + // Give the scan a moment to reach its completion condition while the pump + // is still parked; then let the hand-off proceed. + time.Sleep(50 * time.Millisecond) + close(release) + h.join() + + assertProtocolFatalDrain(t, h, store, caughtUp) + assertIDs(t, h.deliveredIDs(), 41) +} diff --git a/go/pkg/basecamp/eventfeed/connector.go b/go/pkg/basecamp/eventfeed/connector.go index fd17ae4640..3a32e9faab 100644 --- a/go/pkg/basecamp/eventfeed/connector.go +++ b/go/pkg/basecamp/eventfeed/connector.go @@ -230,6 +230,12 @@ type testHooks struct { // observable well before the pump's release runs, and only the release // settles which window a subsequent expiry is evaluated against. pumpReleased func() + // pumpRead fires immediately after a pump read returns and after the + // holding flag is set, but BEFORE the hand-off — the one window in which + // the pump owns a frame no other goroutine can observe through the queue. + // It exists so the protocol-fatal scan's rendezvous with that window can + // be driven deterministically instead of raced. + pumpRead func() // pumpHandedOff fires once an item reaches the hand-off queue, reporting // whether it is the pump's terminating error. pumpHandedOff func(isErr bool) diff --git a/go/pkg/basecamp/eventfeed/connector_test.go b/go/pkg/basecamp/eventfeed/connector_test.go index 01b8189920..5f4fe64d86 100644 --- a/go/pkg/basecamp/eventfeed/connector_test.go +++ b/go/pkg/basecamp/eventfeed/connector_test.go @@ -1412,3 +1412,76 @@ func TestCloseFromConnectedOutranksAQueuedFatalFrame(t *testing.T) { terminals, rounds) } } + +// ctxAwareStore is a CheckpointStore that HONORS its context, which the +// interface permits and this package's own tests never exercised: both +// feedtest.Store and the built-in FileCheckpointStore ignore ctx, so a save +// handed a cancelled context still landed and every assertion passed. +type ctxAwareStore struct { + mu sync.Mutex + saves []string +} + +func (s *ctxAwareStore) Load(context.Context, eventfeed.CheckpointKey) (string, bool, error) { + return "pos-0", true, nil +} + +func (s *ctxAwareStore) Save(ctx context.Context, _ eventfeed.CheckpointKey, position string) error { + if err := ctx.Err(); err != nil { + return err + } + s.mu.Lock() + defer s.mu.Unlock() + s.saves = append(s.saves, position) + return nil +} + +func (s *ctxAwareStore) recorded() []string { + s.mu.Lock() + defer s.mu.Unlock() + return append([]string(nil), s.saves...) +} + +// TestAcceptedPositionSavesAgainstAContextHonoringStore is what +// TestAnAcceptedPositionSavesEvenWhenCloseLandsFirst could not see. +// +// Deleting the durable gate was supposed to make an accepted page's position +// durable even when Close lands at the save boundary — its events were already +// delivered, so dropping the write silently re-delivers them. But the save was +// still handed l.runCtx, which Close cancels synchronously. A store that +// ignores ctx (both of the ones this package ships) writes anyway; a store that +// HONORS it returns ctx.Err() and the position is lost — the exact outcome the +// gate was deleted to prevent, reached by a different route and invisible to +// every existing test. +// +// CheckpointStore.Save takes a context and nothing in its contract says to +// ignore it, so honoring it is compliant. The save therefore runs under a +// context detached from the run's cancellation. +func TestAcceptedPositionSavesAgainstAContextHonoringStore(t *testing.T) { + store := &ctxAwareStore{} + var h *harness + h = newHarness(t, + eventfeed.WithCheckpointStore(store), + eventfeed.WithConsumerNamespace("agent"), + eventfeed.WithObserver(eventfeed.Observer{ + PageDelivered: func(int, string) { + if err := h.conn.Close(); err != nil { + t.Errorf("Close: %v", err) + } + }, + })) + h.minter.ScriptTicket(ticket(1)) + h.polls.ScriptPage(eventfeed.PollPage{ + Events: []eventfeed.Event{pollEvent(101)}, + Position: "pos-1", + }) + h.start() + sock := h.driveToSubscribed() + sock.Serve(frameConfirm(noFilterIdentifier)) + h.join() + + if got := store.recorded(); len(got) != 1 || got[0] != "pos-1" { + t.Errorf("recorded saves = %v, want [pos-1] — event 101 was delivered, so its position must be durable "+ + "even against a store that honors the cancelled run context", got) + } +} diff --git a/go/pkg/basecamp/eventfeed/export_test.go b/go/pkg/basecamp/eventfeed/export_test.go index 9238953b41..11eeda9c50 100644 --- a/go/pkg/basecamp/eventfeed/export_test.go +++ b/go/pkg/basecamp/eventfeed/export_test.go @@ -63,6 +63,12 @@ func (c *Connector) OnPumpReleased(f func()) { c.hooks.pumpReleased = f } // rendezvous for "the socket's death is queued for the state machine". func (c *Connector) OnPumpHandedOff(f func(isErr bool)) { c.hooks.pumpHandedOff = f } +// OnPumpRead registers a hook fired after a pump read returns and before its +// hand-off — the window in which a frame is READ but not yet observable +// through the queue. Parking here is the only deterministic way to put the +// protocol-fatal scan and an already-read frame in the same instant. +func (c *Connector) OnPumpRead(f func()) { c.hooks.pumpRead = f } + // OnFrameDeferred registers a hook fired when the in-flight-poll servicing // parks one receive — always a socket outcome — for the walk's next dispatch // point. It is the rendezvous for "the walk is now waiting on nothing but the diff --git a/go/pkg/basecamp/eventfeed/loop.go b/go/pkg/basecamp/eventfeed/loop.go index 9a202b6229..0c115ea771 100644 --- a/go/pkg/basecamp/eventfeed/loop.go +++ b/go/pkg/basecamp/eventfeed/loop.go @@ -6,6 +6,7 @@ import ( "fmt" "strconv" "sync" + "sync/atomic" "time" ) @@ -138,6 +139,13 @@ type liveConn struct { frames chan pumpItem stale *staleHolder hooks testHooks + // holding reports that the pump has READ a frame it has not yet handed + // off. It is what makes "every frame the pump has already read" an + // observable condition rather than an aspiration: without it, the window + // between ReadFrame returning and the channel send is invisible, and a + // non-blocking scan that samples inside it concludes the queue is empty + // when a frame — possibly the protocol-fatal one — has already arrived. + holding atomic.Bool } // newLiveConn arms staleness (socket open) and then starts the pump, in that @@ -164,12 +172,22 @@ func (lc *liveConn) pump(ctx context.Context) { defer close(lc.frames) for { data, err := lc.conn.ReadFrame(ctx) + // Set BEFORE anything else the read leads to, and cleared only once + // the item is in the queue: the flag must cover the whole interval in + // which this goroutine holds a frame no other goroutine can see. + lc.holding.Store(true) + if lc.hooks.pumpRead != nil { + lc.hooks.pumpRead() + } if err != nil { lc.handOff(ctx, pumpItem{err: err}) + lc.holding.Store(false) return } lc.stale.reset() - if !lc.handOff(ctx, pumpItem{data: data}) { + handed := lc.handOff(ctx, pumpItem{data: data}) + lc.holding.Store(false) + if !handed { return } } @@ -775,8 +793,21 @@ func (l *loop) runCycle(delay time.Duration) cycleOutcome { hs.Stop() var derr *DialError if errors.As(r.err, &derr) && derr.Kind == DialPolicy { + // The KIND is read; the text is not. This DialError came out + // of CableTransport.Dial, a documented extension point, so + // its Reason and its cause are host-authored — and the dialed + // URL carries the ticket in its query string. Copying either + // into the terminal would hand it to every consumer through + // the channel Observer.Disconnected was hardened for, so the + // message is the connector's own and no cause is retained. + // + // The pre-check above keeps its text and is not the same + // case: its reasons are written in transport.go, and its two + // interpolations are a URL scheme and a port, which url.Parse + // constrains to alphanumerics and digits respectively. return cycleOutcome{kind: outcomeTerminal, term: &TerminalError{ - Reason: ReasonInvalidCableURL, Msg: derr.Reason, Err: derr, + Reason: ReasonInvalidCableURL, + Msg: "the cable transport refused the mint's url as a policy violation", }} } return cycleOutcome{kind: outcomeFailed} // transition 7 @@ -1337,9 +1368,14 @@ func observableSocketError(err error) error { } var ce *CloseError if errors.As(err, &ce) { - // Flat by construction and renders only its code — an integer cannot - // carry a credential. See CloseError.Error. - return ce + // REBUILT code-only, not passed along. CloseError.Error renders the + // integer alone, which is what made passing the value through look + // safe — but Reason is an exported field holding the peer's string, + // and it survives the rendering untouched. A host that logs the + // error's fields, formats it with %+v, or simply reads ce.Reason gets + // the peer's text on the surface this vocabulary exists to protect. + // Rendering is not the only way out of a struct. + return &CloseError{Code: ce.Code} } var ife *invalidFrameError if errors.As(err, &ife) { @@ -1492,7 +1528,27 @@ func (l *loop) saveCheckpoint(position string) { // earlier silently re-delivers them. Ordering a second connector against // this one is the consumer's to arrange, and Connector.Wait is what it // arranges with. - err := l.cfg.store.Save(l.runCtx, l.checkpointKey(), position) + // + // "Unconditionally" requires DETACHING the context, and passing l.runCtx + // here made the guarantee empty for half the stores that could implement + // it. Close cancels runCtx synchronously, so a store that honors its + // context — which CheckpointStore's contract permits, and says nothing + // against — receives a cancelled one and returns ctx.Err(). The position + // is then lost exactly as it was under the durable gate, by a different + // route. Neither store this package ships honors ctx, which is why every + // test passed: the defect was invisible to the only implementations it was + // tested against. + // + // WithoutCancel keeps the run's context VALUES — a store carrying a trace + // or a tenant on the context still sees them — and drops only the + // cancellation. The trade is that a store which blocks forever now blocks + // the run's exit, and therefore Wait, rather than being released by Close. + // That is the lesser harm and the narrower one: a store is already + // entitled to ignore ctx entirely (the built-in FileCheckpointStore + // documents that it does), so a blocking store could already do this, + // while a silently dropped position is unbounded re-delivery with nothing + // recording that it happened. + err := l.cfg.store.Save(context.WithoutCancel(l.runCtx), l.checkpointKey(), position) if err != nil { if l.cfg.observer.CheckpointSaveFailed != nil { l.cfg.observer.CheckpointSaveFailed(err) diff --git a/go/pkg/basecamp/eventfeed/loop_test.go b/go/pkg/basecamp/eventfeed/loop_test.go index 6ba0b4befd..53d6c9420b 100644 --- a/go/pkg/basecamp/eventfeed/loop_test.go +++ b/go/pkg/basecamp/eventfeed/loop_test.go @@ -2083,3 +2083,88 @@ func TestDisconnectObserverNeverEchoesPeerText(t *testing.T) { }) } } + +// TestSeamAuthoredDialPolicyDoesNotReachTheTerminal is the terminal-channel +// half of the ticket-secrecy rule, and the half an earlier pass exempted. +// +// The reasoning that exempted it was "terminal elements keep their typed +// errors and their SPEC-mandated text", which is true of the ONE case §23 +// actually mandates — filter_invalid preserves the server's message — and +// false as a blanket. A DialPolicy verdict is not SPEC-mandated text: it is +// authored by whatever CableTransport the host installed, and CableTransport +// is a documented extension point. Copying its Reason into TerminalError.Msg +// and the error itself into the chain hands that text to every consumer, +// through exactly the channel that was hardened for Observer.Disconnected. +// +// The connector's OWN pre-check keeps its text and is not affected: its two +// interpolations are a URL scheme and a port, both of which url.Parse +// constrains to alphanumerics and digits, so neither can carry a credential. +func TestSeamAuthoredDialPolicyDoesNotReachTheTerminal(t *testing.T) { + const canary = "sekrit-ticket-value" + cableURL := "wss://28.cable.basecamp.com/cable?ticket=" + canary + + for _, tc := range []struct { + name string + err error + }{ + {"reason", &eventfeed.DialError{Kind: eventfeed.DialPolicy, Reason: cableURL}}, + {"cause", &eventfeed.DialError{Kind: eventfeed.DialPolicy, Err: errors.New("dial " + cableURL)}}, + {"wrapped", fmt.Errorf("transport: %w", &eventfeed.DialError{Kind: eventfeed.DialPolicy, Reason: cableURL})}, + } { + t.Run(tc.name, func(t *testing.T) { + h := newHarness(t) + h.minter.ScriptTicket(ticket(1)) + h.tr.FailNextDial(tc.err) + h.start() + h.join() + + _, terminal, _ := h.snapshot() + if terminal == nil || terminal.Reason != eventfeed.ReasonInvalidCableURL { + t.Fatalf("terminal = %v, want reason %q", terminal, eventfeed.ReasonInvalidCableURL) + } + // The whole chain, not just the top: Error() walks it, and so does + // any consumer that logs %+v or unwraps. + for err := error(terminal); err != nil; err = errors.Unwrap(err) { + if strings.Contains(err.Error(), canary) { + t.Errorf("%T in the terminal chain echoed the ticket: %q", err, err.Error()) + } + if strings.Contains(err.Error(), "ticket=") { + t.Errorf("%T in the terminal chain echoed a ticket-bearing query: %q", err, err.Error()) + } + } + // A seam-authored DialError must not be recoverable from the + // chain at all: retaining it would hand back through errors.As + // exactly what the rendering withheld. + var de *eventfeed.DialError + if errors.As(error(terminal), &de) { + t.Errorf("the terminal retains the seam's *DialError (%+v); the chain is the same leak as the rendering", de) + } + }) + } +} + +// TestObservableCloseErrorCarriesNoPeerReason: CloseError.Error renders only +// the integer code, which is why the reduction let the value through — but +// Reason is an EXPORTED field holding peer text, and it survived intact. A +// host that logs the error's fields, formats it with %+v, or reads ce.Reason +// gets the peer's string on the surface the closed vocabulary exists to +// protect. Rendering is not the only way out of a struct. +func TestObservableCloseErrorCarriesNoPeerReason(t *testing.T) { + const canary = "sekrit-ticket-value" + peerText := "wss://28.cable.basecamp.com/cable?ticket=" + canary + + got := eventfeed.ExportObservableSocketError(&eventfeed.CloseError{Code: 1011, Reason: peerText}) + var ce *eventfeed.CloseError + if !errors.As(got, &ce) { + t.Fatalf("reduced to %T, want a *CloseError — the code is diagnostic and must survive", got) + } + if ce.Reason != "" { + t.Errorf("the reduced CloseError still carries the peer's reason %q", ce.Reason) + } + if ce.Code != 1011 { + t.Errorf("the reduced CloseError lost its code: %d", ce.Code) + } + if fmt.Sprintf("%+v", ce) != fmt.Sprintf("%+v", &eventfeed.CloseError{Code: 1011}) { + t.Errorf("field-wise rendering still differs from a code-only close error: %+v", ce) + } +} From ea4672e6f9260df683fd85832dc234b624d962d5 Mon Sep 17 00:00:00 2001 From: Jeremy Daer Date: Wed, 19 Aug 2026 12:54:01 -0700 Subject: [PATCH 17/51] Event feed: the save's context is contract, not an implementation detail MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit §23 said a save decided just before close "may still be written just after" because "a store is entitled to ignore the cancelled context" — which describes the accident that made the Go implementation work rather than the rule the other five SDKs need. A store that HONORS its context is equally compliant, and against one the guarantee evaporates: the position for events already delivered is dropped, silently, and the next run re-delivers them. So the rule is now the detached context, stated with its trade: values carried, cancellation dropped, and a store that blocks indefinitely delays the run's exit — and `wait()` — instead of being released by `close()`. Bounded by the store's own behavior, against re-delivery that is bounded by nothing. --- SPEC.md | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/SPEC.md b/SPEC.md index f339a5d99d..e9784b9f8c 100644 --- a/SPEC.md +++ b/SPEC.md @@ -3228,10 +3228,16 @@ is shared: returns; it does **not** wait for the run to unwind, because every consumer callback runs on the run's own execution context and waiting there would deadlock on the caller. - **`close()` does not order a second connector over the same checkpoint store, and cannot.** - A save decided just before the close may still be written just after — a store is entitled - to ignore the cancelled context, and the reference `FileCheckpointStore` does. That is - intended: the position was accepted and its events delivered before the close, so dropping - the write would silently re-deliver them. **`wait()`** is the quiescence point — it blocks + A save decided just before the close is still written after it. That is intended: the + position was accepted and its events delivered before the close, so dropping the write + would silently re-deliver them. **The save therefore runs under a context detached from + the run's cancellation** — carrying the run's context values, but not its cancellation. + Passing the live run context instead makes the guarantee conditional on the store: one + that ignores its context writes anyway, and one that honors it — which this contract + permits, and says nothing against — sees a cancelled context and drops the position. The + trade is explicit: a store that blocks indefinitely delays the run's exit, and therefore + `wait()`, rather than being released by `close()`. That is bounded by the store's own + behavior, where a dropped position is unbounded re-delivery with nothing recording it. **`wait()`** is the quiescence point — it blocks until the run has exited, after which no save can be in flight. A consumer that owns the iteration needs nothing extra, since the iteration terminating is the same guarantee. Await termination — or `wait()` — before opening a second connector over the same store. From d3a30f9be2621a98050067cb20b76f077561930b Mon Sep 17 00:00:00 2001 From: Jeremy Daer Date: Wed, 19 Aug 2026 15:25:54 -0700 Subject: [PATCH 18/51] Event feed: a sampled flag cannot make a happens-before, so the boundary moves MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous commit claimed the fatal scan covered "every frame the pump had ALREADY READ" and added an atomic flag to deliver it. daybreak is right that it does not: the flag is published AFTER the read returns, leaving a pre-publish window, and the scan loads it AFTER its own empty select, leaving a second window in which the frame lands and the flag clears between the two reads. Two samples are not a happens-before, and the hook I added to test it sat after the store, so it excluded both interleavings it was supposed to catch. Closing the window for real needs the read to complete inside a critical section the scan can enter. The read blocks indefinitely on a quiet socket, so that lock deadlocks the drain against a peer that stopped talking — which is why this is a narrowed boundary rather than a third sample. So the flag, its hook, and its test are gone, and §23 now states what every implementation must actually cover: everything HANDED to the state machine, plus the one frame a blocked hand-off is holding. "Observed" was always the spec's word; the reader is a single goroutine, so the plus-one is exactly one, and it is why the budget is pump depth + 1. The test drives the guarantee through the hand-off rendezvous rather than through a window no scan can see. Two more from the same review, both current-head: A dial returning (conn, err) leaked the connection — legal in Go, and easy for a transport that fails mid-handshake — which under the reconnect cycle is a leak per backoff round. It is closed before the error is classified. A (nil, nil) return now takes the transient failure edge instead of installing a nil CableConn and panicking on the first read. The detached save's test could not tell WithoutCancel from Background: both survive cancellation, and only the first keeps the caller's context values — a trace span, a tenant, a request id. The store now records what it saw and the test asserts it, which kills the Background mutant for that reason alone. doc.go still said FOUNDATIONS ONLY with no Connector and no run loop. It names the one piece genuinely outstanding instead: the Layer-1 adapters, and the two obligations that ride on them rather than on anything here. --- SPEC.md | 12 ++++ go/pkg/basecamp/eventfeed/catchup.go | 39 ++++++----- go/pkg/basecamp/eventfeed/catchup_test.go | 73 ++++++++------------- go/pkg/basecamp/eventfeed/connector.go | 6 -- go/pkg/basecamp/eventfeed/connector_test.go | 30 ++++++++- go/pkg/basecamp/eventfeed/doc.go | 35 +++++----- go/pkg/basecamp/eventfeed/export_test.go | 6 -- go/pkg/basecamp/eventfeed/loop.go | 36 +++++----- go/pkg/basecamp/eventfeed/loop_test.go | 9 ++- 9 files changed, 131 insertions(+), 115 deletions(-) diff --git a/SPEC.md b/SPEC.md index e9784b9f8c..f13012c00b 100644 --- a/SPEC.md +++ b/SPEC.md @@ -3460,6 +3460,18 @@ Draining is Terminal(`protocol_fatal`) immediately (the state-generic rule under Disconnect Dispatch) — the drain is not completed, the held entry position is NOT saved, and no `caught_up` is announced; only recoverable failures defer. +**"Observed" means handed to the state machine, and the boundary is normative.** A frame +the transport reader has taken off the socket but not yet handed over is not observed, and +no implementation is required to find it. That is not a tolerance granted for convenience: +making it observable requires the read to complete inside a critical section the scan can +enter, and the read blocks indefinitely on a quiet socket, so the lock deadlocks the drain +against a peer that simply stopped talking. Sampling a flag the reader sets after its read +does not close it either — the flag is published after the read returns, and the scan reads +it after its own check of the queue, so a frame can arrive and the flag clear between the +two. What every implementation MUST cover is everything handed over, **plus the one frame a +blocked hand-off is holding** — the reader is a single goroutine, so there is exactly one, +and it is why the scan's budget is pump depth + 1 rather than pump depth. + ### Disconnect Dispatch `[conformance]` Action Cable's `disconnect` is a **text frame**, not a WebSocket close frame, and stock diff --git a/go/pkg/basecamp/eventfeed/catchup.go b/go/pkg/basecamp/eventfeed/catchup.go index 57fde7de05..fa893687fa 100644 --- a/go/pkg/basecamp/eventfeed/catchup.go +++ b/go/pkg/basecamp/eventfeed/catchup.go @@ -3,7 +3,6 @@ package eventfeed import ( "errors" "fmt" - "runtime" "time" ) @@ -869,24 +868,28 @@ func (l *loop) fatalScan(at *attempt, budget *int) (cycleOutcome, bool) { l.deferForDrain(&deferredFrame{item: item}) } default: - // An empty queue is proof there is nothing to find ONLY if the - // pump also holds nothing it has already read. The two are not - // the same instant: the pump sets `holding` when ReadFrame - // returns and clears it when the item is in the queue, and a scan - // sampling between those points sees an empty queue with the - // frame — possibly the fatal one — sitting in the pump's own - // stack. The carve-out is stated over what the pump has READ, so - // the scan waits out that window rather than reading it as quiet. + // The scan's boundary is what the STATE MACHINE can observe, and + // observation happens at hand-off. A frame the pump has read but + // not yet handed off is not reachable from here at any price: + // closing that window needs the read itself to complete inside a + // critical section the scan can enter, and the read blocks + // indefinitely on a quiet socket, so the lock would deadlock the + // drain against a peer that simply stopped talking. // - // It terminates. The interval `holding` covers contains no I/O - // and no blocking except the hand-off itself, and a hand-off can - // only block on a FULL queue — which is not this branch, since a - // full queue is dequeued above and each dequeue releases it. The - // budget bounds the loop either way. - if !at.lc.holding.Load() { - return cycleOutcome{}, false - } - runtime.Gosched() + // A sampled flag does not close it either, and one was tried: the + // pump publishing "I hold a frame" after the read leaves the + // pre-publish window, and the scan loading that flag after its own + // empty select leaves a second window where the frame arrives and + // the flag clears between the two reads. Two samples cannot make + // a happens-before, and a mechanism that narrows a race while the + // spec claims it closed is worse than the honest boundary. + // + // So the boundary is stated where it holds: every frame handed + // off, plus the one a blocked hand-off is holding — which the + // pumpDepth+1 budget reaches, because Go moves a blocked sender's + // value into the buffer on the first receive. That is also what + // §23 means by "observed during Draining". + return cycleOutcome{}, false } } return cycleOutcome{}, false diff --git a/go/pkg/basecamp/eventfeed/catchup_test.go b/go/pkg/basecamp/eventfeed/catchup_test.go index f07841f102..11544dcad3 100644 --- a/go/pkg/basecamp/eventfeed/catchup_test.go +++ b/go/pkg/basecamp/eventfeed/catchup_test.go @@ -15,7 +15,6 @@ import ( "slices" "strings" "sync" - "sync/atomic" "testing" "time" @@ -2043,31 +2042,29 @@ func TestDeferredFatalOutranksAMalformedPage(t *testing.T) { assertPositions(t, store.Saves()) } -// TestProtocolFatalReadButNotYetHandedOffIsImmediate closes the last gap in -// the carve-out's stated boundary. +// TestProtocolFatalHandedOffDuringADrainIsImmediate pins the carve-out at the +// boundary it can actually hold, and that boundary is narrower than an earlier +// revision of this file claimed. // -// The guarantee is over "every frame the pump had ALREADY READ", and the -// pumpDepth+1 budget delivers that only for a frame blocked INSIDE the -// hand-off with a full queue — Go moves a blocked sender's value into the -// buffer on the first receive, so the budget's +1 reaches it. It says nothing -// about the window between ReadFrame returning and the send starting. A scan -// sampling inside that window sees an empty queue, concludes there is nothing -// to find, and completes the drain: the held entry position saves and -// caught_up announces, with the fatal frame sitting in the pump's own stack. +// That revision said the guarantee covered "every frame the pump had ALREADY +// READ", and added an atomic flag the pump set after its read with the scan +// spinning on it. It does not work and cannot. The flag is published AFTER the +// read returns, leaving a pre-publish window; and the scan loads it AFTER its +// own empty select, leaving a second window in which the frame lands and the +// flag clears between the two reads. Two samples do not make a happens-before. +// Closing the window for real needs the read to complete inside a critical +// section the scan can enter, and the read blocks indefinitely on a quiet +// socket, so that lock deadlocks the drain against a peer that stopped talking. // -// The window is short but it is not theoretical — it is a preemption point -// like any other, and the drain's scan runs on every replay iteration, so the -// last iteration is one scheduling decision away from missing it. +// So the mechanism is gone and the boundary is stated where it holds: every +// frame HANDED OFF, plus the one a blocked hand-off is holding. That is what +// §23 means by "observed during Draining" — the state machine observes at +// hand-off — and it is what the pumpDepth+1 budget reaches, because Go moves a +// blocked sender's value into the buffer on the first receive. // -// It cannot be closed by peeking at what the pump holds: the pump would still -// have to publish it, leaving the same race one instruction earlier. So the -// scan's completion condition changes instead, from "the queue is momentarily -// empty" to "the queue is empty AND the pump holds nothing it has read". The -// second half is an atomic flag the pump sets before the hand-off and clears -// after it, and the scan spins on it — bounded, because the flag covers a -// stretch of code with no I/O and no blocking except the hand-off itself, -// which the scan's own dequeue releases. -func TestProtocolFatalReadButNotYetHandedOffIsImmediate(t *testing.T) { +// This drives the guarantee through the hand-off rendezvous rather than +// through a window no scan can see. +func TestProtocolFatalHandedOffDuringADrainIsImmediate(t *testing.T) { var caughtUp int store := feedtest.NewStore() h := storedHarness(t, store, eventfeed.WithObserver(eventfeed.Observer{ @@ -2077,21 +2074,12 @@ func TestProtocolFatalReadButNotYetHandedOffIsImmediate(t *testing.T) { h.polls.ScriptPage(eventfeed.PollPage{Position: "pos-1"}) h.drainHandled() - // The pump parks in the read→hand-off window on the fatal frame, and only - // on it: parking every read would stall the handshake. - parked := make(chan struct{}, 1) - release := make(chan struct{}) - var armed atomic.Bool - h.conn.OnPumpRead(func() { - if !armed.Load() { - return - } - armed.Store(false) + queued := make(chan struct{}, eventfeed.ExportPumpDepth) + h.conn.OnPumpHandedOff(func(bool) { select { - case parked <- struct{}{}: + case queued <- struct{}{}: default: } - <-release }) h.pauseAfter = 1 h.start() @@ -2101,23 +2089,18 @@ func TestProtocolFatalReadButNotYetHandedOffIsImmediate(t *testing.T) { conn.Serve(frameConfirm(noFilterIdentifier)) // The consumer parks inside the drain's delivery of the retained event, so - // the drain is mid-flight when the fatal frame is read. + // the drain is still in flight when the fatal frame is handed off. h.waitUntil("the drain parked mid-delivery", func() bool { return len(h.deliveredIDs()) == 1 }) - armed.Store(true) + drain(queued) conn.Serve(frameDisconnect("invalid_event_stream_command", false)) select { - case <-parked: + case <-queued: case <-time.After(watchdog): - t.Fatal("the pump never parked between reading the fatal frame and handing it off") + t.Fatal("the fatal frame never reached the hand-off queue") } - // Resume the drain with the fatal frame READ and unqueued. The scan must - // not treat the empty queue as proof there is nothing to find. + // Resumed with the fatal frame observable: the drain must not complete. h.resume() - // Give the scan a moment to reach its completion condition while the pump - // is still parked; then let the hand-off proceed. - time.Sleep(50 * time.Millisecond) - close(release) h.join() assertProtocolFatalDrain(t, h, store, caughtUp) diff --git a/go/pkg/basecamp/eventfeed/connector.go b/go/pkg/basecamp/eventfeed/connector.go index 3a32e9faab..fd17ae4640 100644 --- a/go/pkg/basecamp/eventfeed/connector.go +++ b/go/pkg/basecamp/eventfeed/connector.go @@ -230,12 +230,6 @@ type testHooks struct { // observable well before the pump's release runs, and only the release // settles which window a subsequent expiry is evaluated against. pumpReleased func() - // pumpRead fires immediately after a pump read returns and after the - // holding flag is set, but BEFORE the hand-off — the one window in which - // the pump owns a frame no other goroutine can observe through the queue. - // It exists so the protocol-fatal scan's rendezvous with that window can - // be driven deterministically instead of raced. - pumpRead func() // pumpHandedOff fires once an item reaches the hand-off queue, reporting // whether it is the pump's terminating error. pumpHandedOff func(isErr bool) diff --git a/go/pkg/basecamp/eventfeed/connector_test.go b/go/pkg/basecamp/eventfeed/connector_test.go index 5f4fe64d86..ccdd6d2546 100644 --- a/go/pkg/basecamp/eventfeed/connector_test.go +++ b/go/pkg/basecamp/eventfeed/connector_test.go @@ -1418,8 +1418,9 @@ func TestCloseFromConnectedOutranksAQueuedFatalFrame(t *testing.T) { // feedtest.Store and the built-in FileCheckpointStore ignore ctx, so a save // handed a cancelled context still landed and every assertion passed. type ctxAwareStore struct { - mu sync.Mutex - saves []string + mu sync.Mutex + saves []string + valueSeen []any } func (s *ctxAwareStore) Load(context.Context, eventfeed.CheckpointKey) (string, bool, error) { @@ -1433,15 +1434,30 @@ func (s *ctxAwareStore) Save(ctx context.Context, _ eventfeed.CheckpointKey, pos s.mu.Lock() defer s.mu.Unlock() s.saves = append(s.saves, position) + // WithoutCancel drops cancellation and KEEPS values, and the difference + // matters to real stores: a trace span, a tenant, or a request id rides on + // the context, and swapping in context.Background() to dodge cancellation + // would silently strip all of it. Recorded here so the distinction is + // asserted rather than assumed. + s.valueSeen = append(s.valueSeen, ctx.Value(saveProbeKey{})) return nil } +// saveProbeKey types the context value the run carries into the store. +type saveProbeKey struct{} + func (s *ctxAwareStore) recorded() []string { s.mu.Lock() defer s.mu.Unlock() return append([]string(nil), s.saves...) } +func (s *ctxAwareStore) values() []any { + s.mu.Lock() + defer s.mu.Unlock() + return append([]any(nil), s.valueSeen...) +} + // TestAcceptedPositionSavesAgainstAContextHonoringStore is what // TestAnAcceptedPositionSavesEvenWhenCloseLandsFirst could not see. // @@ -1475,7 +1491,7 @@ func TestAcceptedPositionSavesAgainstAContextHonoringStore(t *testing.T) { Events: []eventfeed.Event{pollEvent(101)}, Position: "pos-1", }) - h.start() + h.startCtx(context.WithValue(context.Background(), saveProbeKey{}, "probe-value")) sock := h.driveToSubscribed() sock.Serve(frameConfirm(noFilterIdentifier)) h.join() @@ -1484,4 +1500,12 @@ func TestAcceptedPositionSavesAgainstAContextHonoringStore(t *testing.T) { t.Errorf("recorded saves = %v, want [pos-1] — event 101 was delivered, so its position must be durable "+ "even against a store that honors the cancelled run context", got) } + // Detaching must drop CANCELLATION only. context.Background() would also + // pass the assertion above while stripping every value the caller put on + // the context, which a store using them would notice and this test would + // not. + if got := store.values(); len(got) != 1 || got[0] != "probe-value" { + t.Errorf("context values seen by the store = %v, want [probe-value] — the save's context must keep "+ + "the run's values and drop only its cancellation", got) + } } diff --git a/go/pkg/basecamp/eventfeed/doc.go b/go/pkg/basecamp/eventfeed/doc.go index c777f85308..92d3fd69d8 100644 --- a/go/pkg/basecamp/eventfeed/doc.go +++ b/go/pkg/basecamp/eventfeed/doc.go @@ -4,24 +4,25 @@ // // # Experimental // -// This package is experimental and incomplete, and today it is FOUNDATIONS -// ONLY: there is no consumer entry point yet — no Connector, no constructor, -// no run loop — so nothing here can be pointed at the live API. What has -// landed is the material a run loop is assembled from: the event and filter -// models, the seam interfaces and their error taxonomies, the Action Cable -// frame codecs, deduplication, the backoff and repair-jitter envelopes, the -// checkpoint store, the cable-URL policy and its WebSocket transport, and the -// deterministic fakes the conformance harness drives. +// This package is experimental and incomplete, and its exported surface may +// still change. The run loop has landed: Connector, Events, Close and Wait +// play the protocol described below — the first mint through the catch-up +// walk, the entry boundary, the drain, streaming delivery, the repair poll, +// and the recovery matrix a poll's 410, 400-position, or 409 re-enters +// through — alongside the event and filter models, the seam interfaces and +// their error taxonomies, the Action Cable frame codecs, deduplication, the +// backoff and repair-jitter envelopes, the checkpoint store, the cable-URL +// policy and its WebSocket transport, and the deterministic fakes the +// conformance harness drives. // -// Two pieces are still to land, and the package is not usable until both do: -// the run loop that plays the protocol described below — the first mint -// through the catch-up walk, the entry boundary, the drain, streaming -// delivery, the repair poll, and the recovery matrix a poll's 410, -// 400-position, or 409 re-enters through — and the Layer-1 adapters over the -// generated CreateStreamTicket and PollEvents operations that back the mint -// and poll seams. Everything below this section documents the architecture -// those pieces implement, not behavior this package performs today, and the -// exported surface may still change as they land. +// ONE piece is still to land, and it is what keeps the package unusable +// against the live API: the Layer-1 adapters over the generated +// CreateStreamTicket and PollEvents operations that back the TicketMinter and +// PollSource seams. Until they exist, the seams have no production +// implementation and a host must supply its own. Two obligations ride on +// those adapters rather than on anything here — zero egress to a foreign +// redirect target (conformance/event-feed/README.md's row-15 note), and the +// no-automatic-redirect-following rule §23 places on PollEvents. // // # Seams-first architecture // diff --git a/go/pkg/basecamp/eventfeed/export_test.go b/go/pkg/basecamp/eventfeed/export_test.go index 11eeda9c50..9238953b41 100644 --- a/go/pkg/basecamp/eventfeed/export_test.go +++ b/go/pkg/basecamp/eventfeed/export_test.go @@ -63,12 +63,6 @@ func (c *Connector) OnPumpReleased(f func()) { c.hooks.pumpReleased = f } // rendezvous for "the socket's death is queued for the state machine". func (c *Connector) OnPumpHandedOff(f func(isErr bool)) { c.hooks.pumpHandedOff = f } -// OnPumpRead registers a hook fired after a pump read returns and before its -// hand-off — the window in which a frame is READ but not yet observable -// through the queue. Parking here is the only deterministic way to put the -// protocol-fatal scan and an already-read frame in the same instant. -func (c *Connector) OnPumpRead(f func()) { c.hooks.pumpRead = f } - // OnFrameDeferred registers a hook fired when the in-flight-poll servicing // parks one receive — always a socket outcome — for the walk's next dispatch // point. It is the rendezvous for "the walk is now waiting on nothing but the diff --git a/go/pkg/basecamp/eventfeed/loop.go b/go/pkg/basecamp/eventfeed/loop.go index 0c115ea771..4cf7335823 100644 --- a/go/pkg/basecamp/eventfeed/loop.go +++ b/go/pkg/basecamp/eventfeed/loop.go @@ -6,7 +6,6 @@ import ( "fmt" "strconv" "sync" - "sync/atomic" "time" ) @@ -139,13 +138,6 @@ type liveConn struct { frames chan pumpItem stale *staleHolder hooks testHooks - // holding reports that the pump has READ a frame it has not yet handed - // off. It is what makes "every frame the pump has already read" an - // observable condition rather than an aspiration: without it, the window - // between ReadFrame returning and the channel send is invisible, and a - // non-blocking scan that samples inside it concludes the queue is empty - // when a frame — possibly the protocol-fatal one — has already arrived. - holding atomic.Bool } // newLiveConn arms staleness (socket open) and then starts the pump, in that @@ -172,22 +164,12 @@ func (lc *liveConn) pump(ctx context.Context) { defer close(lc.frames) for { data, err := lc.conn.ReadFrame(ctx) - // Set BEFORE anything else the read leads to, and cleared only once - // the item is in the queue: the flag must cover the whole interval in - // which this goroutine holds a frame no other goroutine can see. - lc.holding.Store(true) - if lc.hooks.pumpRead != nil { - lc.hooks.pumpRead() - } if err != nil { lc.handOff(ctx, pumpItem{err: err}) - lc.holding.Store(false) return } lc.stale.reset() - handed := lc.handOff(ctx, pumpItem{data: data}) - lc.holding.Store(false) - if !handed { + if !lc.handOff(ctx, pumpItem{data: data}) { return } } @@ -791,6 +773,14 @@ func (l *loop) runCycle(delay time.Duration) cycleOutcome { } if r.err != nil { hs.Stop() + // A seam may return BOTH a connection and an error — Go permits + // it and a transport that fails during the handshake can easily + // do it — and the connection is ours to dispose of the moment we + // decline it. Not closing leaked a socket per failed dial, which + // under the reconnect cycle is a leak per backoff round. + if r.conn != nil { + _ = r.conn.Close(closeCodeNormal, "") + } var derr *DialError if errors.As(r.err, &derr) && derr.Kind == DialPolicy { // The KIND is read; the text is not. This DialError came out @@ -812,6 +802,14 @@ func (l *loop) runCycle(delay time.Duration) cycleOutcome { } return cycleOutcome{kind: outcomeFailed} // transition 7 } + if r.conn == nil { + // (nil, nil) from a seam. Proceeding would install a nil + // CableConn and panic on the first read; treating it as a + // transient dial failure keeps the reconnect cycle honest and + // costs one backoff round against a transport that is already + // violating its contract. + return cycleOutcome{kind: outcomeFailed} + } conn = r.conn case <-hs.C(): // Transition 7: the deadline expired mid-dial — cancel the pending diff --git a/go/pkg/basecamp/eventfeed/loop_test.go b/go/pkg/basecamp/eventfeed/loop_test.go index 53d6c9420b..45d4877915 100644 --- a/go/pkg/basecamp/eventfeed/loop_test.go +++ b/go/pkg/basecamp/eventfeed/loop_test.go @@ -121,7 +121,14 @@ func (h *harness) ledger() []string { // start ranges Events on a collector goroutine. func (h *harness) start() { h.t.Helper() - ctx, cancel := context.WithCancel(context.Background()) + h.startCtx(context.Background()) +} + +// startCtx starts the iteration under a caller-supplied parent context, for +// the one assertion that needs a context VALUE to travel into a seam. +func (h *harness) startCtx(parent context.Context) { + h.t.Helper() + ctx, cancel := context.WithCancel(parent) h.cancel = cancel h.t.Cleanup(cancel) h.t.Cleanup(func() { h.conn.Close() }) From 82c6e3eba8d0c48fe71bf1bb2f51c6ded2c5eb91 Mon Sep 17 00:00:00 2001 From: Jeremy Daer Date: Wed, 19 Aug 2026 22:41:56 -0700 Subject: [PATCH 19/51] Event feed: the driver was order-strict where the family rule says arrival-strict MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `conformance/event-feed/README.md` binds every driver to one rule: a checkpoint save or an outbound frame observed while the current step is anything other than its matching expect step fails the scenario there and then. The Go reference driver did not do that. `Save` appended to a queue, `readPeer` appended to another, and the matching expect step later consumed whatever was sitting there — so an action emitted during an earlier, non-matching step still satisfied its expectation. There was no active-step field to judge against, and the README carried a paragraph saying so, which this deletes. The rule's own second sentence names why this is not a queue check. The connector reacts to a rendezvous being satisfied before the driver has stepped over it, so a pointer compared naively reports every rendezvous-adjacent action as an early arrival — not an edge case: with the read-through below removed, five of the twenty-two fixtures fail outright. Three things make the handoff atomic with respect to the harness mutex. `await` advances the pointer inside the critical section that observed the satisfying state, which covers a driver blocked ON the rendezvous. `currentStepLocked` reads through steps the recorded history has already satisfied but the driver has not consumed, which covers a driver that has not reached the rendezvous at all — a poll's page is delivered and then checkpointed by one causal chain in the connector, and the driver need not have run its `expectDelivered` step by the time the save lands. And a step the driver PERFORMS rather than waits for hands the pointer on before it acts, since nothing the connector does can precede the act; that is also what stops the read-through dead at an action step the driver has not performed, which is what keeps a subscribe written at dial — before the `serve welcome` that legalizes it — an early arrival rather than a match on the step after. The judgement is made before the action is recorded, so the matching step's own "one of mine is waiting" condition is still false and the read-through cannot run past it. `expectConnect` now takes the dial and installs the socket under the mutex, because the lookahead reads `d.peer` and `d.connectsTaken` from the connector's goroutines. Fixture 02's reordering probe now fails on arrival rather than on the save's delivery-count witness, which is the stronger diagnosis of the same defect. The witness stays anyway: the read-through is deliberately permissive — its job is only to say where the script has got to, and erring strict would stall the pointer and fail a compliant connector — so the witness is what still speaks if an arm is ever loosened. What the rule cannot reach in this lane is worth writing down, since five more drivers follow. A save is observed where the connector makes it, but an outbound frame is observed when the harness reads it off the loopback socket, a hop behind the write. A connector mutated to write its subscribe at dial is read only after the driver has served the welcome and stepped on, so no fixture catches it on arrival — 8 of 8 runs, with and without the action-step handoff. Fixture 07 catches that mutant structurally instead, as a frame where the script expects a close, and now names the step it belonged to. The rule is not the weaker half; the observation is, and pinning the write's own instant would need a witness at the write rather than a stricter reading of the rule. The driver's own tests pin each half against the mutation that deletes it, on the harness record paths where the rule lives, with every legal case paired against the illegal one it differs from by a single fact — including the subscribe-at-dial shape, which is deterministic there even though the socket makes it racy in a fixture. --- conformance/event-feed/README.md | 35 ++-- .../eventfeed/scenario_conformance_test.go | 167 +++++++++++++--- .../eventfeed/scenario_harness_test.go | 103 +++++++++- .../eventfeed/scenario_selftest_test.go | 183 +++++++++++++++++- 4 files changed, 437 insertions(+), 51 deletions(-) diff --git a/conformance/event-feed/README.md b/conformance/event-feed/README.md index 18725c9541..ec6ab4f481 100644 --- a/conformance/event-feed/README.md +++ b/conformance/event-feed/README.md @@ -45,22 +45,25 @@ The strict-match rule splits by action class; every driver implements exactly th unmatched when the script ends fails the scenario, and the seam-call counts in `finally` count it either way. -**The Go reference driver does not implement the arrival-strict half yet**, and -saying so here is the point: it queues saves and outbound frames and matches them -to their expect steps in ORDER, so an action that arrived during an earlier, -non-matching step still satisfies its expectation. The rule above is the contract -every driver is held to; this is a gap in one implementation, not a weakening of -it. The hard part is the atomic handoff the rule already names — an action that -becomes legal the instant a rendezvous is satisfied must match the next step -rather than fail against the stale one — which is why it is a tracked change -rather than a queue check. - -What that gap does NOT cover for is early-save ordering, which has its own -witness rather than relying on arrival strictness: every `save` records the -delivery count at the moment it arrived, and fixture 12 fails a save that -precedes its page's deliveries. The residue is narrower than the rule — an -outbound frame written earlier than its step, most concretely a subscribe -command sent before `welcome`. Tracked in #792. +A driver's step pointer moves in two ways, and both are needed for the atomic +handoff to hold. It advances inside the critical section that satisfies a +rendezvous, which covers a driver blocked ON one; and where the pointer has not +yet reached a step the recorded history has already satisfied, the arrival rule +reads THROUGH that step to the one after it — a poll's page is delivered and +then checkpointed by one causal chain in the connector, and the driver need not +have reached its `expectDelivered` step by the time the save lands. A step the +DRIVER performs rather than waits for (`serve`, `advance`, `fireTimer`, +`sever`, `serverClose`) is not read through: the connector cannot react before +the driver acts, so the pointer is handed to the next step before such a step +acts, and an action arriving under one the driver has not performed is early. + +What a driver can hold an action to is when it OBSERVED it. A save is observed +where the connector makes it, but a driver running a real socket observes an +outbound frame only when it reads one, which lags the write by a hop — so a +frame written a step early can still be read after the driver has stepped on. +The rule is not weaker for it; the observation is. A driver that wants the +write's own instant pinned needs a witness at the write, not a stricter reading +of this rule. ## Validation diff --git a/go/pkg/basecamp/eventfeed/scenario_conformance_test.go b/go/pkg/basecamp/eventfeed/scenario_conformance_test.go index 4a78d28518..a75bf078ff 100644 --- a/go/pkg/basecamp/eventfeed/scenario_conformance_test.go +++ b/go/pkg/basecamp/eventfeed/scenario_conformance_test.go @@ -2,17 +2,18 @@ // executed against the real connector through its seams. // // Strictness follows the family README's per-action-class rules exactly. -// Checkpoint saves and outbound frames are ARRIVAL-STRICT: a save carries the -// ordering witness (how many events had been delivered when it was made) and -// is matched against the deliveries its step's predecessors required, so a -// save that arrives while an `expectDelivered` rendezvous is still pending -// fails; outbound frames are matched in socket order and an unmatched one -// fails at `finally`. Mint and poll seam calls are PARKED: the call blocks -// inside the seam and its scripted response is released only when the driver -// reaches the matching expect step, so an early call cannot fail on arrival -// and one still parked at the end fails. Observation directives are -// rendezvous assertions under a wall-clock watchdog while virtual time stays -// frozen. +// Checkpoint saves and outbound frames are ARRIVAL-STRICT: each is judged, in +// the critical section that records it, against the step the script is on, and +// one arriving anywhere but its matching expect step fails the scenario there +// and then. `beginStep`, `currentStepLocked` and `await`'s advance are what +// make that survivable across a step boundary; a save additionally carries an +// ordering witness (how many events had been delivered when it was made), and +// an outbound frame no expect step matched still fails at `finally`. Mint and +// poll seam calls are PARKED: the call blocks inside the seam and its scripted +// response is released only when the driver reaches the matching expect step, +// so an early call cannot fail on arrival and one still parked at the end +// fails. Observation directives are rendezvous assertions under a wall-clock +// watchdog while virtual time stays frozen. package eventfeed_test import ( @@ -86,6 +87,11 @@ func runScenario(h *scenarioHarness, sc *scenario) (err error) { if cerr != nil { return fmt.Errorf("connector construction: %w", cerr) } + // The driver is bound to the harness BEFORE the iteration starts: the + // arrival rule judges against its step script, and a connector action + // observed before there is a script to judge it against would escape. + d := &driver{h: h, sc: sc} + h.attach(d) ctx, cancel := context.WithCancel(context.Background()) iteration := make(chan struct{}) go func() { @@ -110,12 +116,13 @@ func runScenario(h *scenarioHarness, sc *scenario) (err error) { } }() - d := &driver{h: h, sc: sc} for i, step := range sc.Steps { + d.beginStep(i, step) if stepErr := d.runStep(step); stepErr != nil { return fmt.Errorf("step %d (%s): %w", i+1, step.Kind, stepErr) } } + d.h.enterStep(len(sc.Steps)) return d.runFinally() } @@ -156,6 +163,114 @@ type driver struct { requiredDeliveries int } +// --- the arrival rule's step pointer ------------------------------------- + +// beginStep moves the arrival rule's pointer onto step i before the step runs. +// +// A DRIVER ACTION — serving a frame, closing or severing the socket, advancing +// or firing the clock — hands the pointer straight on to the next step instead. +// The connector cannot react before the driver acts, so everything an action +// provokes belongs to what FOLLOWS it; judging that reaction against the action +// step itself would fail every fixture whose `serve` is answered with a +// subscribe. It is also the only way an unexecuted action step keeps its teeth: +// the lookahead below stops dead at one, which is exactly what makes a +// subscribe written at dial — before the `serve welcome` that legalizes it — +// an early arrival rather than a match on the step after. +func (d *driver) beginStep(i int, step scenarioStep) { + d.h.enterStep(i) + if stepIsDriverAction(step) { + d.h.enterStep(i + 1) + } +} + +// currentStepLocked names the step an arriving action is judged against: the +// pointer, advanced over every step the recorded history has ALREADY satisfied +// but the driver has not consumed yet. +// +// That lookahead is the other half of the atomic handoff. `await` advances the +// pointer in the critical section that satisfies a rendezvous, which covers the +// driver blocked ON that rendezvous; the lookahead covers the driver that has +// not yet reached it — a page released at `expectPoll` is delivered and then +// checkpointed by one causal chain in the connector, and the driver need not +// have run its `expectDelivered` step by the time the save lands. Scanning is +// safe because a step's condition is checked BEFORE the arriving action is +// recorded: the matching step is still unsatisfied, so the scan stops there +// rather than running past it. +// +// Callers hold h.mu. +func (d *driver) currentStepLocked() (kind, where string) { + for i := d.h.cursor; i < len(d.sc.Steps); i++ { + step := d.sc.Steps[i] + if !d.stepSatisfiedLocked(step) { + return step.Kind, fmt.Sprintf("step %d (%s)", i+1, step.Kind) + } + } + return "", "the finally block" +} + +// stepIsDriverAction reports whether a directive is something the DRIVER does +// rather than something it waits for. +func stepIsDriverAction(step scenarioStep) bool { + switch step.Payload.(type) { + case *serveStep, *serverCloseStep, *severStep, *advanceStep, *fireTimerStep: + return true + default: + return false + } +} + +// stepSatisfiedLocked reports whether the recorded history already satisfies a +// step, without consuming anything. It is deliberately a touch more permissive +// than the executor's own rendezvous — it asks "has enough happened" rather +// than "does it match" — because its job is only to say where the script has +// got to. Erring permissive lets the pointer run on; erring strict would stall +// it and fail a compliant connector, so the arms that gate the two +// arrival-strict classes (expectCheckpoint, expectSubscribe/expectClientClose) +// are the ones written tight. +// +// A directive with no arm is a driver action, and the scan stops at one. +// +//nolint:gocyclo,cyclop // one arm per directive, mirroring runStep's dispatch +func (d *driver) stepSatisfiedLocked(step scenarioStep) bool { + h := d.h + switch payload := step.Payload.(type) { + case *expectMintStep: + return len(h.pendingMints) > 0 + case *expectConnectStep: + return len(h.connects) > d.connectsTaken + case *expectSubscribeStep, *expectClientCloseStep: + return d.peer != nil && d.peer.taken < len(d.peer.frames) + case *expectPollStep: + return len(h.pendingPolls) > 0 + case *exactIDs: + return len(h.delivered) >= len(payload.Exact) + case *expectCheckpointStep: + return h.savesTaken < len(h.saves) + case *timerSet: + return maps.Equal(timerCounts(h.clock), payload.Exact) + case *expectStateStep: + return h.state == payload.Is + case *errorExpect: + return h.terminal != nil + case *expectGapStep: + return h.gapsTaken < len(h.gaps) + case *expectBufferedStep: + return h.occupancy == payload.Count || slices.Contains(h.occupancyHistory, payload.Count) + case *expectSignalStep: + return h.signalsTaken < len(h.signals) + case *exactInvocations: + return len(h.invocations) >= len(payload.Exact) + case *expectSaveFailedStep: + return h.saveFailuresTaken < h.saveFailures + case *expectPositionRejectedStep: + return h.posRejectedTaken < len(h.positionRejected) + case *expectDisconnectedInvalidFrameStep: + return h.invalidFramesTaken < h.invalidFrames + default: + return false + } +} + // runStep dispatches one directive. // // table: flat dispatch is what keeps the mapping auditable. @@ -242,30 +357,30 @@ func (d *driver) expectMint(step *expectMintStep) error { return nil } +// expectConnect matches the next cable dial. It takes the dial and installs +// the accepted socket UNDER h.mu, on the satisfying call: `d.peer` and +// `d.connectsTaken` are read by the arrival rule's lookahead from the +// connector's goroutines, so they cannot be settled after the rendezvous +// returns. func (d *driver) expectConnect(step *expectConnectStep) error { index := d.connectsTaken - var attempt *connectAttempt - err := d.h.await(fmt.Sprintf("a cable dial to %s", step.URL), func() (bool, string) { + return d.h.await(fmt.Sprintf("a cable dial to %s", step.URL), func() (bool, string) { if len(d.h.connects) <= index { return false, "" } - attempt = d.h.connects[index] + attempt := d.h.connects[index] if attempt.url != step.URL { return false, fmt.Sprintf("the connector dialed\n %s\nwant the mint's url verbatim\n %s", attempt.url, step.URL) } - if attempt.kind != "accept" { - return true, "" + if attempt.kind == "accept" { + if attempt.peer == nil { + return false, "" + } + d.peer = attempt.peer } - return attempt.peer != nil, "" + d.connectsTaken++ + return true, "" }) - if err != nil { - return err - } - d.connectsTaken++ - if attempt.peer != nil { - d.peer = attempt.peer - } - return nil } func (d *driver) serve(step *serveStep) error { diff --git a/go/pkg/basecamp/eventfeed/scenario_harness_test.go b/go/pkg/basecamp/eventfeed/scenario_harness_test.go index 331c4ff992..cc77a320d5 100644 --- a/go/pkg/basecamp/eventfeed/scenario_harness_test.go +++ b/go/pkg/basecamp/eventfeed/scenario_harness_test.go @@ -59,6 +59,15 @@ type scenarioHarness struct { clock *feedtest.Clock + // Arrival strictness. program is the step script a checkpoint save or an + // outbound frame is judged against when it arrives, and cursor is the step + // the driver has reached in it. Both live under h.mu because the judgement + // has to be made in the same critical section that records the action — + // that is the whole of the family README's "observed while the current + // step is anything other than their matching expect step". + program *driver + cursor int + cableSrv *httptest.Server apiSrv *httptest.Server apiOrigin string @@ -218,6 +227,60 @@ func (h *scenarioHarness) violate(format string, args ...any) { h.notifyLocked() } +// --- arrival strictness -------------------------------------------------- + +// attach binds the driver whose step script the arrival rule judges against. +// Until it is bound the harness has no script — the driver's own unit tests +// construct a bare harness and drive its record paths directly — and the rule +// records nothing. +func (h *scenarioHarness) attach(d *driver) { + h.mu.Lock() + defer h.mu.Unlock() + h.program = d +} + +// enterStep moves the step pointer onto index. It only ever moves FORWARD: +// `await` hands the pointer to the next step at the instant a rendezvous is +// satisfied, so by the time the step loop reaches that step its own call is +// already stale and must not rewind it. +func (h *scenarioHarness) enterStep(index int) { + h.mu.Lock() + defer h.mu.Unlock() + if index > h.cursor { + h.cursor = index + } +} + +// advanceLocked is the atomic half of the handoff the family README names: it +// runs inside the SAME critical section that observed a rendezvous being +// satisfied, so an action the connector emits the instant it is satisfied is +// judged against the next step rather than against the stale one. Callers hold +// h.mu. +func (h *scenarioHarness) advanceLocked() { + if h.program != nil && h.cursor < len(h.program.sc.Steps) { + h.cursor++ + } +} + +// arrivalStrictLocked records a violation when an arrival-strict action — +// a checkpoint save, or an outbound frame — is observed while the current step +// is anything other than the expect step that matches it. Callers hold h.mu +// and call this BEFORE recording the action, so the matching step's own +// "an action of mine is waiting" condition is still false and cannot be +// mistaken for the step having already been satisfied. +func (h *scenarioHarness) arrivalStrictLocked(want, what string) { + if h.program == nil { + return + } + kind, where := h.program.currentStepLocked() + if kind == want { + return + } + h.violations = append(h.violations, fmt.Sprintf( + "%s arrived at %s, not at its matching %s step — checkpoint saves and outbound frames are arrival-strict", + what, where, want)) +} + // await blocks until check reports satisfaction, fails immediately when check // reports an unsatisfiable state, and gives up at the watchdog. check runs // under h.mu and may consume harness state on the satisfying call. @@ -231,7 +294,9 @@ func (h *scenarioHarness) await(what string, check func() (bool, string)) error } ok, bad := false, "" if violation == "" { - ok, bad = check() + if ok, bad = check(); ok { + h.advanceLocked() + } } wake := h.changed h.mu.Unlock() @@ -261,6 +326,10 @@ func (h *scenarioHarness) describe() string { h.mu.Lock() defer h.mu.Unlock() var b strings.Builder + if h.program != nil { + _, where := h.program.currentStepLocked() + fmt.Fprintf(&b, " current step: %s\n", where) + } fmt.Fprintf(&b, " state=%s delivered=%v buffered=%d\n", h.state, h.delivered, h.occupancy) fmt.Fprintf(&b, " mintCalls=%d (%d parked) pollCalls=%d (%d parked) connects=%d\n", h.mintCalls, len(h.pendingMints), h.pollCalls, len(h.pendingPolls), len(h.connects)) @@ -319,10 +388,12 @@ func (h *scenarioHarness) Load(_ context.Context, _ eventfeed.CheckpointKey) (st // Save implements eventfeed.CheckpointStore's save, recording the call with // its ordering witness and returning the scripted outcome. A save beyond the // script is a violation — the exact store-call script is what proves a -// subsequent save is attempted after a failure. +// subsequent save is attempted after a failure — and so is a save that arrives +// while the script is anywhere but its matching expectCheckpoint step. func (h *scenarioHarness) Save(_ context.Context, _ eventfeed.CheckpointKey, position string) error { h.mu.Lock() defer h.mu.Unlock() + h.arrivalStrictLocked("expectCheckpoint", fmt.Sprintf("the checkpoint save of %q", position)) h.saves = append(h.saves, saveRecord{position: position, deliveredAt: len(h.delivered)}) var err error if h.saveScripted { @@ -390,7 +461,7 @@ func (h *scenarioHarness) readPeer(peer *cablePeer) { var closeErr websocket.CloseError h.mu.Lock() if errors.As(err, &closeErr) { - peer.frames = append(peer.frames, clientFrame{kind: "close", code: int(closeErr.Code)}) + h.recordClientFrameLocked(peer, clientFrame{kind: "close", code: int(closeErr.Code)}) } peer.dead = true h.notifyLocked() @@ -401,11 +472,29 @@ func (h *scenarioHarness) readPeer(peer *cablePeer) { h.violate("the connector sent a non-text cable frame") continue } - h.mu.Lock() - peer.frames = append(peer.frames, clientFrame{kind: "text", data: slices.Clone(data)}) - h.notifyLocked() - h.mu.Unlock() + h.recordClientFrame(peer, clientFrame{kind: "text", data: slices.Clone(data)}) + } +} + +// recordClientFrame records one outbound action under the arrival rule. +func (h *scenarioHarness) recordClientFrame(peer *cablePeer, frame clientFrame) { + h.mu.Lock() + defer h.mu.Unlock() + h.recordClientFrameLocked(peer, frame) + h.notifyLocked() +} + +// recordClientFrameLocked is the arrival-strict half: an outbound frame is +// legal only while the script is at the expect step that matches it. The close +// arm shares this path because a close observed early is the same defect as a +// subscribe observed early — the connector acting ahead of the script. +func (h *scenarioHarness) recordClientFrameLocked(peer *cablePeer, frame clientFrame) { + if frame.kind == "close" { + h.arrivalStrictLocked("expectClientClose", "the connector's close of the socket") + } else { + h.arrivalStrictLocked("expectSubscribe", fmt.Sprintf("the outbound frame %s", frame.data)) } + peer.frames = append(peer.frames, frame) } // handleAPI is the API origin's sentinel: the connector reaches the mint and diff --git a/go/pkg/basecamp/eventfeed/scenario_selftest_test.go b/go/pkg/basecamp/eventfeed/scenario_selftest_test.go index 55a33ee2dc..9753328aed 100644 --- a/go/pkg/basecamp/eventfeed/scenario_selftest_test.go +++ b/go/pkg/basecamp/eventfeed/scenario_selftest_test.go @@ -9,6 +9,7 @@ package eventfeed_test import ( + "context" "encoding/json" "fmt" "os" @@ -134,6 +135,15 @@ func TestScenarioDriverRejectsMutatedFixtures(t *testing.T) { // the fixture scripts exactly that order. Swapping its last two steps demands // the delivery first; the save still arrives first, and the driver must say so // rather than matching it against a later step. +// +// It is the ARRIVAL rule that says so. The save's own ordering witness — the +// delivery count `saveRecord` captures — is the second line, and on a scenario +// path it no longer gets to speak: a save that beats the deliveries its script +// demands lands while the pointer is still on the unsatisfied `expectDelivered` +// step, which is an early arrival before the witness is ever compared. The +// witness is retained because the lookahead that computes the current step errs +// deliberately permissive (see stepSatisfiedLocked), and it is the check that +// still fires if a step's arm is ever loosened. func TestScenarioDriverEnforcesDeliveryBeforeCheckpoint(t *testing.T) { const fixture = "02-confirmation-gating.json" control := readFixture(t, fixture) @@ -163,9 +173,178 @@ func TestScenarioDriverEnforcesDeliveryBeforeCheckpoint(t *testing.T) { if err == nil { t.Fatal("a save that arrives before the deliveries the script requires must fail the scenario") } - if !strings.Contains(err.Error(), "delivery strictly precedes") { - t.Fatalf("failed for the wrong reason:\n got %v\n want the save-ordering rule", err) + if !strings.Contains(err.Error(), "arrival-strict") || !strings.Contains(err.Error(), "expectDelivered") { + t.Fatalf("failed for the wrong reason:\n got %v\n want the save rejected on arrival, under the still-pending expectDelivered step", err) + } +} + +// TestScenarioDriverIsArrivalStrict pins the two arrival-strict classes and, +// more to the point, the handoff that makes them survivable. +// +// A naive "is the pointer sitting on my expect step?" check reports every +// rendezvous-adjacent action as an early arrival, because the connector reacts +// to a rendezvous being satisfied before the driver has stepped over it. Two +// mechanisms answer that, and this test is written to fail if either is +// removed: `await` advances the pointer inside the critical section that +// satisfies a rendezvous, and `currentStepLocked` looks ahead over steps the +// history has already satisfied but the driver has not consumed yet. Each +// legal case below is paired with the illegal one it differs from by a single +// fact, so a mechanism that stops discriminating fails loudly rather than +// making everything legal. +// +// Every case drives the harness's own record paths — `Save`, and the frame +// recorder `readPeer` funnels through — which is where the rule lives. +func TestScenarioDriverIsArrivalStrict(t *testing.T) { + save := func(h *scenarioHarness) { _ = h.Save(context.Background(), eventfeed.CheckpointKey{}, arrivalPosition) } + subscribe := func(h *scenarioHarness, peer *cablePeer) { + h.recordClientFrame(peer, clientFrame{kind: "text", data: []byte(`{"command":"subscribe"}`)}) + } + + t.Run("a save at its expectCheckpoint step is legal", func(t *testing.T) { + h, _ := arrivalProgram(t, 1, stepServe(), stepExpectCheckpoint()) + save(h) + requireNoViolation(t, h) + }) + + t.Run("a save under an unexecuted action step is an early arrival", func(t *testing.T) { + h, _ := arrivalProgram(t, 0, stepServe(), stepExpectCheckpoint()) + save(h) + requireViolation(t, h, "step 1 (serve)", "expectCheckpoint") + }) + + t.Run("beginStep hands an action step's reaction to the step after it", func(t *testing.T) { + h, d := arrivalProgram(t, 0, stepServe(), stepExpectCheckpoint()) + d.beginStep(0, d.sc.Steps[0]) + save(h) + requireNoViolation(t, h) + }) + + // The rendezvous-adjacency pair. Both saves arrive with the pointer parked + // on an expectDelivered step the driver has not reached; only the delivery + // separates them. A driver that judged against the parked step alone would + // reject the first, which is what fixture 01, 12, 22 and 26 do on every run. + t.Run("a save handed over by a satisfied rendezvous is legal", func(t *testing.T) { + h, _ := arrivalProgram(t, 0, stepExpectDelivered(1), stepExpectCheckpoint()) + h.recordDelivered(1) + save(h) + requireNoViolation(t, h) + }) + + t.Run("a save ahead of that rendezvous is an early arrival", func(t *testing.T) { + h, _ := arrivalProgram(t, 0, stepExpectDelivered(1), stepExpectCheckpoint()) + save(h) + requireViolation(t, h, "step 1 (expectDelivered)", "expectCheckpoint") + }) + + t.Run("a client close before its step is an early arrival", func(t *testing.T) { + h, _ := arrivalProgram(t, 0, stepExpectSubscribe(), stepExpectClientClose()) + h.recordClientFrame(&cablePeer{}, clientFrame{kind: "close", code: 1000}) + requireViolation(t, h, "step 1 (expectSubscribe)", "expectClientClose") + }) + + t.Run("the handoff out of a consumed rendezvous is atomic", func(t *testing.T) { + // expectCheckpoint's own condition goes FALSE again the moment it + // consumes the save, so nothing downstream could look ahead over it: + // only the advance inside `await` puts the pointer on the close step. + h, d := arrivalProgram(t, 0, stepExpectCheckpoint(), stepExpectClientClose()) + save(h) + if err := d.expectCheckpoint(arrivalPosition); err != nil { + t.Fatalf("the checkpoint step must match its save: %v", err) + } + h.recordClientFrame(&cablePeer{}, clientFrame{kind: "close", code: 1000}) + requireNoViolation(t, h) + }) + + // The subscribe-at-dial mutant, deterministically. The dial is recorded, so + // the expectConnect step is satisfied and the lookahead runs past it — and + // stops at the `serve welcome` the driver has not performed, which is the + // whole reason an unexecuted action step must not be skippable. + t.Run("a subscribe written at dial is an early arrival", func(t *testing.T) { + h, d := arrivalProgram(t, 0, stepExpectConnect(), stepServe(), stepExpectSubscribe()) + peer := recordDial(h, d) + subscribe(h, peer) + requireViolation(t, h, "step 2 (serve)", "expectSubscribe") + }) + + t.Run("a subscribe written on welcome is legal", func(t *testing.T) { + h, d := arrivalProgram(t, 0, stepExpectConnect(), stepServe(), stepExpectSubscribe()) + peer := recordDial(h, d) + d.beginStep(1, d.sc.Steps[1]) + subscribe(h, peer) + requireNoViolation(t, h) + }) +} + +// arrivalPosition is the checkpoint position every case above saves. +const arrivalPosition = "pos-1" + +// arrivalProgram binds a harness to a driver over a synthetic step program, +// with the arrival rule's pointer parked on step `at`. +func arrivalProgram(t *testing.T, at int, steps ...scenarioStep) (*scenarioHarness, *driver) { + t.Helper() + h := newScenarioHarness() + t.Cleanup(h.close) + d := &driver{h: h, sc: &scenario{Steps: steps}} + h.attach(d) + h.enterStep(at) + return h, d +} + +// recordDial records one accepted cable dial and hands the driver its socket, +// exactly as a matched expectConnect step would. The peer carries no live +// socket — these cases exercise the recording paths, not the loopback — so it +// deliberately stays out of h.peers, which teardown closes. +func recordDial(h *scenarioHarness, d *driver) *cablePeer { + peer := &cablePeer{} + h.mu.Lock() + defer h.mu.Unlock() + h.connects = append(h.connects, &connectAttempt{url: h.cableURL(1), kind: "accept", peer: peer}) + d.peer = peer + return peer +} + +func requireNoViolation(t *testing.T, h *scenarioHarness) { + t.Helper() + if err := h.violation(); err != nil { + t.Fatalf("an action legal under the arrival rule was rejected: %v", err) + } +} + +func requireViolation(t *testing.T, h *scenarioHarness, fragments ...string) { + t.Helper() + err := h.violation() + if err == nil { + t.Fatal("an action arriving outside its matching expect step must be a violation") } + for _, fragment := range fragments { + if !strings.Contains(err.Error(), fragment) { + t.Fatalf("rejected for the wrong reason:\n got %v\n want a failure naming %q", err, fragment) + } + } +} + +func stepServe() scenarioStep { + return scenarioStep{Kind: "serve", Payload: &serveStep{Frame: "welcome"}} +} + +func stepExpectConnect() scenarioStep { + return scenarioStep{Kind: "expectConnect", Payload: &expectConnectStep{}} +} + +func stepExpectSubscribe() scenarioStep { + return scenarioStep{Kind: "expectSubscribe", Payload: &expectSubscribeStep{Channel: "EventsChannel"}} +} + +func stepExpectClientClose() scenarioStep { + return scenarioStep{Kind: "expectClientClose", Payload: &expectClientCloseStep{}} +} + +func stepExpectDelivered(ids ...int64) scenarioStep { + return scenarioStep{Kind: "expectDelivered", Payload: &exactIDs{Exact: ids}} +} + +func stepExpectCheckpoint() scenarioStep { + return scenarioStep{Kind: "expectCheckpoint", Payload: &expectCheckpointStep{Position: arrivalPosition}} } func stepIs(step any, directive string) bool { From 274b40f0ef1679abd9dc4ec07b929dd00f1670cd Mon Sep 17 00:00:00 2001 From: Jeremy Daer Date: Wed, 19 Aug 2026 22:47:44 -0700 Subject: [PATCH 20/51] Event feed: a socket with nothing to say was the one thing the poll wait could not hear MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #758. Every socket-open wait in the connector carries the staleness cases; pollPage did not. A socket that goes silently half-open produces no frame and no read error, so the firing is the only evidence there will ever be — and it sat unconsumed while a PollSource that returns only on cancellation held CatchingUp forever, with nothing able to cancel it, because the cancel would have come from the teardown the wait was preventing. Both reviewers prescribed disposing the attempt where the expiry is observed, and that breaks a pinned ordering: TestWalkFailureBetweenPages/staleness_expiry advances a full window while the first poll is in flight and requires the walk to still accept, deliver and SAVE that page. It is transition 21's finish-the-page rule, and it is load-bearing — the reconnect re-enters at pos-1 rather than replaying. That mutant is in the record: checkpoint ledger = [], want [pos-1]. So the expiry is treated as what it is, a socket outcome, and takes the same route as a disconnect frame: deferred into the single slot, dispatched at the page boundary. The slot gains a third form, carrying the age rather than a frame, and the protocol-fatal probe excludes it by name rather than trusting what parsing a zero item happens to return. That left the bounded wait unbounded again, which is the part worth explaining. The wait relies on the staleness timer's firings and re-arms as WAKE-UPS. An authoritative expiry latches, and nothing re-arms over a latch — that is deliberate, and it is what keeps a frame arriving after the deadline from erasing the verdict — so consuming the firing here consumed the last wake the wait had. graceWake re-arms the same `staleness` kind purely as a wake source, keeping the latch so the pump's per-frame reset and a blocked hand-off's release cannot touch it. The outstanding set is untouched: {staleness} stays {staleness}, and §23's six kinds and per-state exact sets are not a thing a Go fix gets to change. The bound is now published rather than living in a Go comment. Worst case is DETECTION WINDOW + GRACE PHASE, both one staleness window, and the grace phase is a deadline read at the instant of deferral with two immunities that are the whole point of naming it: a frame arriving inside it must not extend it, and a blocked hand-off must not suspend it. The second is what the earlier borrowed bound failed — this wait is the one that deliberately stops draining, so the full-queue premise the suspension rule rests on is false here by construction. awaitSupersededPoll's "close to two windows ... cannot exceed two" paragraph is replaced by that statement rather than deleted; it was describing an accident of borrowing the timer, where this is a contract for six SDKs. Six tests, each able to fail alone. Four of them were written first and shown to hang against the unfixed code, reported as bounded watchdog failures rather than a wedged binary. The immunity pair needed more care than expected: a frame inside the grace phase hands the wait a WAKE as well as a re-armed window, and a wake alone is enough to carry an implementation that also moved its deadline to a correct-looking teardown — so the test consumes that wake first, through a new hook, and then requires the phase to finish on a wake of its own. Without that, the no-latch mutant survived. One guard is honestly unkillable and stays anyway. Dropping the probe's `!d.stale` changes no behavior today, because protocolFatalFrame rejects a zero item through parseFrame failing on nil bytes. That is an accident of an unrelated function's error handling, and a probe resting on it is one refactor away from reading a staleness expiry as a disconnect frame. --- SPEC.md | 44 ++ go/pkg/basecamp/eventfeed/catchup.go | 190 ++++++--- go/pkg/basecamp/eventfeed/catchup_test.go | 479 ++++++++++++++++++++++ go/pkg/basecamp/eventfeed/connector.go | 9 + go/pkg/basecamp/eventfeed/export_test.go | 7 + go/pkg/basecamp/eventfeed/loop.go | 37 ++ 6 files changed, 716 insertions(+), 50 deletions(-) diff --git a/SPEC.md b/SPEC.md index f13012c00b..08d28d77c3 100644 --- a/SPEC.md +++ b/SPEC.md @@ -3460,6 +3460,47 @@ Draining is Terminal(`protocol_fatal`) immediately (the state-generic rule under Disconnect Dispatch) — the drain is not completed, the held entry position is NOT saved, and no `caught_up` is announced; only recoverable failures defer. +**A socket outcome observed while a poll seam call is in flight is deferred to the page +boundary, and the wait for that call is bounded by DETECTION WINDOW + GRACE PHASE.** This +is transition 21 in CatchingUp, the only state that holds a wire call open across a socket +event, and it is normative for every SDK. + +The deferral itself is the finish-the-page ordering the rest of §23 already states: the +in-flight page is accepted, delivered and saved, and only then is the socket's outcome +dispatched. It applies to **every** socket outcome, and that expressly includes a staleness +expiry — a socket that goes silently half-open produces no frame and no read error, so its +expiry is the only evidence there will ever be, and an implementation whose in-flight-poll +wait does not observe `staleness` cannot detect that socket at all. Disposing the attempt +where the expiry is observed is NOT conformant: it strands the deliveries and the save of a +page the server had already served. + +The two intervals bound the whole sequence, and their sum is the worst case: + +- **Detection window** — `EVENT_FEED_STALE_AFTER`, the staleness window, measured from the + last inbound frame. It is what decides the socket is dead, under the ordinary evaluation + rule: a firing superseded by a frame the reader took first, or one whose window + overlapped a blocked hand-off, is not evidence and must not be deferred. +- **Grace phase** — one further `EVENT_FEED_STALE_AFTER`, measured from the deferral, after + which the seam call is abandoned to the deferred outcome's teardown (the teardown + cancels it — Seam-Call Semantics). It is a **deadline read from the clock at the instant + of deferral**, not a window, and it carries two immunities that are the point of naming + it: + - **Immune to frame resets.** A frame arriving inside the phase re-arms `staleness` in + the ordinary way and MUST NOT move the deadline. The phase bounds how long the + connector waits for an abandoned call, which is unrelated to whether the peer is still + talking. + - **Immune to suspension.** The full-queue suspension rule below rests on a full queue + proving the peer is outrunning a connector that is still consuming. This is the one + wait that deliberately stops consuming — an outcome already occupies the single + deferral slot — so the premise is false here by construction, and a suspended + evaluation MUST NOT extend the phase. + +An implementation may wake as often as it likes and from whatever source it has; wakes may +be early or late, and the deadline is what decides. In particular this introduces **no new +timer kind**: the six kinds and the per-state exact timer sets are both unchanged, and an +implementation that re-arms `staleness` purely to obtain a wake keeps CatchingUp's set at +{`staleness`}. + **"Observed" means handed to the state machine, and the boundary is normative.** A frame the transport reader has taken off the socket but not yet handed over is not observed, and no implementation is required to find it. That is not a tolerance granted for convenience: @@ -3787,6 +3828,9 @@ INTERFACE PollSource -- triggered on close(), caller cancellation, AND any teardown of the attempt the call -- belongs to (mid-walk socket failure, staleness, a terminal): a superseded poll must -- not stall reconnection or return into a disposed attempt. Prompt return required. + -- The connector's own side of that is bounded rather than trusting: an outcome observed + -- while the call is in flight is awaited for the grace phase and then abandoned to the + -- teardown (detection window + grace phase, under the state machine above). END RECORD Cursor -- exactly one field set; the zero Cursor is the bare present entry diff --git a/go/pkg/basecamp/eventfeed/catchup.go b/go/pkg/basecamp/eventfeed/catchup.go index fa893687fa..7d8b977237 100644 --- a/go/pkg/basecamp/eventfeed/catchup.go +++ b/go/pkg/basecamp/eventfeed/catchup.go @@ -364,9 +364,18 @@ func (l *loop) walk(at *attempt, cursor Cursor, presentClass bool) (out cycleOut } } -// deferredFrame is one pump receive taken out of band while a poll seam call -// was in flight and left for the walk's ordinary dispatch point. closed -// records the pump's channel closing, which carries no item. +// deferredFrame is one socket outcome taken out of band while a poll seam call +// was in flight and left for the walk's ordinary dispatch point. It has three +// forms, and only the first carries a frame: an ordinary pump receive; closed, +// the pump's channel closing, which carries no item; and stale, a staleness +// expiry, which carries no item either and instead carries the age +// Observer.StaleConnection reports. +// +// The stale form exists because a silently half-open socket produces NEITHER a +// frame nor a read error. The expiry is the only evidence there is, and it is +// as much a socket outcome as a read error — so it takes the same treatment +// rather than a teardown where it is observed, which would strand the in-flight +// page's deliveries and its save. // // Every deferral is a SOCKET outcome (or the pump's exit), which is transition // 21: the page boundary deliberately observes it AFTER the in-flight page has @@ -376,6 +385,9 @@ func (l *loop) walk(at *attempt, cursor Cursor, presentClass bool) (out cycleOut type deferredFrame struct { item pumpItem closed bool + stale bool + // age is the silence the expiry measured, for the stale form only. + age time.Duration } // pollResult is one poll seam call's outcome, carried back from the call's @@ -428,13 +440,25 @@ type pollAttempt struct { // before the walk stops, and frame order is preserved because the servicing // stops at the deferred receive. // +// A STALENESS EXPIRY defers on identical terms, and this select is where it is +// observed — every other socket-open wait in the connector carries the case, +// and for a long time this one did not (#758). The hole it left is the socket +// that goes silently half-open: it produces no frame and no read error, so the +// firing is the only evidence there is. Unconsumed, it sat there while a +// PollSource that returns only on cancellation held CatchingUp forever, with +// nothing able to cancel it — the cancel would have come from the teardown the +// wait was preventing. +// +// Disposing the attempt where the expiry is observed is the obvious remedy and +// it is wrong: transition 21 says an accepted page's deliveries and its save +// are not stranded by the socket's death, and the expiry is a socket outcome +// like any other. So it is deferred, and dispatched at the page boundary. +// // It reports superseded=true when it gave up on the call: a deferred SOCKET -// outcome is still awaited, but only for as long as the staleness window, -// because a PollSource that returns only on context cancellation would -// otherwise hold the consumer's goroutine forever behind a socket that has -// already spoken (and the staleness expiry that would tear that socket down -// is unobservable while the call is outstanding). The caller dispatches the -// deferred outcome, whose teardown cancels the abandoned call. +// outcome is still awaited, but only for the grace phase, because a PollSource +// that returns only on context cancellation would otherwise hold the consumer's +// goroutine forever behind a socket that has already spoken. The caller +// dispatches the deferred outcome, whose teardown cancels the abandoned call. func (l *loop) pollPage(at *attempt, cursor Cursor) pollAttempt { done := make(chan pollResult, 1) go func() { @@ -446,9 +470,25 @@ func (l *loop) pollPage(at *attempt, cursor Cursor) pollAttempt { done <- pollResult{page: p, err: perr} }() for { + staleTimer, staleGen := at.lc.stale.current() + stale := false select { case r := <-done: return pollAttempt{page: r.page, err: r.err} + case <-at.lc.stale.rearmed(): + // The pump swapped the window out; a parked select holds one timer, + // so the swap is invisible until something wakes it. Re-read it. + continue + case <-staleTimer.C(): + age, ok := at.lc.stale.evaluate(staleGen) + if !ok { + // Superseded by a frame the pump received first, or suspended + // by a blocked hand-off and re-armed. Neither is evidence, and + // neither may consume the call: the wait continues on the + // window evaluate just re-armed. + continue + } + l.deferred, stale = &deferredFrame{stale: true, age: age}, true case item, ok := <-at.lc.frames: if !ok { l.deferred = &deferredFrame{closed: true} @@ -464,15 +504,24 @@ func (l *loop) pollPage(at *attempt, cursor Cursor) pollAttempt { } l.deferred = &deferredFrame{item: item} } - // The superseded-poll bound starts HERE, at the deferral, and is - // read before the hook fires so nothing observing the deferral - // can race the deadline into existence behind it. - deadline := l.cfg.clock.Now().Add(l.cfg.staleAfter) - if l.hooks.frameDeferred != nil { - l.hooks.frameDeferred() - } - return l.awaitSupersededPoll(at, done, deadline) } + // The grace phase starts HERE, at the deferral, and its deadline is + // read before the hook fires so nothing observing the deferral can + // race the deadline into existence behind it. + deadline := l.cfg.clock.Now().Add(l.cfg.staleAfter) + if stale { + // The firing this branch consumed was the wait's own wake source, + // and an authoritative expiry latches — so without this there is + // nothing left to wake the grace phase at all, and the bounded wait + // becomes unbounded again. Armed BEFORE the hook, so a test that + // rendezvouses on the deferral cannot advance past a wake that is + // not yet armed. + at.lc.stale.graceWake() + } + if l.hooks.frameDeferred != nil { + l.hooks.frameDeferred() + } + return l.awaitSupersededPoll(at, done, deadline, stale) } } @@ -498,43 +547,47 @@ func (l *loop) pollPage(at *attempt, cursor Cursor) pollAttempt { // then enough to keep an already-dead socket's verdict from ever landing. // // The staleness firing and the re-arm wake are demoted to WAKE-UPS; the -// deadline alone decides. evaluate is still called on a firing, and still -// re-arms a suspended window — that re-arm is what guarantees the next wake -// when the pump is blocked and no frame can deliver one — so the lapse is -// observed at the first wake at or after the deadline. +// deadline alone decides, and the post-select check runs on every wake, so an +// early or late one costs a loop turn rather than the verdict. Where the wake +// comes from depends on which outcome was deferred. For a frame, evaluate is +// still called on a firing and still re-arms a suspended window — that re-arm +// is what guarantees the next wake when the pump is blocked and no frame can +// deliver one. For a staleness expiry, evaluate has already spoken and the +// firing that carried its verdict is consumed, so graceWake supplies the wake +// instead. // // A dedicated timer would express this more directly, and is deliberately not // used: SPEC.md §23 pins exactly six timer kinds AND every state's exact // timer set, both asserted by the cross-SDK fixtures, so a seventh kind is a // spec change across six SDKs rather than a fix to the Go reference. // -// # The worst case is TWO staleness windows, and that is the published bound +// # The published bound: detection window + grace phase // -// Borrowing the window means the wait wakes on the window's cadence, not the -// deadline's, so it can overshoot — and the size of the overshoot is worth -// stating rather than discovering. +// SPEC.md §23 ("the bound on a superseded poll seam call") states it +// normatively for all six SDKs, and this is the Go reference's realization of +// it. A socket that dies during a poll is detected within one DETECTION WINDOW +// — EVENT_FEED_STALE_AFTER, and for a silently half-open socket that window IS +// the detection, there being no frame and no read error to observe — and the +// call it superseded is then abandoned within one GRACE PHASE, which is +// `deadline` here and is also one window long. Nothing waits longer than their +// sum. // -// The window in flight was armed at the last frame receipt, at or before the -// deferral, so it fires at or before the deadline. If the pump is blocked at -// that moment the firing is not evidence (a full queue is a fast peer, not a -// dead one), evaluate re-arms, and the NEXT wake is a full window later — past -// the deadline by however much of the first window had already elapsed when -// the deferral happened. A deferral landing immediately after a frame -// therefore waits close to two windows in total: one for the in-flight window -// to close, one for the re-armed window to deliver the wake that lets the -// post-select predicate observe the lapse. +// The grace phase is a value, not a timer, and it has two immunities that are +// the point of naming it. It is immune to FRAME RESETS: a frame arriving inside +// it re-arms staleness, and must not move a deadline that bounds the wait for +// an abandoned call rather than the peer's liveness. And it is immune to +// SUSPENSION: a blocked hand-off suspends staleness evaluation on the premise +// that a full queue proves the peer is outrunning a connector that is still +// consuming, and this is the one wait that deliberately stops consuming, so the +// premise is false here by construction. // -// It cannot exceed two. The post-select check runs on EVERY wake, and by the -// first re-armed firing the clock is necessarily past a deadline set one -// window after the deferral — so a second re-arm is never waited out, however -// long the pump stays blocked. -// -// This is a property of borrowing the timer, not a defect to patch: a shorter -// re-armed window would be a staleness window that is not one, and a dedicated -// one is the seventh kind above. It is carried to bc3 as an open §23 contract -// question (staleness × in-flight-poll deferral) rather than asserted as -// settled contract here. -func (l *loop) awaitSupersededPoll(at *attempt, done <-chan pollResult, deadline time.Time) pollAttempt { +// The stale caller gets both from graceWake's latch, which stops the pump's +// resets and a hand-off's release from touching the timer at all. The frame +// caller gets them from the deadline: its wakes come from an ordinary staleness +// window that a peer CAN push, but the post-select check runs on every wake, +// and a re-armed window under a blocked pump still delivers one within a window +// — so a peer that keeps sending can move a wake, never the verdict. +func (l *loop) awaitSupersededPoll(at *attempt, done <-chan pollResult, deadline time.Time, stale bool) pollAttempt { for { staleTimer, staleGen := at.lc.stale.current() select { @@ -544,10 +597,20 @@ func (l *loop) awaitSupersededPoll(at *attempt, done <-chan pollResult, deadline return pollAttempt{superseded: true} case <-at.lc.stale.rearmed(): case <-staleTimer.C(): - if _, ok := at.lc.stale.evaluate(staleGen); ok { + if stale { + // The verdict is already rendered and parked; this firing is a + // wake and nothing else, so nothing re-evaluates it. Re-arming + // keeps a wake in hand for the next turn, which is what makes + // the phase's end depend on the deadline rather than on a + // timer landing exactly on it. + at.lc.stale.graceWake() + } else if _, ok := at.lc.stale.evaluate(staleGen); ok { return pollAttempt{superseded: true} } } + if l.hooks.supersededWake != nil { + l.hooks.supersededWake() + } if !l.cfg.clock.Now().Before(deadline) { return pollAttempt{superseded: true} } @@ -603,8 +666,8 @@ func (l *loop) admitDuringPoll(at *attempt, item pumpItem) (handled bool, out cy return true, cycleOutcome{}, false } -// dispatchDeferred dispatches the frame the in-flight-poll servicing deferred, -// if any, through the ordinary post-confirmation dispatch. +// dispatchDeferred dispatches the socket outcome the in-flight-poll servicing +// deferred, if any, through the ordinary post-confirmation dispatch. // // Its two call sites are the walk's dispatch points — the page boundary and // the walk's end — and deliberately not the `poll-retry` wait, which holds a @@ -612,13 +675,33 @@ func (l *loop) admitDuringPoll(at *attempt, item pumpItem) (handled bool, out cy // omission: a deferred receive is either the pump's terminating error, which // nothing can follow, or a disconnect frame, the server's last word on the // socket; both are dispatched at the next page boundary either way. +// +// The stale form takes transition 21's staleness trigger verbatim — the same +// teardown, a StaleConnection carrying the age, then the same +// errStaleConnection disconnect, in the order socketCheck fires them in. That +// identity is the contract: whether the expiry was observed at the page +// boundary or deferred from inside a poll is an implementation detail of WHEN, +// never of WHAT an observer sees. +// +// The age is the one thing that is carried rather than recomputed, and +// deliberately: it was measured when the verdict was RENDERED, so it reports +// the silence the connector actually detected rather than the silence plus +// however long the abandoned call went on for afterwards. func (l *loop) dispatchDeferred(at *attempt) (cycleOutcome, bool) { d := l.deferred if d == nil { return cycleOutcome{}, false } l.deferred = nil - if d.closed { + switch { + case d.stale: + l.disposeAttempt(at, nil) + if l.cfg.observer.StaleConnection != nil { + l.cfg.observer.StaleConnection(d.age) + } + l.observeDisconnected("", errStaleConnection) + return cycleOutcome{kind: outcomeFailed}, true + case d.closed: return l.pumpExited(at, nil), true } return l.handleLiveFrame(at, nil, d.item, false) @@ -830,7 +913,14 @@ func (l *loop) fatalScan(at *attempt, budget *int) (cycleOutcome, bool) { // A fatal frame already in the slot is dispatched now rather than after // the save: the carve-out is about what governs, not about which queue the // frame is sitting in. - if d := l.deferred; d != nil && !d.closed { + // + // Only ONE of the slot's three forms carries a frame, and the other two are + // excluded by name rather than by what parsing their zero pumpItem happens + // to return. protocolFatalFrame does reject it today — parseFrame fails on + // nil bytes — but that is an accident of an unrelated function's error + // handling, and a probe whose correctness rests on it is one refactor away + // from reading a staleness expiry as a disconnect frame. + if d := l.deferred; d != nil && !d.closed && !d.stale { if f, ok := protocolFatalFrame(d.item); ok { l.deferred = nil return l.terminateProtocolFatal(at, f), true diff --git a/go/pkg/basecamp/eventfeed/catchup_test.go b/go/pkg/basecamp/eventfeed/catchup_test.go index 11544dcad3..2979a66317 100644 --- a/go/pkg/basecamp/eventfeed/catchup_test.go +++ b/go/pkg/basecamp/eventfeed/catchup_test.go @@ -2106,3 +2106,482 @@ func TestProtocolFatalHandedOffDuringADrainIsImmediate(t *testing.T) { assertProtocolFatalDrain(t, h, store, caughtUp) assertIDs(t, h.deliveredIDs(), 41) } + +// staleDeferralHarness wires the shape every grace-phase test needs: a +// position-resume entry, a stalled entry poll — a PollSource that returns only +// on cancellation — and observation points for the deferral and for the +// StaleConnection age. The socket is SILENT throughout: no frame and no read +// error, which is the whole point. A half-open socket produces neither, so the +// staleness firing is the only evidence the connector will ever get, and until +// #758 the one wait that could observe it did not carry the case. +func staleDeferralHarness(t *testing.T, store *feedtest.Store, ages *[]time.Duration, mu *sync.Mutex) (*harness, *deferralWatch) { + t.Helper() + h := storedHarness(t, store, eventfeed.WithObserver(eventfeed.Observer{ + StaleConnection: func(age time.Duration) { + mu.Lock() + *ages = append(*ages, age) + mu.Unlock() + }, + })) + h.minter.ScriptTicket(ticket(1)) + h.minter.ScriptTicket(ticket(2)) + w := &deferralWatch{ch: make(chan struct{}, 4)} + h.conn.OnFrameDeferred(func() { w.ch <- struct{}{} }) + t.Cleanup(func() { w.report(t) }) + return h, w +} + +// deferralWatch is the rendezvous for "the in-flight-poll servicing has parked +// an outcome", waited on from the poll seam call's OWN goroutine — which is the +// only place that can hold the call open across the deferral. +// +// It records a miss instead of failing, and the report is registered as a test +// cleanup, because that goroutine outlives the test: a t.Error raised there +// after the test has finished panics the whole binary, replacing a legible +// assertion failure with a stack dump. The cleanup still runs while the test +// can be failed, so the diagnostic survives. +type deferralWatch struct { + ch chan struct{} + mu sync.Mutex + // missed records a lapsed wait — the shape the defect this file's + // grace-phase tests cover produces. + missed bool +} + +// await blocks until a deferral lands, bounded by the watchdog, reporting +// whether it did. A caller that gets false must stop rather than advance a +// clock nothing is waiting on. +func (w *deferralWatch) await() bool { + select { + case <-w.ch: + return true + case <-time.After(watchdog): + w.mu.Lock() + w.missed = true + w.mu.Unlock() + return false + } +} + +func (w *deferralWatch) report(t *testing.T) { + w.mu.Lock() + defer w.mu.Unlock() + if w.missed { + t.Error("the staleness expiry was never deferred while the poll seam call was in flight") + } +} + +// TestStalenessDuringAStalledPollTearsDownAtTheGraceDeadline is #758: a +// silently half-open socket observed while a poll seam call is in flight. +// +// Nothing else can report it. The socket produces no frame and no read error, +// so the `staleness` firing is the only evidence there is; the PollSource +// returns only on cancellation, and the cancel would come from the teardown the +// wait is preventing. Before this, pollPage's select carried neither staleness +// case, the firing sat unconsumed, and CatchingUp waited forever. +// +// The wait terminates within DETECTION WINDOW + GRACE PHASE of the last frame: +// one window to decide the socket is dead, one to bound the abandoned call. +func TestStalenessDuringAStalledPollTearsDownAtTheGraceDeadline(t *testing.T) { + store := feedtest.NewStore() + store.Stored("pos-0") + var mu sync.Mutex + var ages []time.Duration + h, deferral := staleDeferralHarness(t, store, &ages, &mu) + h.polls.StallNext() + + h.polls.OnCall(func(feedtest.PollCall) { + // The detection window lapses with the call outstanding. + h.clock.Advance(staleAfter) + if !deferral.await() { + return + } + // And then the grace phase. + h.clock.Advance(staleAfter) + }) + h.start() + conn := h.driveToSubscribed() + conn.Serve(frameConfirm(noFilterIdentifier)) + + h.awaitTimer(timerBackoff) + if !conn.Closed() { + t.Fatal("the deferred staleness expiry must dispose the attempt") + } + if _, terminal, _ := h.snapshot(); terminal != nil { + t.Fatalf("staleness is never terminal; got %v", terminal) + } + assertPositions(t, store.Saves()) + assertTimers(t, h.clock, map[string]int{timerBackoff: 1}) + mu.Lock() + defer mu.Unlock() + if len(ages) != 1 || ages[0] != staleAfter { + t.Fatalf("StaleConnection ages = %v, want exactly one of %s", ages, staleAfter) + } +} + +// TestStalenessDuringAnInFlightPollFinishesThePage is acceptance criterion 1 +// made deterministic. TestWalkFailureBetweenPages/staleness_expiry pins the +// same ordering, but its poll returns while the firing is still unobserved, so +// the state machine's select may take either ready case; this one holds the +// seam call open until the deferral has demonstrably landed. +// +// The expiry is a SOCKET outcome, and transition 21 observes those at the page +// boundary: the in-flight page is accepted, delivered and SAVED, and only then +// is the socket torn down. Disposing the attempt where the expiry is observed +// is the obvious remedy and it strands exactly that save. +func TestStalenessDuringAnInFlightPollFinishesThePage(t *testing.T) { + store := feedtest.NewStore() + store.Stored("pos-0") + var mu sync.Mutex + var ages []time.Duration + h, deferral := staleDeferralHarness(t, store, &ages, &mu) + next := testOrigin + "/999/events.json?after=101" + h.polls.ScriptPage(eventfeed.PollPage{ + Events: []eventfeed.Event{pollEvent(101)}, + Position: "pos-1", + Next: next, + }) + h.polls.ScriptPage(eventfeed.PollPage{Events: []eventfeed.Event{pollEvent(102)}, Position: "pos-2"}) + + var once sync.Once + h.polls.OnCall(func(feedtest.PollCall) { + once.Do(func() { + h.clock.Advance(staleAfter) + deferral.await() + }) + }) + h.start() + conn := h.driveToSubscribed() + conn.Serve(frameConfirm(noFilterIdentifier)) + + h.awaitTimer(timerBackoff) + if got := h.polls.CallCount(); got != 1 { + t.Fatalf("poll seam calls = %d, want 1 — the walk must not follow `next` on a stale socket", got) + } + if !conn.Closed() { + t.Fatal("the deferred staleness expiry must dispose the attempt") + } + assertLedger(t, h.ledger(), []string{"event 101", "save pos-1"}) + assertPositions(t, store.Saves(), "pos-1") + assertIDs(t, h.deliveredIDs(), 101) + assertTimers(t, h.clock, map[string]int{timerBackoff: 1}) + mu.Lock() + defer mu.Unlock() + if len(ages) != 1 || ages[0] != staleAfter { + t.Fatalf("StaleConnection ages = %v, want exactly one of %s", ages, staleAfter) + } +} + +// settleWake bounds the one NEGATIVE rendezvous in this file: "the connector +// did not take another wake". Nothing can satisfy it, so it is deliberately far +// shorter than the watchdog — it is a wall-clock cost the passing path always +// pays, and the price of the assertion having teeth. No virtual time and no +// timing assertion depends on it; it only orders a wake that already happened +// (or did not) ahead of the next clock advance. +const settleWake = 250 * time.Millisecond + +// TestGracePhaseIsImmuneToFrameResets is the first half of acceptance +// criterion 3. The grace phase is a fixed deadline read at the deferral, not a +// window the peer can push: it bounds how long the connector waits for an +// abandoned seam call, which has nothing to do with whether the peer is still +// talking. +// +// The distinction is sharper than "does it eventually end", and the sharper +// form is what is asserted. A frame arriving inside the phase re-arms staleness +// and thereby hands the wait a WAKE, and a wake is enough to carry an +// implementation that also moved its deadline to a correct-looking teardown. +// So the test consumes that wake first, and then requires the phase to reach +// its deadline on a wake of its OWN — which only a deadline the frame could not +// move can produce. +func TestGracePhaseIsImmuneToFrameResets(t *testing.T) { + store := feedtest.NewStore() + store.Stored("pos-0") + var mu sync.Mutex + var ages []time.Duration + h, deferral := staleDeferralHarness(t, store, &ages, &mu) + h.polls.StallNext() + h.drainHandled() + queued := make(chan struct{}, eventfeed.ExportPumpDepth) + h.conn.OnPumpHandedOff(func(bool) { + select { + case queued <- struct{}{}: + default: + } + }) + wakes := make(chan struct{}, 8) + h.conn.OnSupersededWake(func() { + select { + case wakes <- struct{}{}: + default: + } + }) + + var conn *feedtest.Conn + h.polls.OnCall(func(feedtest.PollCall) { + h.clock.Advance(staleAfter) + if !deferral.await() { + return + } + // Half a window into the grace phase, the peer speaks again. + h.clock.Advance(staleAfter / 2) + drain(queued) + drain(wakes) + conn.Serve(framePing()) + select { + case <-queued: + case <-time.After(watchdog): + t.Error("the ping never reached the hand-off queue") + return + } + // The ping's re-arm is legitimate and so is any wake it delivers — + // wakes may be early, and the deadline is what decides. Consuming it + // here is what leaves the phase nothing but its own wake to finish on. + select { + case <-wakes: + case <-time.After(settleWake): + } + // The remaining half still lapses the grace phase: had the ping moved + // it, nothing fires here and the wait is left holding a window half a + // window out of reach, with the wake it might have ridden already spent. + h.clock.Advance(staleAfter / 2) + }) + h.start() + conn = h.driveToSubscribed() + conn.Serve(frameConfirm(noFilterIdentifier)) + + h.awaitTimer(timerBackoff) + if !conn.Closed() { + t.Fatal("a frame inside the grace phase must not extend it") + } + assertPositions(t, store.Saves()) + mu.Lock() + defer mu.Unlock() + if len(ages) != 1 { + t.Fatalf("StaleConnection ages = %v, want exactly one", ages) + } +} + +// TestGracePhaseIsImmuneToPumpSuspension is the second half of acceptance +// criterion 3, and the one the previous borrowed bound failed. +// +// A blocked hand-off suspends staleness EVALUATION, on the premise that a full +// queue proves the peer is outrunning a connector that is still consuming. This +// wait is the one that deliberately stops consuming — an outcome is already +// parked in the single deferral slot — so the premise is false by construction +// here, and a bound that borrows the staleness VERDICT is re-armed forever by a +// peer that keeps sending. The grace phase is a deadline instead, and a +// deadline cannot be suspended. +func TestGracePhaseIsImmuneToPumpSuspension(t *testing.T) { + store := feedtest.NewStore() + store.Stored("pos-0") + var mu sync.Mutex + var ages []time.Duration + h, deferral := staleDeferralHarness(t, store, &ages, &mu) + h.polls.StallNext() + h.drainHandled() + blocked := make(chan struct{}, 1) + h.conn.OnPumpBlocked(func() { + select { + case blocked <- struct{}{}: + default: + } + }) + + var conn *feedtest.Conn + h.polls.OnCall(func(feedtest.PollCall) { + h.clock.Advance(staleAfter) + if !deferral.await() { + return + } + // A peer that keeps sending fills the queue nothing is draining. + for i := 0; i <= eventfeed.ExportPumpDepth; i++ { + conn.Serve(framePing()) + } + select { + case <-blocked: + case <-time.After(watchdog): + t.Error("the pump never blocked on a full hand-off queue") + return + } + // One grace phase from the deferral, with the suspension in force. + h.clock.Advance(staleAfter) + }) + h.start() + conn = h.driveToSubscribed() + conn.Serve(frameConfirm(noFilterIdentifier)) + + h.awaitTimer(timerBackoff) + if !conn.Closed() { + t.Fatal("a blocked hand-off must not suspend the grace phase") + } + assertPositions(t, store.Saves()) + mu.Lock() + defer mu.Unlock() + if len(ages) != 1 { + t.Fatalf("StaleConnection ages = %v, want exactly one", ages) + } +} + +// TestDeferredFatalOutranksADeferredStalenessExpiry is acceptance criterion 4's +// probe half. The deferral slot's third form carries no frame, and the +// protocol-fatal probe runs at every deferred-disposition boundary — so it must +// read the slot as "not a frame" explicitly rather than by accident of what +// parsing a zero item happens to return, and it must still reach the fatal +// frame the pump queued behind the expiry. +// +// The server's own verdict outranks the connector's: an +// `invalid_event_stream_command` already read is Terminal(protocol_fatal), not +// a staleness reconnect. +func TestDeferredFatalOutranksADeferredStalenessExpiry(t *testing.T) { + store := feedtest.NewStore() + store.Stored("pos-0") + var mu sync.Mutex + var ages []time.Duration + h, deferral := staleDeferralHarness(t, store, &ages, &mu) + // A second mint that fails terminally: a connector that wrongly reconnects + // ends at Terminal(mint_failed) with two mint calls, so both assertions + // below discriminate rather than merely time out. + h.minter.ScriptError(&eventfeed.MintError{Kind: eventfeed.MintUnrecoverable}) + h.polls.StallNext() + h.drainHandled() + queued := make(chan struct{}, eventfeed.ExportPumpDepth) + h.conn.OnPumpHandedOff(func(bool) { + select { + case queued <- struct{}{}: + default: + } + }) + + var conn *feedtest.Conn + h.polls.OnCall(func(feedtest.PollCall) { + h.clock.Advance(staleAfter) + if !deferral.await() { + return + } + // The fatal frame is served only once the expiry holds the slot, and + // the wait is for the pump's hand-off: the guarantee is about a frame + // the state machine can observe, so a test that raced the read would + // prove nothing. + drain(queued) + conn.Serve(frameDisconnect("invalid_event_stream_command", false)) + select { + case <-queued: + case <-time.After(watchdog): + t.Error("the fatal frame never reached the hand-off queue") + return + } + h.clock.Advance(staleAfter) + }) + h.start() + conn = h.driveToSubscribed() + conn.Serve(frameConfirm(noFilterIdentifier)) + h.awaitEndOrReconnect() + h.join() + + _, terminal, _ := h.snapshot() + if terminal == nil || terminal.Reason != eventfeed.ReasonProtocolFatal { + t.Fatalf("terminal = %v, want reason %q — the server's verdict was already read", + terminal, eventfeed.ReasonProtocolFatal) + } + if got := h.minter.Calls(); got != 1 { + t.Fatalf("mint calls = %d, want 1: a protocol-fatal rejection must not reconnect from any state", got) + } + assertPositions(t, store.Saves()) + mu.Lock() + defer mu.Unlock() + if len(ages) != 0 { + t.Fatalf("StaleConnection ages = %v, want none — the fatal verdict governs", ages) + } +} + +// TestSuspendedStalenessDuringAnInFlightPollDoesNotDefer is the other half of +// acceptance criterion 1: the new cases in the in-flight-poll select evaluate +// the firing under the SAME rule every other select applies, rather than +// treating any firing as the socket's death. +// +// A firing whose window overlapped a blocked hand-off is not evidence — a full +// queue proves the peer was sending faster than the connector consumed, the +// opposite of a dead peer — so it is disregarded and re-armed, and the call is +// still awaited. Deferring it instead abandons a live socket and a page the +// server had already served. +// +// The arrangement is the only one that holds the suspension open across the +// select: this wait DOES drain the hand-off queue, so a pump blocked while the +// state machine is sitting in the select would be released by the very next +// receive. Parking the state machine inside a frame-handled callback is what +// keeps it from draining while the peer fills the queue. +func TestSuspendedStalenessDuringAnInFlightPollDoesNotDefer(t *testing.T) { + store := feedtest.NewStore() + store.Stored("pos-0") + var mu sync.Mutex + var ages []time.Duration + h, _ := staleDeferralHarness(t, store, &ages, &mu) + h.polls.ScriptPage(eventfeed.PollPage{Position: "pos-1"}) + + gate := make(chan struct{}) + parked := make(chan struct{}) + var parkOnce sync.Once + h.conn.OnFrameHandled(func(kind string) { + if kind != "ping" { + return + } + parkOnce.Do(func() { + close(parked) + <-gate + }) + }) + blocked := make(chan struct{}, 1) + h.conn.OnPumpBlocked(func() { + select { + case blocked <- struct{}{}: + default: + } + }) + + var conn *feedtest.Conn + var once sync.Once + h.polls.OnCall(func(feedtest.PollCall) { + once.Do(func() { + // The state machine parks inside the callback for this ping, so it + // stops dequeuing while the peer keeps sending. + conn.Serve(framePing()) + select { + case <-parked: + case <-time.After(watchdog): + t.Error("the state machine never parked in the frame-handled callback") + return + } + for i := 0; i <= eventfeed.ExportPumpDepth; i++ { + conn.Serve(framePing()) + } + select { + case <-blocked: + case <-time.After(watchdog): + t.Error("the pump never blocked on a full hand-off queue") + return + } + // The window closes over a hand-off the pump spent blocked. + h.clock.Advance(staleAfter) + close(gate) + }) + }) + h.start() + conn = h.driveToSubscribed() + conn.Serve(frameConfirm(noFilterIdentifier)) + + // The socket survives, the page is served, and the walk reaches its head: + // a disregarded firing consumes neither the call nor the connection. + h.awaitStreaming() + if conn.Closed() { + t.Fatal("a firing whose window overlapped a blocked hand-off must not tear the socket down") + } + if _, terminal, _ := h.snapshot(); terminal != nil { + t.Fatalf("a suspended staleness firing is never terminal; got %v", terminal) + } + assertPositions(t, store.Saves(), "pos-1") + mu.Lock() + defer mu.Unlock() + if len(ages) != 0 { + t.Fatalf("StaleConnection ages = %v, want none — the firing was not evidence", ages) + } +} diff --git a/go/pkg/basecamp/eventfeed/connector.go b/go/pkg/basecamp/eventfeed/connector.go index fd17ae4640..00503887c0 100644 --- a/go/pkg/basecamp/eventfeed/connector.go +++ b/go/pkg/basecamp/eventfeed/connector.go @@ -239,6 +239,15 @@ type testHooks struct { // on", which is what makes transition 21's deferral assertable without // racing the seam call's own completion. frameDeferred func() + // supersededWake fires each time the grace phase that follows a deferral + // takes a WAKE — a staleness firing or a window swap — and before the + // deadline that wake is checked against. Wakes are allowed to be early, and + // the deadline is what decides; the difference is invisible from outside, + // which is why the phase's immunity to frame resets could not be asserted + // without it. A frame arriving inside the phase re-arms staleness and + // produces exactly one such wake, and a test must know that wake has been + // consumed before it can assert the phase still ends on its own. + supersededWake func() // subscribeWritten fires once the subscribe command has been written and // BEFORE the phase deadline is stopped. It exists for the one ordering // this package cannot otherwise assert: the deadline expiring in the diff --git a/go/pkg/basecamp/eventfeed/export_test.go b/go/pkg/basecamp/eventfeed/export_test.go index 9238953b41..e0306eb53b 100644 --- a/go/pkg/basecamp/eventfeed/export_test.go +++ b/go/pkg/basecamp/eventfeed/export_test.go @@ -71,6 +71,13 @@ func (c *Connector) OnPumpHandedOff(f func(isErr bool)) { c.hooks.pumpHandedOff // and the completed call. func (c *Connector) OnFrameDeferred(f func()) { c.hooks.frameDeferred = f } +// OnSupersededWake registers a hook fired each time the grace phase following a +// deferral takes a wake, before that wake's deadline check. It is the +// rendezvous for "the connector has consumed the wake a frame's staleness +// re-arm produced" — the only way to tell a phase that ends on its own deadline +// from one a peer's frame carried it to. +func (c *Connector) OnSupersededWake(f func()) { c.hooks.supersededWake = f } + // ExportPumpDepth is the frame pump's bounded hand-off queue depth: the // number of frames a test must serve to make the pump block. const ExportPumpDepth = pumpDepth diff --git a/go/pkg/basecamp/eventfeed/loop.go b/go/pkg/basecamp/eventfeed/loop.go index 4cf7335823..5f1746d134 100644 --- a/go/pkg/basecamp/eventfeed/loop.go +++ b/go/pkg/basecamp/eventfeed/loop.go @@ -422,6 +422,43 @@ func (h *staleHolder) evaluate(gen int) (time.Duration, bool) { return h.clock.Now().Sub(h.last), true } +// graceWake re-arms the staleness timer purely as a WAKE, after an +// authoritative expiry has already been decided and parked. It is the grace +// phase's only source of wake-ups (SPEC.md §23 "the bound on a superseded poll +// seam call"), and it is deliberately not a re-armed staleness WINDOW: the +// verdict is already rendered, and nothing re-evaluates it. +// +// Two things make it that rather than an ordinary re-arm, and both are +// requirements of the phase it wakes rather than conveniences. +// +// It LATCHES the expiry, which is what makes the phase immune to frame resets +// and to suspension. Once expired is set, arm returns false to everyone: the +// pump's per-frame reset no longer swaps the timer out, and neither does a +// blocked hand-off's release. The wake therefore lands at a fixed instant no +// peer can push, which is the property the phase needs — it bounds how long the +// connector waits for an abandoned seam call, and that has nothing to do with +// whether the peer is still talking. +// +// And it replaces the FIRED timer rather than leaving it. An authoritative +// firing is consumed from the channel, and latching stops anything from arming +// another, so after the expiry there is no wake left at all — which is exactly +// how the bounded wait became unbounded again. +// +// It arms the same `staleness` kind: SPEC.md §23 pins exactly six timer kinds +// and every state's exact timer set, both asserted by the cross-SDK fixtures, +// so a dedicated grace timer would be a spec change across six SDKs. The +// outstanding set is unchanged — {staleness} stays {staleness}. +func (h *staleHolder) graceWake() { + h.mu.Lock() + defer h.mu.Unlock() + if h.stopped { + return + } + h.expired = true + h.timer.Stop() + h.timer = h.clock.NewTimer(h.d, timerStaleness) +} + // stop cancels the timer permanently. func (h *staleHolder) stop() { h.mu.Lock() From bc26ba94ea819b440c44f9577f92d86387cae7aa Mon Sep 17 00:00:00 2001 From: Jeremy Daer Date: Wed, 19 Aug 2026 23:17:56 -0700 Subject: [PATCH 21/51] Event feed: two of Codex's four, and an honest account of a third MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Four threads arrived on the push. One is stale — the connection returned alongside a dial error is already closed before classification, two commits back. The post-confirmation welcome is real and filed as #800, with the reason it is not a one-liner: that dispatch arm has no bounded-write discipline, and §23 does not say whether a retransmit re-arms the confirmation deadline. The nil dial result was mine, introduced when I added the (nil, nil) guard two rounds ago. Every other exit from that select stops the handshake deadline; this one returned without doing so, carrying an armed handshake-deadline into Backoff — whose exact outstanding set §23 pins at {backoff} — and a transport returning (nil, nil) repeatedly accumulates one ghost timer per cycle, each firing later into a state with no edge for it. The test asserts the exact SET rather than the absence of a panic, because a defensive edge that leaves a timer behind has only traded a crash for a leak. Mutation confirms it: map[backoff:1 handshake-deadline:1]. The terminal race is real and narrower than it reads. Close latches isClosed and THEN cancels, both inside one critical section, so a reader of runCtx landing between those two statements sees a live context while Close is already committed and about to return — and publishes an error element to a consumer that closed. emitTerminal now decides by claiming under the mutex that owns the latch, which has no such interval: the claim runs wholly before or wholly after Close's critical section. What it does NOT do is stop a terminal from being published after Close returns, and the shape it replaced did not either — a claim that wins publishes, and Close was then called against a feed that had already terminated. That is correct, and worth stating so the next reader does not read more into the change than it does. No test discriminates it, and the first one I wrote passed against the code it was meant to fail — the hook sat where both versions behave identically, which is the same placement error I criticised in the pump-read hook a round ago. It is deleted rather than kept as decoration. Reaching the real window needs a hook between two adjacent statements inside Close, existing for nothing else, and even then the behaviour it replaces is racy rather than reliably wrong. The change is kept because deciding against the authoritative latch is the correct shape, not because a proof forced it. --- go/pkg/basecamp/eventfeed/connector.go | 35 ++++++++++++++++++++ go/pkg/basecamp/eventfeed/loop.go | 25 +++++++++++++- go/pkg/basecamp/eventfeed/loop_test.go | 46 ++++++++++++++++++++++++++ 3 files changed, 105 insertions(+), 1 deletion(-) diff --git a/go/pkg/basecamp/eventfeed/connector.go b/go/pkg/basecamp/eventfeed/connector.go index 00503887c0..4ec98a290c 100644 --- a/go/pkg/basecamp/eventfeed/connector.go +++ b/go/pkg/basecamp/eventfeed/connector.go @@ -120,6 +120,9 @@ type config struct { // rand is the uniform [0, 1) source behind the jitter draws — white-box // overridable in tests, no public option. rand func() float64 + // claimTerminal serializes the publication of a terminal element against + // Close. Installed by Events, which is the only thing that builds a run. + claimTerminal func() bool } // Option configures a Connector, functional-options house style (SPEC.md §23 @@ -198,6 +201,10 @@ type Connector struct { // cancelRun cancels the active run's context. Registered by Events for // exactly the span of one iteration, nil otherwise. cancelRun context.CancelFunc + // terminalClaimed records that a terminal element won the race against + // Close and will be published. It is written under this mutex so the two + // decisions are ordered rather than merely likely to be. + terminalClaimed bool // runDone is closed when the iteration function returns. Published by // Events at the start of a run and never cleared, so Wait either finds no // run at all or a channel that is closed exactly when the run has exited. @@ -432,6 +439,9 @@ func (c *Connector) Events(ctx context.Context) iter.Seq2[Event, error] { // cancels it itself. Either way runCtx is done before Close returns — // there is no window in which the run proceeds past a returned Close. c.mu.Lock() + // Installed before the loop is built, and under the same mutex Close + // takes, so the run cannot reach a terminal before the claim exists. + c.cfg.claimTerminal = c.claimTerminal c.runDone = done if c.isClosed { // Closed before the first iteration: end cleanly with zero wire @@ -513,6 +523,31 @@ func (c *Connector) Events(ctx context.Context) iter.Seq2[Event, error] { // thing Close promises does not happen. Holding the mutex across the cancel is // safe: a CancelFunc runs no consumer code, and any context.AfterFunc it // triggers runs on its own goroutine, so it cannot re-enter Close. +// claimTerminal decides, atomically against Close, whether a terminal element +// may still be published. +// +// emitTerminal used to read the run context and then yield, which is two acts +// with a gap: a Close landing in that gap cancels the run and RETURNS while the +// yield goes ahead, so a consumer that closed still receives an error element — +// the one thing §23 says close() does not produce. A context read cannot fix +// that, because the thing being raced is not the context's value but the order +// of two decisions. +// +// So the decision moves under the mutex Close already serializes on. Whichever +// call takes it first wins outright: a Close that arrives first makes this +// return false and the run takes the Closed edge, while a terminal claimed +// first is published even though Close cancels a moment later — which is +// correct, because the feed had already terminated when the consumer closed. +func (c *Connector) claimTerminal() bool { + c.mu.Lock() + defer c.mu.Unlock() + if c.isClosed { + return false + } + c.terminalClaimed = true + return true +} + func (c *Connector) Close() error { c.mu.Lock() c.isClosed = true diff --git a/go/pkg/basecamp/eventfeed/loop.go b/go/pkg/basecamp/eventfeed/loop.go index 5f1746d134..a022883021 100644 --- a/go/pkg/basecamp/eventfeed/loop.go +++ b/go/pkg/basecamp/eventfeed/loop.go @@ -732,7 +732,22 @@ func (l *loop) run(yield func(Event, error) bool) { // check is a rule every future select has to remember, which is the shape that // produced this defect in the first place. func (l *loop) emitTerminal(term *TerminalError) { - if l.runCtx.Err() != nil { + // The claim is atomic against Close (see Connector.claimTerminal), and + // what it closes is narrow enough to name exactly. Close latches isClosed + // and THEN cancels, both inside one critical section; a reader of runCtx + // landing between those two statements sees a live context while Close is + // already committed and about to return, and publishes a terminal element + // to a consumer that closed. Deciding against the latch under the lock + // that owns it has no such interval — the claim runs wholly before or + // wholly after Close's critical section. + // + // No test discriminates this, and pretending otherwise would be worse than + // saying so: the window is two adjacent statements inside Close, reaching + // it needs a hook in production code that exists for nothing else, and + // even then the behaviour it replaces is racy rather than reliably wrong. + // It is kept because deciding against the authoritative latch is simply + // the correct shape, not because a red proof forced it. + if !l.cfg.claimTerminal() { l.setState(stateClosed) return } @@ -845,6 +860,14 @@ func (l *loop) runCycle(delay time.Duration) cycleOutcome { // transient dial failure keeps the reconnect cycle honest and // costs one backoff round against a transport that is already // violating its contract. + // + // The handshake deadline is stopped FIRST, exactly as the error + // branch above does. Leaving it armed carries it into Backoff, + // whose exact outstanding-timer set is {backoff} — and a + // transport returning this repeatedly accumulates one ghost + // handshake-deadline per cycle, each of which fires later into a + // state that has no edge for it. + hs.Stop() return cycleOutcome{kind: outcomeFailed} } conn = r.conn diff --git a/go/pkg/basecamp/eventfeed/loop_test.go b/go/pkg/basecamp/eventfeed/loop_test.go index 45d4877915..f9bf33188b 100644 --- a/go/pkg/basecamp/eventfeed/loop_test.go +++ b/go/pkg/basecamp/eventfeed/loop_test.go @@ -2175,3 +2175,49 @@ func TestObservableCloseErrorCarriesNoPeerReason(t *testing.T) { t.Errorf("field-wise rendering still differs from a code-only close error: %+v", ce) } } + +// nilDialTransport returns (nil, nil) from Dial exactly once — a seam +// violating its own contract — and then delegates. Nothing in feedtest can +// express this, deliberately: it is not a shape a compliant transport +// produces, and the connector's handling of it is a defensive edge rather +// than a scripted one. +type nilDialTransport struct { + inner eventfeed.CableTransport + once sync.Once +} + +func (t *nilDialTransport) Dial(ctx context.Context, url string, maxFrameBytes int64) (eventfeed.CableConn, error) { + first := false + t.once.Do(func() { first = true }) + if first { + return nil, nil + } + return t.inner.Dial(ctx, url, maxFrameBytes) +} + +// TestNilDialResultLeavesBackoffsTimerSetExact: a (nil, nil) dial takes the +// transient failure edge, and must arrive in Backoff with the SAME exact timer +// set every other transient failure produces. +// +// The handshake deadline is armed before the dial and every other exit from +// this select stops it. This branch did not, so the deadline rode into Backoff +// — whose exact set §23 pins at {backoff} — and a transport returning (nil, +// nil) repeatedly accumulated one ghost handshake-deadline per cycle, each +// firing later into a state with no edge for it. +// +// The assertion is the exact SET rather than "no panic", because the panic +// this branch prevents was never the interesting part: a defensive edge that +// leaves a timer behind has traded a crash for a leak. +func TestNilDialResultLeavesBackoffsTimerSetExact(t *testing.T) { + h := newHarness(t) + h.tr = feedtest.NewTransport() + tr := &nilDialTransport{inner: h.tr} + h2 := newHarness(t, eventfeed.WithTransport(tr)) + h2.minter.ScriptTicket(ticket(1)) + h2.minter.ScriptTicket(ticket(2)) + h2.start() + + // The dial returns (nil, nil), the cycle fails, and Backoff is entered. + h2.awaitTimer(timerBackoff) + assertTimers(t, h2.clock, map[string]int{timerBackoff: 1}) +} From dab67a85a1808e9678a16aeb6644da5e0ddfd5ed Mon Sep 17 00:00:00 2001 From: Jeremy Daer Date: Fri, 21 Aug 2026 23:12:00 -0700 Subject: [PATCH 22/51] Event feed: cancellation could not reach the one exit, and the origin was checked after the mangle MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two of Codex's three P2s taken; the third declined below. emitTerminal decides against Close by claiming under the mutex that owns the isClosed latch — and against nothing else. §23's universal edge reads "close() / cancellation / consumer break", and Events promises all three end iteration with no error element, but a caller's cancel is not Close: it sets no latch and holds no connector lock, so a ctx cancelled while a subscription-rejection teardown was still closing the socket lost nothing at the claim, and the consumer was handed the subscription_rejected element after it had cancelled. The window is not two adjacent statements this time — it spans the whole disposal, socket close and pump join included — so it proves red deterministically: a transport whose first Close parks until the test cancels the caller context failed with "got 1 elements, terminal ... subscription_rejected". The exit now reads the run context before the claim. That read is the achievable ordering, and the comment says so: nothing serializes an external cancel, so a cancel observed at the exit takes the Closed edge, and one landing after the read publishes the already-claimed terminal — the same tolerance the claim grants a Close that loses. validateConfig ran checkIdentityText("the API base origin", ...) on the CANONICAL origin, and canonicalization is the lossy step the check exists to precede: strings.ToLower rewrites every invalid byte to U+FFFD, so two origins differing only in an invalid byte — checkIdentityText's own doc names 0xff and 0xfe — collapsed to one canonical form, validated clean, and became CheckpointKey.Origin for a single shared lineage, each consumer skipping the other's events. The red run shows the collapse on its face: both new table rows print New succeeding with the identical https://.example.com config. The origin is now validated raw, before CanonicalOrigin, alongside the accountID and namespace checks that already were; canonicalization of valid UTF-8 cannot produce invalid UTF-8, so the canonical form needs no second check. Codex's third finding — check the confirmation deadline's failed Stop before accepting a matching reject_subscription — is declined: §23 pins rejection as always terminal, first attempt or reconnect, so a lapse-first reading would re-mint, re-dial, and re-send the byte-identical subscribe only to collect the same rejection one wire cycle later. The confirm branch's Stop guard prevents a lapse becoming a false success, and terminal is no false anything. --- go/pkg/basecamp/eventfeed/connector.go | 14 +++- go/pkg/basecamp/eventfeed/connector_test.go | 79 +++++++++++++++++++++ go/pkg/basecamp/eventfeed/loop.go | 16 ++++- 3 files changed, 105 insertions(+), 4 deletions(-) diff --git a/go/pkg/basecamp/eventfeed/connector.go b/go/pkg/basecamp/eventfeed/connector.go index 4ec98a290c..e75ec8e84b 100644 --- a/go/pkg/basecamp/eventfeed/connector.go +++ b/go/pkg/basecamp/eventfeed/connector.go @@ -320,6 +320,17 @@ func validateConfig(cfg *config) error { if cfg.origin == "" { return usageError("an API base origin is required") } + // Validated RAW, before canonicalization: CanonicalOrigin lowercases + // through strings.ToLower, which rewrites every invalid byte to U+FFFD, + // so a post-canonical check would pass two origins differing only in + // invalid bytes AFTER they had collapsed to one canonical form — one + // checkpoint lineage for two configured origins, the exact many-to-one + // identity checkIdentityText exists to refuse. Canonicalization of valid + // UTF-8 cannot produce invalid UTF-8, so the canonical form needs no + // second check. + if err := checkIdentityText("the API base origin", cfg.origin); err != nil { + return err + } canonical, err := CanonicalOrigin(cfg.origin) if err != nil { return usageError(err.Error()) @@ -327,9 +338,6 @@ func validateConfig(cfg *config) error { if err := checkOriginScheme(canonical); err != nil { return err } - if err := checkIdentityText("the API base origin", canonical); err != nil { - return err - } cfg.origin = canonical if cfg.accountID == "" { return usageError("accountID must be non-empty") diff --git a/go/pkg/basecamp/eventfeed/connector_test.go b/go/pkg/basecamp/eventfeed/connector_test.go index ccdd6d2546..ce593f4ac2 100644 --- a/go/pkg/basecamp/eventfeed/connector_test.go +++ b/go/pkg/basecamp/eventfeed/connector_test.go @@ -103,6 +103,14 @@ func TestNewValidation(t *testing.T) { {"invalid utf-8 filter type", testOrigin, "1", minter, polls, []eventfeed.Option{eventfeed.WithFilters(eventfeed.Filters{Types: []string{"message.\xff"}})}, "UTF-8"}, + // The origin must be validated RAW, before canonicalization: + // CanonicalOrigin lowercases through strings.ToLower, which rewrites + // every invalid byte to U+FFFD, so a post-canonical check passes a + // collapsed identity — these two rows would otherwise canonicalize to + // ONE origin and share one checkpoint lineage, the exact collapse + // checkIdentityText exists to refuse (checkpoint.go). + {"invalid utf-8 origin host (0xff)", "https://\xff.example.com", "1", minter, polls, nil, "UTF-8"}, + {"invalid utf-8 origin host (0xfe)", "https://\xfe.example.com", "1", minter, polls, nil, "UTF-8"}, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { @@ -1413,6 +1421,77 @@ func TestCloseFromConnectedOutranksAQueuedFatalFrame(t *testing.T) { } } +// parkingCloseTransport wraps the harness transport so the FIRST socket +// close parks until the test releases it — a deterministic hold inside a +// teardown's disposal, after the terminal outcome is selected and before +// emitTerminal publishes it. +type parkingCloseTransport struct { + inner *feedtest.Transport + parked chan struct{} + release chan struct{} + once sync.Once +} + +func (p *parkingCloseTransport) Dial(ctx context.Context, wsURL string, maxFrameBytes int64) (eventfeed.CableConn, error) { + conn, err := p.inner.Dial(ctx, wsURL, maxFrameBytes) + if err != nil { + return nil, err + } + return &parkingCloseConn{CableConn: conn, tr: p}, nil +} + +type parkingCloseConn struct { + eventfeed.CableConn + tr *parkingCloseTransport +} + +func (c *parkingCloseConn) Close(code int, reason string) error { + c.tr.once.Do(func() { close(c.tr.parked) }) + <-c.tr.release + return c.CableConn.Close(code, reason) +} + +// TestCallerCancellationOutranksATerminalMidTeardown is the caller-cancel +// sibling of TestCloseFromConnectedOutranksAQueuedFatalFrame. §23's universal +// edge reads "close() / cancellation / consumer break", and Events promises +// all three end iteration with NO error element — but the terminal claim +// serializes publication against Close alone, so a caller cancelling ctx +// while a subscription-rejection teardown was still closing the socket was +// handed the subscription_rejected element anyway. The parked Close makes +// the interleaving deterministic rather than a repeated-rounds race: the +// terminal outcome is already selected, the teardown is mid-close, and the +// caller cancels strictly before publication. +func TestCallerCancellationOutranksATerminalMidTeardown(t *testing.T) { + pt := &parkingCloseTransport{parked: make(chan struct{}), release: make(chan struct{})} + h := newHarness(t, eventfeed.WithTransport(pt)) + pt.inner = h.tr + var releaseOnce sync.Once + release := func() { releaseOnce.Do(func() { close(pt.release) }) } + // A parked teardown must never outlive the test, however it ended. + t.Cleanup(release) + h.minter.ScriptTicket(ticket(1)) + h.start() + + h.driveToSubscribed().Serve(frameReject(noFilterIdentifier)) + select { + case <-pt.parked: + case <-time.After(watchdog): + t.Fatal("the reject teardown never reached the socket close") + } + h.cancel() + release() + h.join() + + if _, terminal, elements := h.snapshot(); terminal != nil || elements != 0 { + t.Fatalf("cancellation must end iteration with no error element; got %d elements, terminal %v", + elements, terminal) + } + if sock := h.tr.LastConn(); sock == nil || !sock.Closed() { + t.Fatal("the rejected socket must still be closed") + } + assertTimers(t, h.clock, map[string]int{}) +} + // ctxAwareStore is a CheckpointStore that HONORS its context, which the // interface permits and this package's own tests never exercised: both // feedtest.Store and the built-in FileCheckpointStore ignore ctx, so a save diff --git a/go/pkg/basecamp/eventfeed/loop.go b/go/pkg/basecamp/eventfeed/loop.go index a022883021..6e8734ec59 100644 --- a/go/pkg/basecamp/eventfeed/loop.go +++ b/go/pkg/basecamp/eventfeed/loop.go @@ -747,7 +747,21 @@ func (l *loop) emitTerminal(term *TerminalError) { // even then the behaviour it replaces is racy rather than reliably wrong. // It is kept because deciding against the authoritative latch is simply // the correct shape, not because a red proof forced it. - if !l.cfg.claimTerminal() { + // + // Caller cancellation is the same universal edge — §23 draws it as + // "close() / cancellation / consumer break", and Events promises all + // three end iteration with no error element — but no mutex serializes + // it: the caller's cancel is not Close and holds no connector lock, so + // there is no latch to consult. The run-context read below is the + // achievable ordering — best-effort, at the same one exit the claim + // guards. A cancel observed here takes the Closed edge even though the + // terminal outcome was already selected (the teardown between selection + // and publication can span a socket close, so the cancel may precede + // this point by an arbitrarily long, fully observable interval); one + // landing after the read publishes the already-claimed terminal, the + // same tolerance the claim's comment grants a Close that cancels a + // moment after losing the claim. + if l.runCtx.Err() != nil || !l.cfg.claimTerminal() { l.setState(stateClosed) return } From 383fc3da8fe6247eeb956d7b53c515bad3666692 Mon Sep 17 00:00:00 2001 From: Jeremy Daer Date: Sat, 22 Aug 2026 00:04:46 -0700 Subject: [PATCH 23/51] Event feed: the oversize verdict survived the seam but not the sanitizer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Completes 6ea9d1279 (event-feed-foundations), which typed the transport's read-limit abort: coder/websocket rejects an over-limit message during the read, and ReadFrame now returns the exported ErrFrameOversize sentinel flat. This branch's run loop never learned that vocabulary. observableSocketError preserves package-owned sentinels, *CloseError, and *invalidFrameError, and degrades every stranger to errSocketFailed — so the one invalid-frame shape the SPEC assigns to the transport arrived typed and left generic, and Observer.Disconnected lost the indication §23 requires it to carry. The contract is the invalid-frame class (SPEC.md §23, the three-shapes bullet — SPEC.md:2810-2821 at this revision): "a frame exceeding EVENT_FEED_MAX_FRAME_BYTES" is one of "three shapes, one disposition", each dispatched as a socket failure "with Observer.disconnected carrying an invalid-frame indication; never an untyped decoder error escaping", and "the size check binds inside the transport". The disposition was already right — a pump read error takes the current state's socket-failure edge to Backoff regardless of type — so this is observability, not a verdict change. Classification design: the sentinel passes through, rather than widening invalidFrameError with a third shape. The sanitizer's own taxonomy decides it. invalidFrameError names violations the CONNECTOR judged in frames it read; the oversize frame is never materialized, so there is nothing for the connector to classify — the seam's sentinel IS the classification. A third shape would also widen the security invariant's pinned rendering vocabulary ("names its shape (frame_parse / event_decode) and nothing else") for no gain. The new arm follows the sentinel rule already stated there: reduced to the bare package value, never the argument, because a seam's wrapper text is where a cable URL rides. ErrFrameOversize is exported so seams can return it, but a var can only be referenced — unlike the exported struct types the sanitizer evicted, it cannot be rebuilt around peer text. feedtest.Conn now wraps the sentinel in its read-limit violation, aligning the fake with the seam contract its interface documents — an untyped fake would keep every test green against a classification the real transport defeats. The red proof stages an actually oversized frame through the pump (one byte past the dial's own recorded cap) and failed against the pre-fix sanitizer with "Disconnected err = event feed socket failed, want the ErrFrameOversize indication"; green after, asserting both the errors.Is match and the bare-sentinel rendering, with the Backoff timer arming to pin the disposition unchanged. --- go/pkg/basecamp/eventfeed/loop.go | 17 ++++++++-- go/pkg/basecamp/eventfeed/loop_test.go | 43 ++++++++++++++++++++++++++ go/pkg/basecamp/eventfeed/seams.go | 6 ++-- 3 files changed, 62 insertions(+), 4 deletions(-) diff --git a/go/pkg/basecamp/eventfeed/loop.go b/go/pkg/basecamp/eventfeed/loop.go index 6e8734ec59..73dd1ecd91 100644 --- a/go/pkg/basecamp/eventfeed/loop.go +++ b/go/pkg/basecamp/eventfeed/loop.go @@ -1400,8 +1400,11 @@ var errSocketFailed = errors.New("event feed socket failed") // // So neither passes through any more. The three that remain are safe by // construction rather than by convention: the sentinels are package-level -// values whose text is fixed in this file, *CloseError renders an integer code -// and nothing else, and *invalidFrameError renders one of two shape constants. +// values whose text is fixed in this package (ErrFrameOversize is exported so +// the transport seam can RETURN it, but a var can only be referenced — unlike +// an exported struct type, it cannot be rebuilt around peer text), *CloseError +// renders an integer code and nothing else, and *invalidFrameError renders one +// of two shape constants. // // Nothing is lost on the live path. Dial failures do not reach this function // at all — they are dispatched before any teardown is observed — and a @@ -1433,6 +1436,16 @@ func observableSocketError(err error) error { return errStaleConnection case errors.Is(err, errCableConnClosed): return errCableConnClosed + case errors.Is(err, ErrFrameOversize): + // §23's invalid-frame class, size shape. The size check binds inside + // the transport — the frame is rejected during the read, never + // materialized — so the sentinel the seam returned IS the + // classification, and this is what carries the invalid-frame + // indication to Observer.Disconnected instead of degrading the one + // transport-decided shape to errSocketFailed. Reduced to the bare + // package value like every other sentinel arm: the seam's wrapper + // text is where a cable URL rides. + return ErrFrameOversize case errors.Is(err, context.Canceled): return context.Canceled case errors.Is(err, context.DeadlineExceeded): diff --git a/go/pkg/basecamp/eventfeed/loop_test.go b/go/pkg/basecamp/eventfeed/loop_test.go index f9bf33188b..5615b07547 100644 --- a/go/pkg/basecamp/eventfeed/loop_test.go +++ b/go/pkg/basecamp/eventfeed/loop_test.go @@ -1087,6 +1087,49 @@ func TestNullFrameTearsDownToBackoff(t *testing.T) { } } +// TestOversizeFrameKeepsItsInvalidFrameIndication: an inbound frame over the +// dial's max-frame-bytes cap is one of §23's three invalid-frame shapes, and +// the class's contract is "Observer.disconnected carrying an invalid-frame +// indication; never an untyped decoder error escaping" — with the size check +// binding inside the transport, which rejects the frame during the read and +// surfaces ErrFrameOversize (CableConn.ReadFrame). The classification must +// survive the observer sanitizer as the bare package sentinel — not the +// seam's wrapper (whose text is where a cable URL would ride), and not the +// generic errSocketFailed the closed vocabulary degrades strangers to. The +// disposition stays the socket-failure edge: Backoff, never terminal. +func TestOversizeFrameKeepsItsInvalidFrameIndication(t *testing.T) { + var mu sync.Mutex + var disconnects []error + h := newHarness(t, eventfeed.WithObserver(eventfeed.Observer{ + Disconnected: func(_ string, err error) { + mu.Lock() + disconnects = append(disconnects, err) + mu.Unlock() + }, + })) + h.minter.ScriptTicket(ticket(1)) + h.start() + + conn := h.driveToSubscribed() + readCap := h.tr.Dials()[0].MaxFrameBytes + conn.Serve(make([]byte, readCap+1)) + h.awaitTimer(timerBackoff) + + mu.Lock() + defer mu.Unlock() + if len(disconnects) != 1 { + t.Fatalf("Disconnected fired %d times, want 1", len(disconnects)) + } + got := disconnects[0] + if !errors.Is(got, eventfeed.ErrFrameOversize) { + t.Fatalf("Disconnected err = %v, want the ErrFrameOversize indication", got) + } + if got.Error() != eventfeed.ErrFrameOversize.Error() { + t.Fatalf("Disconnected err renders %q, want the bare sentinel %q (a wrapper's text is where a cable URL rides)", + got.Error(), eventfeed.ErrFrameOversize.Error()) + } +} + // TestFrameDerivedObserverTextIsBounded: both frame-derived strings // Disconnected can carry — a raw disconnect frame's reason, and the // invalid-frame rendering of a decoder error that quotes frame bytes — are diff --git a/go/pkg/basecamp/eventfeed/seams.go b/go/pkg/basecamp/eventfeed/seams.go index 9e4bfcff36..6109751f27 100644 --- a/go/pkg/basecamp/eventfeed/seams.go +++ b/go/pkg/basecamp/eventfeed/seams.go @@ -494,8 +494,10 @@ type Observer struct { // actually acts on are named exactly, an absent reason stays empty, and // every other peer-supplied string — however short — becomes "other". err // goes through observableSocketError, whose vocabulary is CLOSED and - // short: the staleness and conn-closed sentinels, the two context - // sentinels, *CloseError (which renders its integer code alone), and + // short: the staleness, conn-closed, and frame-oversize sentinels (the + // last being ErrFrameOversize, the invalid-frame class's one + // transport-decided shape), the two context sentinels, *CloseError (which + // renders its integer code alone), and // *invalidFrameError (which renders one of two shape constants). ANY other // error — including a *DialError or *TerminalError, both of which a seam // can construct and both of which render free text — becomes From 281eea95abc5c5125721287b9617d576d9d0bcce Mon Sep 17 00:00:00 2001 From: Jeremy Daer Date: Sat, 22 Aug 2026 01:58:02 -0700 Subject: [PATCH 24/51] Event feed: wakes are not verdicts, and three more places the order was the bug MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Five of this round's eight findings taken; the notable decline is recorded at the end. The continuation validator had the defect validateConfig was cured of two commits ago, on its other consumer: checkContinuation canonicalized the server-supplied URL first, and CanonicalOrigin's ToLower rewrites every invalid byte to U+FFFD — so a continuation host carrying a raw 0xff collapsed to the same canonical form as a configured origin that legitimately contains the replacement character (valid UTF-8, so construction accepts it), and §8 same-origin validation equated two distinct byte strings on the authenticated path. The raw bytes are refused before the lossy step; red proof: the collapsed URL was accepted against a U+FFFD base. Two waits treated a staleness firing as a verdict where §23 says it is a wake. In awaitSupersededPoll, the stale=false arm returned superseded on an authoritative expiry — but a read error does not reset staleness, so the PRE-EXISTING window, mostly spent when the failure was deferred, abandoned the in-flight poll almost immediately: a failure landing 7s into a 7.5s window granted the poll 0.5s of the grace phase §23 measures from the deferral ("wakes may be early or late, and the deadline is what decides"). The arm now latches through graceWake exactly as the stale caller does, and the deadline alone ends the wait — red proof: teardown on the old firing, green: a wake, then teardown only when virtual time reaches deferral + EVENT_FEED_STALE_AFTER. In waitPollRetry, a select with both poll-retry and staleness ready could take the retry and start another Poll on a socket whose expired window was already evidence — and a blocking call then defers the stale verdict and buys a fresh grace phase, overrunning §23's published bound ("detection window + grace phase... nothing waits longer than their sum"). The retry arm now re-checks staleness before polling; the red proof parks the run goroutine in an overflow handler while both timers fire, and 17/40 rounds issued the second poll. Transition 6 was announced late: newLiveConn opens the socket, arms staleness and starts the pump — the transition's own definition — but the state stayed Connecting until awaitConfirmation, so Observer.Connected ran with AwaitingWelcome's timer set {handshake-deadline, staleness} against an announced state whose set is {handshake-deadline} alone. The announcement now precedes the callback; red proof: Connected observed "connecting". Wait could report quiescence for a run that had started: the single-shot claim was an atomic CompareAndSwap taken BEFORE the mutex section that publishes runDone, and the claim is observable in that window — a second consumption yields the usage terminal — while Wait still read nil and returned. The claim now lives in the same critical section as the publication, so Wait runs wholly before it (no run to await) or wholly after runDone exists. No red proof discriminates this one honestly: the window was two adjacent statements with no seam between them, and a hook would exist for nothing else — the same account the terminal-claim fix gave in an earlier round. The existing single-shot and Wait suites hold the contract on both sides of the change. AGENTS.md's architecture row still called this package foundations-only with the run loop pending; this branch is where that stops being true, so the row now names foundations, the run loop, and the tier-2 driver, with the Layer-1 adapters and the other SDKs still pending. Declined: terminating on a post-confirmation reject_subscription. The state machine draws transition 12 solely from AwaitingConfirmation; §23's "always terminal — first attempt or reconnect" pins which ATTEMPT a rejection lands on, not a state-independent verdict. Action Cable rejects only from the subscribe callback and the connector subscribes exactly once per socket, so a post-confirmation rejection is an unsolicited frame — and honoring it would hand one such frame the connector's most severe verdict, ZERO reconnects, which is precisely what the pre-subscribe gate was added to prevent in an earlier round. The default arm's liveness-only comment says this on purpose. --- AGENTS.md | 2 +- go/pkg/basecamp/eventfeed/catchup.go | 53 +++++- go/pkg/basecamp/eventfeed/catchup_test.go | 153 ++++++++++++++++++ go/pkg/basecamp/eventfeed/connector.go | 51 ++++-- go/pkg/basecamp/eventfeed/continuation.go | 19 ++- .../basecamp/eventfeed/continuation_test.go | 27 ++++ go/pkg/basecamp/eventfeed/loop.go | 15 +- go/pkg/basecamp/eventfeed/loop_test.go | 39 +++++ 8 files changed, 330 insertions(+), 29 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 75be67a661..93dac107aa 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -33,7 +33,7 @@ All `262` operations across the ~50-service per-SDK layer are generated. Hand-wr |---------|----------| | HTTP helpers, pagination, hooks | `typescript/src/services/base.ts`, `ruby/lib/basecamp/generated/services/base_service.rb`, `swift/Sources/Basecamp/Services/BaseService.swift`, `kotlin/sdk/src/commonMain/kotlin/com/basecamp/sdk/services/BaseService.kt`, `python/src/basecamp/generated/services/_base.py`, `rust/basecamp-sdk/src/http/` | | OAuth flows (not in OpenAPI spec) | `typescript/src/services/authorization.ts`, `ruby/lib/basecamp/services/authorization_service.rb`, `kotlin/sdk/src/commonMain/kotlin/com/basecamp/sdk/oauth/`, `python/src/basecamp/services/authorization.py`, `rust/basecamp-sdk/src/oauth/` (no Swift equivalent) | -| Event Feed connector: the long-lived push+poll transport, hand-written behind its TicketMinter/PollSource seams (SPEC.md §23) | `go/pkg/basecamp/eventfeed/` (Go reference, foundations only so far — the run loop, the Layer-1 seam adapters over the generated operations, and the other SDKs are all pending) | +| Event Feed connector: the long-lived push+poll transport, hand-written behind its TicketMinter/PollSource seams (SPEC.md §23) | `go/pkg/basecamp/eventfeed/` (Go reference — foundations, the run loop, and the tier-2 conformance driver; the Layer-1 seam adapters over the generated operations and the other SDKs are still pending) | | Merge-safe Todos composites (update/edit over generated get+replace; SPEC.md §18) | `typescript/src/services/todos-extensions.ts`, `ruby/lib/basecamp/services/todos_extensions.rb`, `swift/Sources/Basecamp/TodosServiceExtensions.swift`, `kotlin/sdk/src/commonMain/kotlin/com/basecamp/sdk/services/TodosService.kt`, `python/src/basecamp/services/todos.py`, `rust/basecamp-sdk/src/services/todos.rs` | | Merge-safe Cards composite (update over generated get+updateVerbatim; SPEC.md §18) | `typescript/src/services/cards-extensions.ts`, `ruby/lib/basecamp/services/cards_extensions.rb`, `swift/Sources/Basecamp/CardsServiceExtensions.swift`, `kotlin/sdk/src/commonMain/kotlin/com/basecamp/sdk/services/CardsService.kt`, `python/src/basecamp/services/cards.py`, `rust/basecamp-sdk/src/services/cards.rs` | | Merge-safe Todolists composites (update/edit over generated get+replace; SPEC.md §18) | `typescript/src/services/todolists-extensions.ts`, `ruby/lib/basecamp/services/todolists_extensions.rb`, `swift/Sources/Basecamp/TodolistsServiceExtensions.swift`, `kotlin/sdk/src/commonMain/kotlin/com/basecamp/sdk/services/TodolistsService.kt`, `python/src/basecamp/services/todolists.py`, `rust/basecamp-sdk/src/services/todolists.rs` | diff --git a/go/pkg/basecamp/eventfeed/catchup.go b/go/pkg/basecamp/eventfeed/catchup.go index 7d8b977237..d8bd5ea8bf 100644 --- a/go/pkg/basecamp/eventfeed/catchup.go +++ b/go/pkg/basecamp/eventfeed/catchup.go @@ -549,12 +549,16 @@ func (l *loop) pollPage(at *attempt, cursor Cursor) pollAttempt { // The staleness firing and the re-arm wake are demoted to WAKE-UPS; the // deadline alone decides, and the post-select check runs on every wake, so an // early or late one costs a loop turn rather than the verdict. Where the wake -// comes from depends on which outcome was deferred. For a frame, evaluate is -// still called on a firing and still re-arms a suspended window — that re-arm -// is what guarantees the next wake when the pump is blocked and no frame can -// deliver one. For a staleness expiry, evaluate has already spoken and the -// firing that carried its verdict is consumed, so graceWake supplies the wake -// instead. +// comes from depends on which outcome was deferred. For a frame or a read +// error, evaluate is still called on a firing: a suspended window is re-armed +// — the next wake when the pump is blocked and no frame can deliver one — and +// an authoritative expiry is latched by graceWake exactly as the stale +// caller's is, because the deferral slot already holds the socket's verdict +// and the pre-existing window (which a read error does not reset) may be +// nearly spent when that verdict lands. For a staleness expiry, evaluate has +// already spoken and the firing that carried its verdict is consumed, so +// graceWake supplies the wake instead. In every case the wake survives and +// the verdict stays with the deadline. // // A dedicated timer would express this more directly, and is deliberately not // used: SPEC.md §23 pins exactly six timer kinds AND every state's exact @@ -605,7 +609,18 @@ func (l *loop) awaitSupersededPoll(at *attempt, done <-chan pollResult, deadline // timer landing exactly on it. at.lc.stale.graceWake() } else if _, ok := at.lc.stale.evaluate(staleGen); ok { - return pollAttempt{superseded: true} + // An authoritative expiry during another outcome's grace + // phase is ALSO only a wake. The deferral slot already holds + // the socket's verdict — a read error or a disconnect frame, + // which does not reset staleness — so the pre-existing + // window, mostly spent by the time that outcome landed, can + // fire here almost immediately, and abandoning the call on it + // granted the poll whatever sliver of the old window remained + // instead of the phase §23 measures from the deferral. The + // expiry adds nothing the deferred outcome's teardown will + // not do; graceWake latches it and keeps the next wake in + // hand, and the deadline below is what decides. + at.lc.stale.graceWake() } } if l.hooks.supersededWake != nil { @@ -1310,6 +1325,30 @@ func (l *loop) waitPollRetry(at *attempt, d time.Duration) (cycleOutcome, bool) l.disposeAttempt(at, t) return cycleOutcome{kind: outcomeClosed}, true case <-t.C(): + // Both timers can be ready — the select is random — and taking + // the retry over an expired window starts another Poll on a + // socket already known dead: if that call blocks, the stale + // verdict is deferred and granted a fresh grace phase, so the + // teardown lands beyond §23's published bound (detection window + // + grace phase, once — "nothing waits longer than their sum"). + // So the retry arm re-checks staleness before polling; an + // expired window takes transition 21 now, exactly as it would + // have had its case won the select. The retry timer has fired + // and is no longer outstanding, so disposal is handed nil. + select { + case <-staleTimer.C(): + if age, ok := at.lc.stale.evaluate(staleGen); ok { + l.disposeAttempt(at, nil) + if l.cfg.observer.StaleConnection != nil { + l.cfg.observer.StaleConnection(age) + } + l.observeDisconnected("", errStaleConnection) + return cycleOutcome{kind: outcomeFailed}, true + } + // Superseded or suspended: not evidence, and evaluate has + // re-armed the window — proceed with the retry. + default: + } return cycleOutcome{}, false case <-at.lc.stale.rearmed(): continue diff --git a/go/pkg/basecamp/eventfeed/catchup_test.go b/go/pkg/basecamp/eventfeed/catchup_test.go index 2979a66317..85266baf80 100644 --- a/go/pkg/basecamp/eventfeed/catchup_test.go +++ b/go/pkg/basecamp/eventfeed/catchup_test.go @@ -2585,3 +2585,156 @@ func TestSuspendedStalenessDuringAnInFlightPollDoesNotDefer(t *testing.T) { t.Fatalf("StaleConnection ages = %v, want none — the firing was not evidence", ages) } } + +// TestDeferredReadFailureGetsTheFullGracePhase: SPEC.md §23's grace phase is +// "one further EVENT_FEED_STALE_AFTER, measured from the deferral, after +// which the seam call is abandoned to the deferred outcome's teardown" — a +// deadline, with every staleness firing inside it demoted to a wake ("wakes +// may be early or late, and the deadline is what decides"). A read error does +// not reset staleness in the pump, so the PRE-EXISTING window — armed at the +// last inbound frame, mostly spent by the time the failure lands — fires +// inside the grace phase of the deferred read error. That firing must wake +// the wait, not abandon the in-flight poll: abandoned there, a failure +// detected 7s after the last frame grants the poll 0.5s of the 7.5s phase, +// and a page returning during the remainder is cancelled instead of being +// delivered and checkpointed. +func TestDeferredReadFailureGetsTheFullGracePhase(t *testing.T) { + wakes := make(chan struct{}, 64) + deferred := make(chan struct{}, 1) + disconnected := make(chan error, 8) + h := newHarness(t, eventfeed.WithObserver(eventfeed.Observer{ + Disconnected: func(_ string, err error) { disconnected <- err }, + })) + h.conn.OnSupersededWake(func() { + select { + case wakes <- struct{}{}: + default: + } + }) + h.conn.OnFrameDeferred(func() { + select { + case deferred <- struct{}{}: + default: + } + }) + h.minter.ScriptTicket(ticket(1)) + h.polls.StallNext() + h.start() + + conn := h.driveToSubscribed() + conn.Serve(frameConfirm(noFilterIdentifier)) + h.waitUntil("entry poll in flight", func() bool { return h.polls.CallCount() == 1 }) + conn.FailReads(errors.New("read failed mid-poll")) + select { + case <-deferred: + case <-time.After(watchdog): + t.Fatal("the read failure was not deferred") + } + + // The pre-existing window fires INSIDE the grace phase. + h.fireTimer(timerStaleness) + select { + case <-wakes: + // Demoted to a wake; the deadline decides. SPEC behavior. + case err := <-disconnected: + t.Fatalf("the deferred read failure was abandoned at the OLD staleness firing (Disconnected: %v); the grace phase is measured from the deferral", err) + case <-time.After(watchdog): + t.Fatal("neither a wake nor a teardown followed the staleness firing") + } + + // And the deadline still ends the wait: one full EVENT_FEED_STALE_AFTER + // after the deferral (7500ms, SPEC-pinned), the call is abandoned to the + // read error's teardown. + h.clock.Advance(7500 * time.Millisecond) + select { + case <-disconnected: + case <-time.After(watchdog): + t.Fatal("the grace deadline did not end the wait") + } + h.awaitTimer(timerBackoff) +} + +// TestExpiredStalenessOutranksThePollRetry: when `poll-retry` and `staleness` +// are both ready at waitPollRetry's select, taking the retry starts another +// Poll on a socket whose expired window was already evidence — and if that +// call blocks, the stale verdict is deferred and granted a fresh grace phase, +// so the teardown lands beyond §23's published bound ("detection window + +// grace phase... Nothing waits longer than their sum"). The expired window +// must win however the select lands: the retry arm re-checks staleness before +// issuing the poll. The select pick is random, so the race is driven in +// rounds: the run goroutine is parked in a BufferOverflow handler while both +// timers fire, and any round that issues a second poll is a violation. +func TestExpiredStalenessOutranksThePollRetry(t *testing.T) { + const rounds = 40 + violations := 0 + for i := range rounds { + violations += func() int { + parked := make(chan struct{}, 1) + release := make(chan struct{}) + var once sync.Once + rel := func() { once.Do(func() { close(release) }) } + t.Cleanup(rel) + disconnected := make(chan struct{}, 8) + h := newHarness(t, + eventfeed.WithLiveBufferCapacity(1), + eventfeed.WithSignalHandler(func(eventfeed.Signal) eventfeed.Disposition { + select { + case parked <- struct{}{}: + default: + } + <-release + return eventfeed.Accept + }), + eventfeed.WithObserver(eventfeed.Observer{ + Disconnected: func(string, error) { + select { + case disconnected <- struct{}{}: + default: + } + }, + }), + ) + h.minter.ScriptTicket(ticket(1)) + h.polls.ScriptError(&eventfeed.PollError{Kind: eventfeed.PollTransient, Msg: "transient"}) + h.start() + + conn := h.driveToSubscribed() + conn.Serve(frameConfirm(noFilterIdentifier)) + h.awaitTimer(timerPollRetry) + // Two correlated events against capacity 1: the second drops, and + // its overflow handler parks the run goroutine mid-dispatch — so + // both timers below fire while nothing is selecting, and the next + // select finds both ready. + conn.Serve(frameMessage(noFilterIdentifier, 101)) + conn.Serve(frameMessage(noFilterIdentifier, 102)) + select { + case <-parked: + case <-time.After(watchdog): + t.Fatal("the overflow handler never parked the run goroutine") + } + h.fireTimer(timerStaleness) + h.fireTimer(timerPollRetry) + rel() + + deadline := time.Now().Add(watchdog) + for { + if h.polls.CallCount() >= 2 { + t.Logf("round %d: poll-retry won over an expired staleness window (second poll issued)", i) + return 1 + } + select { + case <-disconnected: + return 0 + default: + } + if time.Now().After(deadline) { + t.Fatal("round undecided: neither a second poll nor a teardown") + } + time.Sleep(time.Millisecond) + } + }() + } + if violations != 0 { + t.Errorf("%d/%d rounds issued a poll on an already-expired socket; the retry must re-check staleness (transition 21) before polling", violations, rounds) + } +} diff --git a/go/pkg/basecamp/eventfeed/connector.go b/go/pkg/basecamp/eventfeed/connector.go index e75ec8e84b..a6493dd9c3 100644 --- a/go/pkg/basecamp/eventfeed/connector.go +++ b/go/pkg/basecamp/eventfeed/connector.go @@ -7,7 +7,6 @@ import ( "math/rand/v2" "net/url" "sync" - "sync/atomic" "time" ) @@ -184,9 +183,8 @@ func WithObserver(o Observer) Option { return func(c *config) { c.observer = o } // Connector is the SPEC.md §23 Event Feed connector. Construct with New, // consume with Events (single-shot), stop with Close. type Connector struct { - cfg config - consumed atomic.Bool - hooks testHooks + cfg config + hooks testHooks // mu guards the close latch and the active run's cancellation, which have // to move together: close() is a universal edge from every non-absorbing @@ -195,6 +193,12 @@ type Connector struct { // never across a seam call, a yield, or an observer callback — so a Close // taken from inside any of them cannot contend with the run goroutine. mu sync.Mutex + // consumed latches the single-shot claim (Events). Guarded by mu, in the + // SAME critical section that publishes runDone: claimed-but-unpublished + // was a window in which a concurrent Wait — synchronized with the claim + // through the usage terminal a second consumption yields — read a nil + // runDone and returned while the run it should await was starting. + consumed bool // isClosed latches Close, so a run that starts afterwards cancels itself // immediately rather than making one wire attempt first. isClosed bool @@ -426,30 +430,41 @@ func checkOriginScheme(canonical string) error { // error element — a clean stop, the feed is resumable by design. func (c *Connector) Events(ctx context.Context) iter.Seq2[Event, error] { return func(yield func(Event, error) bool) { - if !c.consumed.CompareAndSwap(false, true) { - yield(Event{}, &TerminalError{ - Reason: ReasonUsage, - Msg: "Events is single-shot: this connector was already consumed", - }) - return - } runCtx, cancel := context.WithCancel(ctx) - defer cancel() - // Published under the same lock as the cancel func, and CLOSED rather - // than cleared on exit: clearing it would leave a window where Wait - // reads nil — and returns at once — while this function has not - // finished unwinding, which is the one instant Wait exists to cover. done := make(chan struct{}) - defer close(done) + // The single-shot claim is taken under c.mu, in the SAME critical + // section that publishes runDone — not before it. A claim taken first + // (the old atomic CompareAndSwap) left a window between winning and + // publishing in which the claim was already OBSERVABLE — a second + // consumption yields the usage terminal — while Wait still read a nil + // runDone and returned at once, reporting quiescence for a run that + // had started and not exited. One section leaves Wait nothing to + // linearize between: it runs wholly before the claim (no run to + // await) or wholly after runDone exists. + // // Register this run's cancellation under the same lock Close latches // under, so the two cannot straddle each other: a Close that already // latched cancels here, and one that latches later finds the func and // cancels it itself. Either way runCtx is done before Close returns — // there is no window in which the run proceeds past a returned Close. c.mu.Lock() + if c.consumed { + c.mu.Unlock() + cancel() + yield(Event{}, &TerminalError{ + Reason: ReasonUsage, + Msg: "Events is single-shot: this connector was already consumed", + }) + return + } + c.consumed = true // Installed before the loop is built, and under the same mutex Close // takes, so the run cannot reach a terminal before the claim exists. c.cfg.claimTerminal = c.claimTerminal + // runDone is CLOSED rather than cleared on exit: clearing it would + // leave a window where Wait reads nil — and returns at once — while + // this function has not finished unwinding, which is the one instant + // Wait exists to cover. c.runDone = done if c.isClosed { // Closed before the first iteration: end cleanly with zero wire @@ -459,6 +474,8 @@ func (c *Connector) Events(ctx context.Context) iter.Seq2[Event, error] { c.cancelRun = cancel } c.mu.Unlock() + defer cancel() + defer close(done) // Fired only once the cancellation is REGISTERED: before that point // the run is protected by the isClosed latch checked just above, not // by the cancel func, so a context handed out earlier would invite an diff --git a/go/pkg/basecamp/eventfeed/continuation.go b/go/pkg/basecamp/eventfeed/continuation.go index e266432191..021c997bfc 100644 --- a/go/pkg/basecamp/eventfeed/continuation.go +++ b/go/pkg/basecamp/eventfeed/continuation.go @@ -1,6 +1,9 @@ package eventfeed -import "strings" +import ( + "strings" + "unicode/utf8" +) // Continuation and resume URL validation (SPEC.md §23 "Continuation and // Resume URL Validation"). The two absolute URLs the poll lane follows — the @@ -28,6 +31,20 @@ import "strings" // pure function of the two URLs: nothing here needs a run, and keeping it // free-standing is what lets this file be reviewed and tested on its own. func checkContinuation(baseOrigin, pageURL string) *TerminalError { + // Validated RAW, before canonicalization, mirroring validateConfig's + // raw-origin check: CanonicalOrigin lowercases through strings.ToLower, + // which rewrites every invalid byte to U+FFFD, so a continuation host + // carrying a raw invalid byte would collapse to the same canonical form + // as a configured origin that legitimately contains the replacement + // character — and same-origin validation must never equate distinct byte + // strings. A conformant server's URLs are valid UTF-8, so refusing the + // bytes outright is fail-closed. + if !utf8.ValidString(pageURL) { + return &TerminalError{ + Reason: ReasonInvalidContinuation, + Msg: "the continuation URL is not valid UTF-8", + } + } origin, err := CanonicalOrigin(pageURL) if err != nil { return &TerminalError{ diff --git a/go/pkg/basecamp/eventfeed/continuation_test.go b/go/pkg/basecamp/eventfeed/continuation_test.go index 3bcd61da50..6a5141cb0b 100644 --- a/go/pkg/basecamp/eventfeed/continuation_test.go +++ b/go/pkg/basecamp/eventfeed/continuation_test.go @@ -132,6 +132,33 @@ func TestCheckContinuation_VerdictFollowsTheConfiguredBase(t *testing.T) { } } +// TestCheckContinuation_RejectsInvalidUTF8BeforeCanonicalizing: same-origin +// validation must never equate distinct byte strings. CanonicalOrigin +// lowercases through strings.ToLower, which rewrites every invalid byte to +// U+FFFD — so a continuation host carrying a raw invalid byte collapses to +// the same canonical form as a configured origin that legitimately contains +// the replacement character (valid UTF-8, so construction accepts it), and +// the authenticated poll would follow a URL the server never named. The raw +// bytes are refused before the lossy step, mirroring validateConfig's +// raw-origin check; a conformant server's URLs are valid UTF-8, so refusal +// is fail-closed. +func TestCheckContinuation_RejectsInvalidUTF8BeforeCanonicalizing(t *testing.T) { + base, err := CanonicalOrigin("https://�.example.com") + if err != nil { + t.Fatalf("CanonicalOrigin: %v", err) + } + terr := checkContinuation(base, "https://\xff.example.com/5951425/events.json?page=2") + if terr == nil { + t.Fatal("an invalid-UTF-8 continuation host collapsed to the configured origin's U+FFFD and was accepted") + } + if terr.Reason != ReasonInvalidContinuation { + t.Fatalf("reason = %q, want %q", terr.Reason, ReasonInvalidContinuation) + } + if !strings.Contains(terr.Msg, "UTF-8") { + t.Fatalf("message %q does not name the UTF-8 refusal", terr.Msg) + } +} + // TestCanonicalOrigin_PortSpellingsCollapse pins that the canonical form is // a function of the ORIGIN, not the port's spelling — the identity-split // class checkIdentityText exists to refuse: two spellings of one origin diff --git a/go/pkg/basecamp/eventfeed/loop.go b/go/pkg/basecamp/eventfeed/loop.go index 73dd1ecd91..b61daad725 100644 --- a/go/pkg/basecamp/eventfeed/loop.go +++ b/go/pkg/basecamp/eventfeed/loop.go @@ -922,6 +922,14 @@ func (l *loop) runCycle(delay time.Duration) cycleOutcome { // so nothing is observed out of order — Connected is observability, not a // gate. at.lc = newLiveConn(at.ctx, conn, l.cfg.clock, l.cfg.staleAfter, l.hooks) + // The state is announced BEFORE Observer.Connected, for the same reason + // the arming is: transition 6 is "dial ok; frame pump started", and both + // are true the moment newLiveConn returns. Announced after, the callback + // runs with staleness armed — AwaitingWelcome's exact timer set — while + // the last announced state is still Connecting, whose set is + // {handshake-deadline} alone, and host telemetry sampling both sees a + // pair §23's per-state invariants say cannot exist. + l.setState(stateAwaitingWelcome) if l.cfg.observer.Connected != nil { l.cfg.observer.Connected() } @@ -969,10 +977,11 @@ func (l *loop) classifyMintFailure(err error) cycleOutcome { // awaitConfirmation drives AwaitingWelcome and AwaitingConfirmation // (transitions 8–15 plus the state-generic protocol-fatal and invalid-frame -// dispatch). deadline is the running handshake-deadline on entry; `welcome` -// re-arms it as the confirmation-deadline (transition 8). +// dispatch). The caller has already taken transition 6 — the state is +// AwaitingWelcome on entry, announced before Observer.Connected fired. +// deadline is the running handshake-deadline on entry; `welcome` re-arms it +// as the confirmation-deadline (transition 8). func (l *loop) awaitConfirmation(at *attempt, deadline Timer) cycleOutcome { - l.setState(stateAwaitingWelcome) for { staleTimer, staleGen := at.lc.stale.current() select { diff --git a/go/pkg/basecamp/eventfeed/loop_test.go b/go/pkg/basecamp/eventfeed/loop_test.go index 5615b07547..803a8c86bf 100644 --- a/go/pkg/basecamp/eventfeed/loop_test.go +++ b/go/pkg/basecamp/eventfeed/loop_test.go @@ -1244,6 +1244,45 @@ func TestDuplicateWelcomeResendsSubscribe(t *testing.T) { h.awaitBoundary() } +// TestConnectedObserverRunsInAwaitingWelcome: transition 6 (Connecting → +// AwaitingWelcome) is "dial ok; frame pump started" (SPEC.md §23), and both +// are true the moment newLiveConn returns — so the state announcement must +// precede Observer.Connected. Announced late, a Connected callback observes +// the {handshake-deadline, staleness} timer set that §23's per-state exact +// sets assign to AwaitingWelcome while the last announced state is still +// Connecting, whose set is {handshake-deadline} — an inconsistent pair no +// host telemetry can reconcile. +func TestConnectedObserverRunsInAwaitingWelcome(t *testing.T) { + var mu sync.Mutex + last := "" + atConnected := make(chan string, 1) + h := newHarness(t, eventfeed.WithObserver(eventfeed.Observer{ + Connected: func() { + mu.Lock() + s := last + mu.Unlock() + select { + case atConnected <- s: + default: + } + }, + })) + // Overrides the harness's state hook; both callbacks run on the run + // goroutine, so `last` is ordered, not sampled. + h.conn.OnStateChanged(func(s string) { mu.Lock(); last = s; mu.Unlock() }) + h.minter.ScriptTicket(ticket(1)) + h.start() + + select { + case s := <-atConnected: + if s != "awaiting_welcome" { + t.Fatalf("Connected fired with announced state %q, want %q: transition 6 must precede the callback", s, "awaiting_welcome") + } + case <-time.After(watchdog): + t.Fatal("Connected never fired") + } +} + // TestSubscribeWriteFailure: a failed subscribe write takes the socket- // failure path — Backoff, never terminal. func TestSubscribeWriteFailure(t *testing.T) { From 50aa3465955b761c962f6e6b289b51703dc83786 Mon Sep 17 00:00:00 2001 From: Jeremy Daer Date: Sat, 22 Aug 2026 02:38:29 -0700 Subject: [PATCH 25/51] Event feed: verdicts that lost coin flips, and a driver blind to eras MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit All five of this round's findings taken — four expired-verdict races in the run loop's family, and a conformance-driver fidelity hole. The dial result could outrun the handshake deadline: both ready, the select random, and accepting the dial installed the pump, armed staleness, and announced Connected for an attempt whose window had already closed — transition 7 bypassed by a coin flip. The success path now drains the deadline before transition 6, as a drain rather than a Stop-probe because an unfired deadline must keep running to welcome. The red proof fires the deadline from inside Dial, sequencing it strictly before every dial result, so any Connected at all is a violation: 1/100 rounds pre-fix, 0 after. The subscribe write's bounded wait selected only {written, close, phase deadline}, on the recorded premise that the phase deadline is always the tighter bound. It is not, twice: staleness (7.5s) undercuts even the default 10s handshake deadline on an immediate first welcome, and a duplicate welcome resends under the confirmation deadline, which is configurable to anything — a dead socket sat blocked in the write arbitrarily past the expiry rows 9/15 say tears it down. The wait now carries the staleness case like every other socket-open wait; red proof: a stalled write plus a fired window produced no StaleConnection within the watchdog. Streaming could deliver past a latched expiry: a frame received AFTER the window fired does not reset it — staleHolder.arm latches, which is §23's authoritative-firing rule working — but the fired timer and that frame were then both ready, and the frame arm delivered first. The arm now re-checks the verdict before delivering, with evaluate as the arbiter: a frame the pump received first moved the generation and delivers as ever, a latched expiry takes transition 25 with the frame discarded to the reconnect walk. Red proof, rounds-driven off a collector parked mid-delivery: 17/40 rounds delivered event 102 past the latched expiry. A throttled Retry-After of zero was read as absence. SPEC §6 fixes the adapter mapping — a retryable outcome whose last response carried a parsed Retry-After maps to throttled(retry_after) whatever its status, one without maps to transient — so PollThrottled always carries a server-directed wait, zero included, waited exactly and cap-exempt. pollRetryDelay is now kind-keyed; a parsed zero stays zero instead of drawing up to 60s of local jitter (the red run drew 315ms), and a negative value clamps rather than arming a negative timer. The tier-2 driver's arrival-strict lookahead judged expectBuffered against GLOBAL occupancy history, so occupancy reached and left in a long-departed era satisfied a pending expectBuffered and let the scan read through to a following expectCheckpoint — an out-of-order save accepted by the very instrument that exists to reject it. History is now scoped to an era boundary captured whenever the step pointer moves (enterStep and the await advance), and expectBuffered's own rendezvous scans from the same boundary, which also closes the reached-and-left-before-runStep flake its run-time capture had. Red proof: the scan judged an arriving save against step 3 (expectCheckpoint) on stale history. Every committed fixture still passes under the scoped scan. --- go/pkg/basecamp/eventfeed/backoff.go | 17 +++- go/pkg/basecamp/eventfeed/backoff_test.go | 8 +- go/pkg/basecamp/eventfeed/catchup.go | 26 ++++- go/pkg/basecamp/eventfeed/catchup_test.go | 75 ++++++++++++++ go/pkg/basecamp/eventfeed/loop.go | 86 +++++++++++----- go/pkg/basecamp/eventfeed/loop_test.go | 97 +++++++++++++++++++ .../eventfeed/scenario_conformance_test.go | 12 ++- .../eventfeed/scenario_harness_test.go | 19 +++- .../eventfeed/scenario_selftest_test.go | 28 ++++++ 9 files changed, 331 insertions(+), 37 deletions(-) diff --git a/go/pkg/basecamp/eventfeed/backoff.go b/go/pkg/basecamp/eventfeed/backoff.go index a6a3bf001f..650891e850 100644 --- a/go/pkg/basecamp/eventfeed/backoff.go +++ b/go/pkg/basecamp/eventfeed/backoff.go @@ -74,9 +74,20 @@ func reconnectDelay(retryAfter time.Duration, n int, r func() float64) time.Dura // reconnect-cycle count, reset by any successful poll page and by socket // teardown. A server-directed Retry-After is waited exactly and is exempt // from local caps; otherwise the wait is a full-jitter draw over k. -func pollRetryDelay(retryAfter time.Duration, k int, r func() float64) time.Duration { - if retryAfter > 0 { - return retryAfter +// The selection is kind-keyed, per SPEC §6's fixed adapter mapping: "a +// retryable outcome exhausted inside the seam whose last response carried a +// parsed Retry-After maps to throttled(retry_after) whatever its status, and +// one without maps to transient". PollThrottled therefore ALWAYS carries a +// server-directed wait, zero included — Retry-After: 0, or an HTTP-date +// already reached — and §23 waits it exactly, cap-exempt. Gating on +// retryAfter > 0 alone read a parsed zero as absence and replaced an +// immediate-retry directive with up to 60 seconds of local jitter. A +// transient carrying a positive RetryAfter is still honored defensively; a +// negative value (an adapter bug) clamps to zero rather than arming a +// negative timer. +func pollRetryDelay(kind PollErrorKind, retryAfter time.Duration, k int, r func() float64) time.Duration { + if kind == PollThrottled || retryAfter > 0 { + return max(retryAfter, 0) } return fullJitterDelay(k, r) } diff --git a/go/pkg/basecamp/eventfeed/backoff_test.go b/go/pkg/basecamp/eventfeed/backoff_test.go index 5ecd73a084..78864789e5 100644 --- a/go/pkg/basecamp/eventfeed/backoff_test.go +++ b/go/pkg/basecamp/eventfeed/backoff_test.go @@ -102,23 +102,23 @@ func TestReconnectDelay_RetryAfterWinsOutrightOverTheCap(t *testing.T) { func TestPollRetryDelay_RetryAfterIsExactAndCapExempt(t *testing.T) { // A server-directed Retry-After is waited exactly — even beyond the cap. - if got := pollRetryDelay(5*time.Minute, 1, fixedRand(0.5)); got != 5*time.Minute { + if got := pollRetryDelay(PollThrottled, 5*time.Minute, 1, fixedRand(0.5)); got != 5*time.Minute { t.Errorf("pollRetryDelay(5m Retry-After) = %v, want 5m", got) } // It is exact, not a floor: a Retry-After below the would-be draw wins. - if got := pollRetryDelay(time.Second, 7, fixedRand(0.99)); got != time.Second { + if got := pollRetryDelay(PollThrottled, time.Second, 7, fixedRand(0.99)); got != time.Second { t.Errorf("pollRetryDelay(1s Retry-After) = %v, want exactly 1s", got) } } func TestPollRetryDelay_FullJitterOverTheFailureIndex(t *testing.T) { // Without Retry-After the wait is a full-jitter draw over k. - if got := pollRetryDelay(0, 2, fixedRand(0.5)); got != time.Second { + if got := pollRetryDelay(PollTransient, 0, 2, fixedRand(0.5)); got != time.Second { t.Errorf("pollRetryDelay(k=2, r=0.5) = %v, want 1s", got) } rng := rand.New(rand.NewSource(2)) for i := 0; i < 1000; i++ { - d := pollRetryDelay(0, 9, rng.Float64) + d := pollRetryDelay(PollTransient, 0, 9, rng.Float64) if d < 0 || d >= backoffCap { t.Fatalf("pollRetryDelay(k=9) = %v, want in [0, %v)", d, backoffCap) } diff --git a/go/pkg/basecamp/eventfeed/catchup.go b/go/pkg/basecamp/eventfeed/catchup.go index d8bd5ea8bf..46c964e6c5 100644 --- a/go/pkg/basecamp/eventfeed/catchup.go +++ b/go/pkg/basecamp/eventfeed/catchup.go @@ -1109,6 +1109,30 @@ func (l *loop) stream(at *attempt) cycleOutcome { if !ok { return l.pumpExited(at, repair) } + // A latched expiry outranks a ready frame. The pump refuses the + // reset for a frame received after the window fired + // (staleHolder.arm latches instead), so the fired timer and that + // frame can both be ready here — and delivering first hands the + // consumer live events from a socket whose staleness verdict is + // already in, where §23 pins the fired deadline as authoritative + // over frames that did not reset it. evaluate is the arbiter: a + // frame the pump received FIRST moved the generation, so its + // firing is not evidence and the frame delivers as ever. + select { + case <-staleTimer.C(): + if age, authoritative := at.lc.stale.evaluate(staleGen); authoritative { + // Transition 25 first; the frame is discarded with the + // socket — the reconnect walk repairs anything the + // stream carried. + l.disposeAttempt(at, repair) + if l.cfg.observer.StaleConnection != nil { + l.cfg.observer.StaleConnection(age) + } + l.observeDisconnected("", errStaleConnection) + return cycleOutcome{kind: outcomeFailed} + } + default: + } if out, done := l.handleLiveFrame(at, repair, item, true); done { return out } @@ -1251,7 +1275,7 @@ func (l *loop) recoverPoll(at *attempt, cursor Cursor, err error) (walkStep, cyc // The self-loop inside CatchingUp: a wait, not a state change. The // same cursor is re-polled. l.pollFailures++ - out, done := l.waitPollRetry(at, pollRetryDelay(pe.RetryAfter, l.pollFailures, l.cfg.rand)) + out, done := l.waitPollRetry(at, pollRetryDelay(pe.Kind, pe.RetryAfter, l.pollFailures, l.cfg.rand)) return walkStep{cursor: cursor}, out, done case PollUnauthorized: // Recovery rides the reconnect cycle — the fresh mint/token pass — diff --git a/go/pkg/basecamp/eventfeed/catchup_test.go b/go/pkg/basecamp/eventfeed/catchup_test.go index 85266baf80..26c134498a 100644 --- a/go/pkg/basecamp/eventfeed/catchup_test.go +++ b/go/pkg/basecamp/eventfeed/catchup_test.go @@ -2738,3 +2738,78 @@ func TestExpiredStalenessOutranksThePollRetry(t *testing.T) { t.Errorf("%d/%d rounds issued a poll on an already-expired socket; the retry must re-check staleness (transition 21) before polling", violations, rounds) } } + +// TestThrottledZeroRetryAfterIsWaitedExactly: SPEC §6 fixes the seam mapping +// — "a retryable outcome exhausted inside the seam whose last response +// carried a parsed Retry-After maps to throttled(retry_after) whatever its +// status, and one without maps to transient" — so PollThrottled always +// carries a server-directed wait, zero included (Retry-After: 0, or an +// HTTP-date already reached), and §23 waits it "exactly, cap-exempt". A +// delay selection gated on retryAfter > 0 reads a parsed zero as absence +// and replaces an immediate-retry directive with up to 60 seconds of local +// full-jitter backoff. +func TestThrottledZeroRetryAfterIsWaitedExactly(t *testing.T) { + h := newHarness(t) + h.minter.ScriptTicket(ticket(1)) + h.polls.ScriptError(&eventfeed.PollError{Kind: eventfeed.PollThrottled, RetryAfter: 0, Msg: "throttled"}) + h.start() + + conn := h.driveToSubscribed() + conn.Serve(frameConfirm(noFilterIdentifier)) + if d := h.fireTimer(timerPollRetry); d != 0 { + t.Fatalf("poll-retry armed for %s after a parsed Retry-After of 0; a throttled zero is server-directed and waited exactly (0 = retry now), never replaced with local jitter", d) + } +} + +// TestLatchedStalenessOutranksAReadyLiveFrame: a frame received AFTER the +// staleness window fired does not reset it — staleHolder.arm latches the +// expiry instead, per §23's "a fired staleness deadline observed on return +// from a slow delivery is authoritative, and frames still queued at that +// moment were received before the firing and already reset the timer then". +// Both the fired timer and that frame can then be ready at Streaming's +// select, and delivering first hands the consumer live events from a socket +// whose verdict is already in. Rounds-driven (the select pick is random): +// the collector parks mid-delivery, the window fires, a second event +// arrives past the firing, and any round that delivers it is a violation. +func TestLatchedStalenessOutranksAReadyLiveFrame(t *testing.T) { + const rounds = 40 + violations := 0 + for i := range rounds { + func() { + h := newHarness(t) + h.pauseAfter = 1 + h.minter.ScriptTicket(ticket(1)) + h.polls.ScriptPage(eventfeed.PollPage{Position: "pos-1"}) + h.start() + + conn := h.driveToSubscribed() + conn.Serve(frameConfirm(noFilterIdentifier)) + h.awaitStreaming() + conn.Serve(frameMessage(noFilterIdentifier, 101)) + // The loop body IS the consumer, so the parked collector parks + // the state machine mid-delivery, between select turns. + h.waitUntil("the collector to park mid-delivery", func() bool { + return len(h.deliveredIDs()) == 1 + }) + h.fireTimer(timerStaleness) + // Received after the firing: the pump's reset is refused and the + // expiry latches; the frame is handed off regardless. + conn.Serve(frameMessage(noFilterIdentifier, 102)) + h.waitUntil("frame 102 to reach the hand-off", func() bool { + return conn.Pending() == 0 + }) + h.resume() + + h.waitUntil("the teardown to settle in Backoff", func() bool { + return slices.Contains(h.clock.Outstanding(), timerBackoff) + }) + if slices.Contains(h.deliveredIDs(), 102) { + violations++ + t.Logf("round %d: event 102 was delivered past a latched staleness expiry", i) + } + }() + } + if violations != 0 { + t.Errorf("%d/%d rounds delivered a live event past a latched staleness expiry; the frame arm must re-evaluate the verdict before delivering (transition 25 first)", violations, rounds) + } +} diff --git a/go/pkg/basecamp/eventfeed/loop.go b/go/pkg/basecamp/eventfeed/loop.go index b61daad725..edac106612 100644 --- a/go/pkg/basecamp/eventfeed/loop.go +++ b/go/pkg/basecamp/eventfeed/loop.go @@ -903,6 +903,23 @@ func (l *loop) runCycle(delay time.Duration) cycleOutcome { return cycleOutcome{kind: outcomeClosed} } + // The dial's result can race the deadline: both ready, the select above + // is random, and accepting the dial then installed the pump and announced + // Connected for an attempt whose handshake window had already closed — + // transition 7 bypassed by a coin flip. The deadline is authoritative, + // the same expired-timer ordering the welcome and confirm branches read + // Stop for — expressed here as a drain, because a deadline that has NOT + // fired must keep running to `welcome` (it spans dial-to-welcome), so + // Stop cannot be the probe. + select { + case <-hs.C(): + // Transition 7, exactly as if the deadline's own case had won. + at.cancel() + _ = conn.Close(closeCodeNormal, "") + return cycleOutcome{kind: outcomeFailed} + default: + } + // Transition 6 → AwaitingWelcome: staleness arms at socket open, the // pump starts, and the handshake deadline keeps running to `welcome`. // @@ -1045,34 +1062,59 @@ func (l *loop) awaitConfirmation(at *attempt, deadline Timer) cycleOutcome { // it restores the bound: a lapse takes transition 9/14 exactly as it would // have had the write never blocked, and disposal's cancel is what returns the // abandoned write (the seam contract requires a cancelled write to return -// promptly, and Close is documented to unblock it too). Staleness is -// deliberately not in this select — the phase deadline is the tighter bound -// on a handshake that is going nowhere, and adding a second timer here would -// duplicate the generation dance for no additional guarantee. +// promptly, and Close is documented to unblock it too). +// +// Staleness is in this select too. An earlier revision left it out on the +// stated premise that the phase deadline is always the tighter bound, and the +// premise is false twice over: the 7.5s staleness window undercuts even the +// default 10s handshake deadline on an immediate first welcome, and a +// duplicate welcome resends the subscribe under the confirmation deadline, +// which is configurable to anything — a dead socket then sat blocked in the +// write arbitrarily past the expiry that rows 9/15 say tears it down. So this +// wait carries the case like every other socket-open wait, generation dance +// and all. func (l *loop) writeSubscribe(at *attempt, deadline *Timer) (cycleOutcome, bool) { written := make(chan error, 1) go func() { written <- at.lc.conn.WriteFrame(at.ctx, l.subscribeFrame) }() - select { - case werr := <-written: - if werr != nil { + for { + staleTimer, staleGen := at.lc.stale.current() + select { + case werr := <-written: + if werr != nil { + l.disposeAttempt(at, *deadline) + l.observeDisconnected("", werr) + return cycleOutcome{kind: outcomeFailed}, true + } + if l.hooks.subscribeWritten != nil { + l.hooks.subscribeWritten() + } + return cycleOutcome{}, false + case <-l.runCtx.Done(): l.disposeAttempt(at, *deadline) - l.observeDisconnected("", werr) + return cycleOutcome{kind: outcomeClosed}, true + case <-(*deadline).C(): + // The deadline is spent, so disposal is handed nil: the attempt's + // timer set is empty from here, as on every other lapse. + lapsed := errDeadlineLapsed(l.state) + l.disposeAttempt(at, nil) + l.observeDisconnected("", lapsed) + return cycleOutcome{kind: outcomeFailed}, true + case <-at.lc.stale.rearmed(): + case <-staleTimer.C(): + age, ok := at.lc.stale.evaluate(staleGen) + if !ok { + continue + } + // Rows 9/15's staleness trigger, observed mid-write: full + // teardown, and disposal's cancel is what returns the abandoned + // write. + l.disposeAttempt(at, *deadline) + if l.cfg.observer.StaleConnection != nil { + l.cfg.observer.StaleConnection(age) + } + l.observeDisconnected("", errStaleConnection) return cycleOutcome{kind: outcomeFailed}, true } - if l.hooks.subscribeWritten != nil { - l.hooks.subscribeWritten() - } - return cycleOutcome{}, false - case <-l.runCtx.Done(): - l.disposeAttempt(at, *deadline) - return cycleOutcome{kind: outcomeClosed}, true - case <-(*deadline).C(): - // The deadline is spent, so disposal is handed nil: the attempt's - // timer set is empty from here, as on every other lapse. - lapsed := errDeadlineLapsed(l.state) - l.disposeAttempt(at, nil) - l.observeDisconnected("", lapsed) - return cycleOutcome{kind: outcomeFailed}, true } } diff --git a/go/pkg/basecamp/eventfeed/loop_test.go b/go/pkg/basecamp/eventfeed/loop_test.go index 803a8c86bf..24750935e1 100644 --- a/go/pkg/basecamp/eventfeed/loop_test.go +++ b/go/pkg/basecamp/eventfeed/loop_test.go @@ -2303,3 +2303,100 @@ func TestNilDialResultLeavesBackoffsTimerSetExact(t *testing.T) { h2.awaitTimer(timerBackoff) assertTimers(t, h2.clock, map[string]int{timerBackoff: 1}) } + +// deadlineFirstTransport fires the handshake deadline from INSIDE Dial, +// before the dial result exists — so whenever the state machine's select +// finds the dial result ready, the deadline's firing is sequenced strictly +// before it. Connected can then fire only if the dial arm won a both-ready +// select over an already-expired deadline. +type deadlineFirstTransport struct { + inner *feedtest.Transport + fire func() +} + +func (d *deadlineFirstTransport) Dial(ctx context.Context, wsURL string, maxFrameBytes int64) (eventfeed.CableConn, error) { + d.fire() + return d.inner.Dial(ctx, wsURL, maxFrameBytes) +} + +// TestExpiredHandshakeDeadlineOutranksTheDialResult: transition 7 says a +// dial that exceeds the handshake window is cancelled; when the dial result +// and the fired deadline are both ready, the select is random, and taking +// the dial installed the pump, armed staleness, and announced Connected for +// an attempt whose window had already closed. The deadline is authoritative +// — the same expired-timer ordering the welcome and confirm branches read +// Stop for. Driven in rounds: the transport fires the deadline before every +// dial result, so any Connected at all is a violation. +func TestExpiredHandshakeDeadlineOutranksTheDialResult(t *testing.T) { + const rounds = 100 + violations := 0 + for i := range rounds { + connected := make(chan struct{}, 1) + dt := &deadlineFirstTransport{} + h := newHarness(t, + eventfeed.WithTransport(dt), + eventfeed.WithObserver(eventfeed.Observer{ + Connected: func() { + select { + case connected <- struct{}{}: + default: + } + }, + })) + dt.inner = h.tr + dt.fire = func() { h.clock.FireTimer(timerHandshakeDeadline) } + h.minter.ScriptTicket(ticket(1)) + h.start() + + // Either path settles in Backoff: transition 7 directly, or — the + // violation — a Connected attempt whose fired deadline then lapses. + h.waitUntil("the cycle to settle in Backoff", func() bool { + return slices.Contains(h.clock.Outstanding(), timerBackoff) + }) + select { + case <-connected: + violations++ + t.Logf("round %d: Connected fired for an attempt whose handshake deadline had already expired", i) + default: + } + } + if violations != 0 { + t.Errorf("%d/%d rounds announced Connected past an expired handshake deadline; the deadline must be probed before the dial result is accepted", violations, rounds) + } +} + +// TestStalledSubscribeWriteObservesStaleness: rows 9/15 give every +// socket-open state a staleness edge, and the per-state exact timer sets +// keep `staleness` armed through AwaitingWelcome and AwaitingConfirmation — +// but the subscribe write's bounded wait selected only {written, close, +// phase deadline}, on the stated premise that the phase deadline is always +// tighter. It is not: staleness (7.5s) undercuts even the default 10s +// handshake deadline on an immediate welcome, and a duplicate welcome under +// a long WithConfirmationDeadline leaves a dead socket blocked in the write +// arbitrarily past its expiry. The write's wait must observe the staleness +// window like every other socket-open wait. +func TestStalledSubscribeWriteObservesStaleness(t *testing.T) { + stale := make(chan time.Duration, 1) + h := newHarness(t, eventfeed.WithObserver(eventfeed.Observer{ + StaleConnection: func(age time.Duration) { + select { + case stale <- age: + default: + } + }, + })) + h.minter.ScriptTicket(ticket(1)) + h.start() + + conn := h.liveConn() + conn.StallWrites() + conn.Serve(frameWelcome()) + h.awaitFrameHandled("welcome") + h.fireTimer(timerStaleness) + select { + case <-stale: + case <-time.After(watchdog): + t.Fatal("a staleness expiry during the blocked subscribe write was never observed; the dead socket sits in the write until the phase deadline") + } + h.awaitTimer(timerBackoff) +} diff --git a/go/pkg/basecamp/eventfeed/scenario_conformance_test.go b/go/pkg/basecamp/eventfeed/scenario_conformance_test.go index a75bf078ff..bffb477f70 100644 --- a/go/pkg/basecamp/eventfeed/scenario_conformance_test.go +++ b/go/pkg/basecamp/eventfeed/scenario_conformance_test.go @@ -255,7 +255,11 @@ func (d *driver) stepSatisfiedLocked(step scenarioStep) bool { case *expectGapStep: return h.gapsTaken < len(h.gaps) case *expectBufferedStep: - return h.occupancy == payload.Count || slices.Contains(h.occupancyHistory, payload.Count) + // History is scoped to the current era — the boundary captured when + // the step pointer last moved — so occupancy reached and left before + // this era cannot satisfy a pending expectBuffered and let the scan + // read through to an arrival-strict step behind it. + return h.occupancy == payload.Count || slices.Contains(h.occupancyHistory[h.occupancyEra:], payload.Count) case *expectSignalStep: return h.signalsTaken < len(h.signals) case *exactInvocations: @@ -726,8 +730,12 @@ func (d *driver) expectGap(epochAfterID int64) error { // change is recorded, so a value reached and left again while the driver was // scheduled out still satisfies the rendezvous. func (d *driver) expectBuffered(count int) error { + // The scan boundary is the step pointer's era, not this function's start: + // occupancy reached and left between the pointer entering this step and + // the driver getting here is this step's to see (excluding it flaked), + // while anything earlier belongs to a previous era and must not satisfy. d.h.mu.Lock() - from := len(d.h.occupancyHistory) + from := d.h.occupancyEra d.h.mu.Unlock() return d.h.await(fmt.Sprintf("live buffer occupancy %d", count), func() (bool, string) { if d.h.occupancy == count { diff --git a/go/pkg/basecamp/eventfeed/scenario_harness_test.go b/go/pkg/basecamp/eventfeed/scenario_harness_test.go index cc77a320d5..5ae6fc9fd6 100644 --- a/go/pkg/basecamp/eventfeed/scenario_harness_test.go +++ b/go/pkg/basecamp/eventfeed/scenario_harness_test.go @@ -97,11 +97,18 @@ type scenarioHarness struct { saveUsed int // Recorded history. - delivered []int64 - saves []saveRecord - savesTaken int - occupancy int - occupancyHistory []int + delivered []int64 + saves []saveRecord + savesTaken int + occupancy int + occupancyHistory []int + // occupancyEra is the history length when the step pointer last moved: + // the boundary expectBuffered satisfaction scans from. Occupancy reached + // and left in an earlier era must not satisfy a later expectBuffered — + // the arrival-strict lookahead would read through it and accept an + // out-of-order save — while the CURRENT value satisfies regardless of + // when it was reached (the buffer simply is at N). + occupancyEra int signals []eventfeed.Signal signalsTaken int invocations []invocation @@ -248,6 +255,7 @@ func (h *scenarioHarness) enterStep(index int) { defer h.mu.Unlock() if index > h.cursor { h.cursor = index + h.occupancyEra = len(h.occupancyHistory) } } @@ -259,6 +267,7 @@ func (h *scenarioHarness) enterStep(index int) { func (h *scenarioHarness) advanceLocked() { if h.program != nil && h.cursor < len(h.program.sc.Steps) { h.cursor++ + h.occupancyEra = len(h.occupancyHistory) } } diff --git a/go/pkg/basecamp/eventfeed/scenario_selftest_test.go b/go/pkg/basecamp/eventfeed/scenario_selftest_test.go index 9753328aed..55643f4d8c 100644 --- a/go/pkg/basecamp/eventfeed/scenario_selftest_test.go +++ b/go/pkg/basecamp/eventfeed/scenario_selftest_test.go @@ -650,3 +650,31 @@ func TestScenarioDriverEnforcesTheFinalErrorElement(t *testing.T) { } }) } + +// TestScenarioDriverScanScopesOccupancyHistoryToTheCurrentEra: the +// arrival-strict lookahead treats a pending expectBuffered(N) as satisfied +// when occupancy EVER reached N — including in an era the pointer has long +// left. The scan then reads through to a following expectCheckpoint and +// judges an arriving save against it, accepting an out-of-order checkpoint +// the fixture ordered after a buffer state that never recurred. History must +// be scoped to a boundary captured when the step pointer moves. +func TestScenarioDriverScanScopesOccupancyHistoryToTheCurrentEra(t *testing.T) { + h := &scenarioHarness{} + d := &driver{h: h, sc: &scenario{Steps: []scenarioStep{ + {Kind: "expectState", Payload: &expectStateStep{Is: "streaming"}}, + {Kind: "expectBuffered", Payload: &expectBufferedStep{Count: 1}}, + {Kind: "expectCheckpoint", Payload: &expectCheckpointStep{}}, + }}} + // Occupancy reached 1 and left it while the pointer was on step 1; the + // pointer then moves on to step 2 (expectBuffered) with occupancy 0. + h.occupancy = 0 + h.occupancyHistory = []int{1, 0} + d.beginStep(1, d.sc.Steps[1]) + + h.mu.Lock() + kind, where := d.currentStepLocked() + h.mu.Unlock() + if kind != "expectBuffered" { + t.Fatalf("the scan read through expectBuffered(1) on occupancy from a previous era: an arriving save would be judged against %s and accepted out of order", where) + } +} From c86a042d4fc6b44f4247f462a8e05298c1ebb5ae Mon Sep 17 00:00:00 2001 From: Jeremy Daer Date: Sat, 22 Aug 2026 03:00:24 -0700 Subject: [PATCH 26/51] Event feed: the parser cannot say zero, and my throttled fix forgot to ask it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Four follow-ups taken, one of them a correction to this branch's own previous round — stated plainly: the round-4 throttled-zero change overshot, and Copilot caught it. The two §6 texts compose; they do not conflict. The seam mapping is presence-keyed — throttled iff the last response carried a PARSED Retry-After — but "parsed" is defined by §6's parsing algorithm, and that algorithm cannot yield zero: step 1 requires an integer > 0, step 2 returns max(0, date − now) only when > 0, and the rounding rationale says why — "zero is read as 'no usable value' and drops the request onto the local backoff curve". So a conformant adapter's throttled ALWAYS carries a positive value, the original value gate honored every one of them exactly, and keying the delay on the kind bought nothing while turning a nonconforming adapter's throttled-with-zero into a zero-delay poll-retry — a tight loop against a server that is throttling the caller. The red run shows it literally: "poll-retry armed for 0s". pollRetryDelay is reverted to the value gate with the reconciliation written above it; the round-4 test asserting the wrong contract is replaced by one asserting the jitter fallthrough. What WAS wrong is the seam doc, which still described both throttled kinds by status ("429/503") — the exact status-keyed mapping the §6 sentence exists to forbid — and both docs now state the presence-keyed definition with the always-positive consequence. The tier-2 driver had the same status-keying in the flesh, on both lanes: a schema-valid 500/502/504 carrying Retry-After became transient and its delay never reached classifyMintFailure's floor, while a bare 429 became a throttled zero. Both outcome mappers now classify retryable statuses by presence of the parsed value. And the driver's Retry-After parse was integer-only — a valid RFC 7231 HTTP-date failed the scenario outright (red: strconv.Atoi rejecting the date), zero and negatives became durations. retryAfterFrom now implements §6 against the harness's virtual clock — integer > 0; else HTTP-date with the sub-second remainder rounded UP; else undefined — returning presence separately from the value. The round-3 grace-phase fix also overshot, in the other direction: an authoritative firing late in the phase re-armed graceWake for a FULL window, so a firing 1ms short of the deadline pushed the only remaining wake almost a whole window past it — the wait ended nearly two windows after the deferral instead of one. graceWake now takes its wake distance and every caller passes the remainder to its deadline (the full window at the deferral itself, where they coincide). The new red proof pins the upper bound the round-3 test could not see: advance to 1ms short, fire, advance the last 1ms, and demand the teardown there — red sat parked until the watchdog; green ends at the deadline exactly. --- go/pkg/basecamp/eventfeed/backoff.go | 30 ++--- go/pkg/basecamp/eventfeed/backoff_test.go | 22 +++- go/pkg/basecamp/eventfeed/catchup.go | 21 ++-- go/pkg/basecamp/eventfeed/catchup_test.go | 106 ++++++++++++++---- go/pkg/basecamp/eventfeed/loop.go | 12 +- .../eventfeed/scenario_conformance_test.go | 76 ++++++++----- .../eventfeed/scenario_selftest_test.go | 93 +++++++++++++++ go/pkg/basecamp/eventfeed/seams.go | 13 ++- 8 files changed, 294 insertions(+), 79 deletions(-) diff --git a/go/pkg/basecamp/eventfeed/backoff.go b/go/pkg/basecamp/eventfeed/backoff.go index 650891e850..02c30ab3de 100644 --- a/go/pkg/basecamp/eventfeed/backoff.go +++ b/go/pkg/basecamp/eventfeed/backoff.go @@ -74,20 +74,24 @@ func reconnectDelay(retryAfter time.Duration, n int, r func() float64) time.Dura // reconnect-cycle count, reset by any successful poll page and by socket // teardown. A server-directed Retry-After is waited exactly and is exempt // from local caps; otherwise the wait is a full-jitter draw over k. -// The selection is kind-keyed, per SPEC §6's fixed adapter mapping: "a -// retryable outcome exhausted inside the seam whose last response carried a +// The gate is the VALUE, not the kind, and the two §6 texts compose to make +// that correct rather than merely convenient. The seam mapping is +// presence-keyed — "a retryable outcome ... whose last response carried a // parsed Retry-After maps to throttled(retry_after) whatever its status, and -// one without maps to transient". PollThrottled therefore ALWAYS carries a -// server-directed wait, zero included — Retry-After: 0, or an HTTP-date -// already reached — and §23 waits it exactly, cap-exempt. Gating on -// retryAfter > 0 alone read a parsed zero as absence and replaced an -// immediate-retry directive with up to 60 seconds of local jitter. A -// transient carrying a positive RetryAfter is still honored defensively; a -// negative value (an adapter bug) clamps to zero rather than arming a -// negative timer. -func pollRetryDelay(kind PollErrorKind, retryAfter time.Duration, k int, r func() float64) time.Duration { - if kind == PollThrottled || retryAfter > 0 { - return max(retryAfter, 0) +// one without maps to transient" — but "parsed" is defined by §6's parsing +// algorithm, which never yields zero: step 1 requires an integer > 0, step 2 +// returns max(0, date − now) only when > 0, and the rounding rationale says +// why — "zero is read as 'no usable value' and drops the request onto the +// local backoff curve". A conformant adapter's throttled therefore ALWAYS +// carries a positive value, and every such value passes this gate exactly, +// cap-exempt. An earlier revision keyed on the kind instead and returned a +// throttled zero verbatim, which turned a nonconforming adapter's +// throttled-with-no-usable-header into a zero-delay poll-retry — a tight +// loop against a server that is throttling the caller. Zero and negative +// are "no usable value" whatever the kind claims, and draw the jitter curve. +func pollRetryDelay(retryAfter time.Duration, k int, r func() float64) time.Duration { + if retryAfter > 0 { + return retryAfter } return fullJitterDelay(k, r) } diff --git a/go/pkg/basecamp/eventfeed/backoff_test.go b/go/pkg/basecamp/eventfeed/backoff_test.go index 78864789e5..7735ac5c66 100644 --- a/go/pkg/basecamp/eventfeed/backoff_test.go +++ b/go/pkg/basecamp/eventfeed/backoff_test.go @@ -102,23 +102,23 @@ func TestReconnectDelay_RetryAfterWinsOutrightOverTheCap(t *testing.T) { func TestPollRetryDelay_RetryAfterIsExactAndCapExempt(t *testing.T) { // A server-directed Retry-After is waited exactly — even beyond the cap. - if got := pollRetryDelay(PollThrottled, 5*time.Minute, 1, fixedRand(0.5)); got != 5*time.Minute { + if got := pollRetryDelay(5*time.Minute, 1, fixedRand(0.5)); got != 5*time.Minute { t.Errorf("pollRetryDelay(5m Retry-After) = %v, want 5m", got) } // It is exact, not a floor: a Retry-After below the would-be draw wins. - if got := pollRetryDelay(PollThrottled, time.Second, 7, fixedRand(0.99)); got != time.Second { + if got := pollRetryDelay(time.Second, 7, fixedRand(0.99)); got != time.Second { t.Errorf("pollRetryDelay(1s Retry-After) = %v, want exactly 1s", got) } } func TestPollRetryDelay_FullJitterOverTheFailureIndex(t *testing.T) { // Without Retry-After the wait is a full-jitter draw over k. - if got := pollRetryDelay(PollTransient, 0, 2, fixedRand(0.5)); got != time.Second { + if got := pollRetryDelay(0, 2, fixedRand(0.5)); got != time.Second { t.Errorf("pollRetryDelay(k=2, r=0.5) = %v, want 1s", got) } rng := rand.New(rand.NewSource(2)) for i := 0; i < 1000; i++ { - d := pollRetryDelay(PollTransient, 0, 9, rng.Float64) + d := pollRetryDelay(0, 9, rng.Float64) if d < 0 || d >= backoffCap { t.Fatalf("pollRetryDelay(k=9) = %v, want in [0, %v)", d, backoffCap) } @@ -182,6 +182,20 @@ func TestRepairJitterSaturates(t *testing.T) { } } +func TestPollRetryDelay_UnusableValuesDrawTheJitterCurve(t *testing.T) { + // §6's parsing algorithm never yields zero ("zero is read as 'no usable + // value'"), so a conformant throttled always carries a positive value — + // and an unusable zero or negative, whatever kind a nonconforming + // adapter claims, draws local jitter rather than arming a zero-delay + // tight loop. + if got := pollRetryDelay(0, 2, fixedRand(0.5)); got != time.Second { + t.Errorf("pollRetryDelay(0, k=2, r=0.5) = %v, want the 1s jitter draw", got) + } + if got := pollRetryDelay(-time.Second, 2, fixedRand(0.5)); got != time.Second { + t.Errorf("pollRetryDelay(-1s, k=2, r=0.5) = %v, want the 1s jitter draw", got) + } +} + // A positive interval must stay positive after jitter: at 1ns, any downward // draw lands the float product in (0,1) and the Duration conversion floors // it to zero — a timer that fires immediately, every cycle, tight-looping diff --git a/go/pkg/basecamp/eventfeed/catchup.go b/go/pkg/basecamp/eventfeed/catchup.go index 46c964e6c5..b92ed3b709 100644 --- a/go/pkg/basecamp/eventfeed/catchup.go +++ b/go/pkg/basecamp/eventfeed/catchup.go @@ -515,8 +515,9 @@ func (l *loop) pollPage(at *attempt, cursor Cursor) pollAttempt { // nothing left to wake the grace phase at all, and the bounded wait // becomes unbounded again. Armed BEFORE the hook, so a test that // rendezvouses on the deferral cannot advance past a wake that is - // not yet armed. - at.lc.stale.graceWake() + // not yet armed. The distance is the remainder to the deadline — + // here, at the deferral itself, the full window. + at.lc.stale.graceWake(l.cfg.staleAfter) } if l.hooks.frameDeferred != nil { l.hooks.frameDeferred() @@ -604,10 +605,10 @@ func (l *loop) awaitSupersededPoll(at *attempt, done <-chan pollResult, deadline if stale { // The verdict is already rendered and parked; this firing is a // wake and nothing else, so nothing re-evaluates it. Re-arming - // keeps a wake in hand for the next turn, which is what makes - // the phase's end depend on the deadline rather than on a - // timer landing exactly on it. - at.lc.stale.graceWake() + // keeps a wake in hand for the next turn — for the REMAINDER + // to the deadline, so the last wake lands at the deadline + // itself rather than a window past it. + at.lc.stale.graceWake(deadline.Sub(l.cfg.clock.Now())) } else if _, ok := at.lc.stale.evaluate(staleGen); ok { // An authoritative expiry during another outcome's grace // phase is ALSO only a wake. The deferral slot already holds @@ -619,8 +620,10 @@ func (l *loop) awaitSupersededPoll(at *attempt, done <-chan pollResult, deadline // instead of the phase §23 measures from the deferral. The // expiry adds nothing the deferred outcome's teardown will // not do; graceWake latches it and keeps the next wake in - // hand, and the deadline below is what decides. - at.lc.stale.graceWake() + // hand — armed for the remainder to the deadline, which is + // what decides. A firing that lands late in the phase must + // not push the only remaining wake a full window past it. + at.lc.stale.graceWake(deadline.Sub(l.cfg.clock.Now())) } } if l.hooks.supersededWake != nil { @@ -1275,7 +1278,7 @@ func (l *loop) recoverPoll(at *attempt, cursor Cursor, err error) (walkStep, cyc // The self-loop inside CatchingUp: a wait, not a state change. The // same cursor is re-polled. l.pollFailures++ - out, done := l.waitPollRetry(at, pollRetryDelay(pe.Kind, pe.RetryAfter, l.pollFailures, l.cfg.rand)) + out, done := l.waitPollRetry(at, pollRetryDelay(pe.RetryAfter, l.pollFailures, l.cfg.rand)) return walkStep{cursor: cursor}, out, done case PollUnauthorized: // Recovery rides the reconnect cycle — the fresh mint/token pass — diff --git a/go/pkg/basecamp/eventfeed/catchup_test.go b/go/pkg/basecamp/eventfeed/catchup_test.go index 26c134498a..4c73e9e0e7 100644 --- a/go/pkg/basecamp/eventfeed/catchup_test.go +++ b/go/pkg/basecamp/eventfeed/catchup_test.go @@ -2739,28 +2739,6 @@ func TestExpiredStalenessOutranksThePollRetry(t *testing.T) { } } -// TestThrottledZeroRetryAfterIsWaitedExactly: SPEC §6 fixes the seam mapping -// — "a retryable outcome exhausted inside the seam whose last response -// carried a parsed Retry-After maps to throttled(retry_after) whatever its -// status, and one without maps to transient" — so PollThrottled always -// carries a server-directed wait, zero included (Retry-After: 0, or an -// HTTP-date already reached), and §23 waits it "exactly, cap-exempt". A -// delay selection gated on retryAfter > 0 reads a parsed zero as absence -// and replaces an immediate-retry directive with up to 60 seconds of local -// full-jitter backoff. -func TestThrottledZeroRetryAfterIsWaitedExactly(t *testing.T) { - h := newHarness(t) - h.minter.ScriptTicket(ticket(1)) - h.polls.ScriptError(&eventfeed.PollError{Kind: eventfeed.PollThrottled, RetryAfter: 0, Msg: "throttled"}) - h.start() - - conn := h.driveToSubscribed() - conn.Serve(frameConfirm(noFilterIdentifier)) - if d := h.fireTimer(timerPollRetry); d != 0 { - t.Fatalf("poll-retry armed for %s after a parsed Retry-After of 0; a throttled zero is server-directed and waited exactly (0 = retry now), never replaced with local jitter", d) - } -} - // TestLatchedStalenessOutranksAReadyLiveFrame: a frame received AFTER the // staleness window fired does not reset it — staleHolder.arm latches the // expiry instead, per §23's "a fired staleness deadline observed on return @@ -2813,3 +2791,87 @@ func TestLatchedStalenessOutranksAReadyLiveFrame(t *testing.T) { t.Errorf("%d/%d rounds delivered a live event past a latched staleness expiry; the frame arm must re-evaluate the verdict before delivering (transition 25 first)", violations, rounds) } } + +// TestThrottledZeroRetryAfterFallsToLocalBackoff: §6's parsing algorithm +// never yields zero — step 1 requires an integer > 0, step 2 takes +// max(0, date − now) and returns only when > 0, and the rounding rationale +// says it outright: "zero is read as 'no usable value' and drops the request +// onto the local backoff curve". So a conformant adapter's PollThrottled +// always carries a positive RetryAfter, and a throttled ZERO — a +// nonconforming status-keyed adapter's 429 with no usable header — must fall +// to the local jitter curve like any other absent value. Waiting it +// "exactly" arms a zero-delay poll-retry and tight-loops the walk against a +// server that is throttling it. +func TestThrottledZeroRetryAfterFallsToLocalBackoff(t *testing.T) { + h := newHarness(t) + h.minter.ScriptTicket(ticket(1)) + h.polls.ScriptError(&eventfeed.PollError{Kind: eventfeed.PollThrottled, RetryAfter: 0, Msg: "throttled"}) + h.start() + + conn := h.driveToSubscribed() + conn.Serve(frameConfirm(noFilterIdentifier)) + if d := h.fireTimer(timerPollRetry); d <= 0 { + t.Fatalf("poll-retry armed for %s on a throttled zero; an unusable Retry-After is undefined and draws local jitter — a zero delay tight-loops against a throttling server", d) + } +} + +// TestGracePhaseEndsAtItsDeadlineNotAWindowAfterItsLastWake pins the grace +// phase's UPPER bound. The phase is a fixed deadline read at the deferral; +// when the pre-existing staleness window fires just short of that deadline, +// the wake re-armed for the remainder must land AT the deadline — re-arming +// a full window there pushes the only remaining wake almost one whole window +// past the deadline, and the wait ends nearly two windows after the deferral +// instead of one. +func TestGracePhaseEndsAtItsDeadlineNotAWindowAfterItsLastWake(t *testing.T) { + wakes := make(chan struct{}, 64) + deferred := make(chan struct{}, 1) + disconnected := make(chan error, 8) + h := newHarness(t, eventfeed.WithObserver(eventfeed.Observer{ + Disconnected: func(_ string, err error) { disconnected <- err }, + })) + h.conn.OnSupersededWake(func() { + select { + case wakes <- struct{}{}: + default: + } + }) + h.conn.OnFrameDeferred(func() { + select { + case deferred <- struct{}{}: + default: + } + }) + h.minter.ScriptTicket(ticket(1)) + h.polls.StallNext() + h.start() + + conn := h.driveToSubscribed() + conn.Serve(frameConfirm(noFilterIdentifier)) + h.waitUntil("entry poll in flight", func() bool { return h.polls.CallCount() == 1 }) + conn.FailReads(errors.New("read failed mid-poll")) + select { + case <-deferred: + // The grace deadline is now Now + 7500ms. + case <-time.After(watchdog): + t.Fatal("the read failure was not deferred") + } + + // The pre-existing window fires 1ms short of the deadline. + h.clock.Advance(7499 * time.Millisecond) + h.fireTimer(timerStaleness) + select { + case <-wakes: + case <-time.After(watchdog): + t.Fatal("the staleness firing inside the grace phase did not wake the wait") + } + + // One more millisecond reaches the deadline exactly; the wait must end + // there — not a full window later. + h.clock.Advance(time.Millisecond) + select { + case <-disconnected: + case <-time.After(watchdog): + t.Fatal("the grace deadline passed with the poll still awaited: the wake was re-armed a FULL window out (deadline + ~7.5s), not at the deadline") + } + h.awaitTimer(timerBackoff) +} diff --git a/go/pkg/basecamp/eventfeed/loop.go b/go/pkg/basecamp/eventfeed/loop.go index edac106612..ed0e69575c 100644 --- a/go/pkg/basecamp/eventfeed/loop.go +++ b/go/pkg/basecamp/eventfeed/loop.go @@ -444,11 +444,19 @@ func (h *staleHolder) evaluate(gen int) (time.Duration, bool) { // another, so after the expiry there is no wake left at all — which is exactly // how the bounded wait became unbounded again. // +// The caller names the wake's distance — the remainder to ITS deadline, not a +// window. An earlier revision always armed a full window, and a firing that +// landed late in the grace phase then pushed the only remaining wake almost a +// whole window past the deadline: the wait ended nearly two windows after the +// deferral instead of one. A non-positive remainder arms at zero — the +// deadline is due, and the immediate firing is the wake that lets the +// post-select check say so. +// // It arms the same `staleness` kind: SPEC.md §23 pins exactly six timer kinds // and every state's exact timer set, both asserted by the cross-SDK fixtures, // so a dedicated grace timer would be a spec change across six SDKs. The // outstanding set is unchanged — {staleness} stays {staleness}. -func (h *staleHolder) graceWake() { +func (h *staleHolder) graceWake(d time.Duration) { h.mu.Lock() defer h.mu.Unlock() if h.stopped { @@ -456,7 +464,7 @@ func (h *staleHolder) graceWake() { } h.expired = true h.timer.Stop() - h.timer = h.clock.NewTimer(h.d, timerStaleness) + h.timer = h.clock.NewTimer(max(d, 0), timerStaleness) } // stop cancels the timer permanently. diff --git a/go/pkg/basecamp/eventfeed/scenario_conformance_test.go b/go/pkg/basecamp/eventfeed/scenario_conformance_test.go index bffb477f70..41f95fc7a1 100644 --- a/go/pkg/basecamp/eventfeed/scenario_conformance_test.go +++ b/go/pkg/basecamp/eventfeed/scenario_conformance_test.go @@ -22,6 +22,7 @@ import ( "errors" "fmt" "maps" + "net/http" "os" "path/filepath" "slices" @@ -353,11 +354,7 @@ func (d *driver) expectMint(step *expectMintStep) error { if err != nil { return err } - outcome, err := mintOutcomeFrom(step.Respond) - if err != nil { - return err - } - call.release <- outcome + call.release <- d.mintOutcomeFrom(step.Respond) return nil } @@ -1014,30 +1011,34 @@ func (h *scenarioHarness) violation() error { // mintOutcomeFrom maps a scripted mint response onto the seam's outcome: a // ticket, or the classified *MintError the Layer-1 adapter would produce. -func mintOutcomeFrom(respond mintRespond) (mintOutcome, error) { +// Throttled-vs-transient is keyed on the PRESENCE of a §6-parsed Retry-After +// (SPEC's fixed adapter mapping — "whatever its status"), never on the +// status: a status key would honour the header at one retryable status and +// drop it at another. Unauthorized keeps a parsed value too — row 4 floors +// the below-threshold reconnect delay on it. +func (d *driver) mintOutcomeFrom(respond mintRespond) mintOutcome { if respond.Body != nil { return mintOutcome{ticket: eventfeed.StreamTicket{ Ticket: respond.Body.Ticket, ExpiresIn: respond.Body.ExpiresIn, URL: respond.Body.URL, - }}, nil - } - retryAfter, err := retryAfterFrom(respond.Headers) - if err != nil { - return mintOutcome{}, err + }} } + retryAfter, present := d.retryAfterFrom(respond.Headers) mintErr := &eventfeed.MintError{RetryAfter: retryAfter, Err: fmt.Errorf("mint responded %d", *respond.Status)} switch *respond.Status { case 401, 403: mintErr.Kind = eventfeed.MintUnauthorized case 404, 422: mintErr.Kind = eventfeed.MintUnrecoverable - case 429, 503: - mintErr.Kind = eventfeed.MintThrottled default: - mintErr.Kind = eventfeed.MintTransient + if present { + mintErr.Kind = eventfeed.MintThrottled + } else { + mintErr.Kind = eventfeed.MintTransient + } } - return mintOutcome{err: mintErr}, nil + return mintOutcome{err: mintErr} } // pollOutcomeFrom maps a scripted poll response onto the seam's outcome: a @@ -1067,12 +1068,12 @@ func (d *driver) pollOutcomeFrom(respond pollRespond) (pollOutcome, error) { Err: errors.New("poll responded 401/403"), }}, nil default: - retryAfter, err := retryAfterFrom(respond.Headers) - if err != nil { - return pollOutcome{}, err - } + // Presence-keyed, exactly as the mint lane: a retryable outcome with + // a §6-parsed Retry-After is throttled whatever its status, one + // without is transient. + retryAfter, present := d.retryAfterFrom(respond.Headers) kind := eventfeed.PollTransient - if status == 429 || status == 503 { + if present { kind = eventfeed.PollThrottled } return pollOutcome{err: &eventfeed.PollError{ @@ -1149,16 +1150,39 @@ func (d *driver) redirectRefusalFrom(headers map[string]string) (pollOutcome, er }}, nil } -func retryAfterFrom(headers map[string]string) (time.Duration, error) { - value, ok := headers["Retry-After"] +// retryAfterFrom is SPEC §6's Retry-After parsing algorithm, judged against +// the harness's virtual clock: an integer valid and > 0 is seconds; else a +// valid HTTP-date yields max(0, date − now) with a sub-second remainder +// rounded UP, returned only when > 0; everything else — zero, negatives, +// already-passed dates, malformed values — is undefined. present is whether +// the algorithm returned at all, and it is what keys the seam mapping +// (throttled with a value, transient without); the value is always positive +// when present, because the algorithm cannot yield zero ("zero is read as +// 'no usable value'"). +func (d *driver) retryAfterFrom(headers map[string]string) (value time.Duration, present bool) { + raw, ok := headers["Retry-After"] if !ok { - return 0, nil + return 0, false } - seconds, err := strconv.Atoi(value) + if seconds, err := strconv.Atoi(raw); err == nil { + if seconds > 0 { + return time.Duration(seconds) * time.Second, true + } + return 0, false + } + date, err := http.ParseTime(raw) if err != nil { - return 0, fmt.Errorf("Retry-After %q is not a delta-seconds value: %w", value, err) + return 0, false + } + remainder := date.Sub(d.h.clock.Now()) + if remainder <= 0 { + return 0, false + } + seconds := int64(remainder / time.Second) + if remainder%time.Second != 0 { + seconds++ } - return time.Duration(seconds) * time.Second, nil + return time.Duration(seconds) * time.Second, true } // --- server frames ------------------------------------------------------- diff --git a/go/pkg/basecamp/eventfeed/scenario_selftest_test.go b/go/pkg/basecamp/eventfeed/scenario_selftest_test.go index 55643f4d8c..0e9bc7d278 100644 --- a/go/pkg/basecamp/eventfeed/scenario_selftest_test.go +++ b/go/pkg/basecamp/eventfeed/scenario_selftest_test.go @@ -11,7 +11,9 @@ package eventfeed_test import ( "context" "encoding/json" + "errors" "fmt" + "net/http" "os" "path/filepath" "strings" @@ -678,3 +680,94 @@ func TestScenarioDriverScanScopesOccupancyHistoryToTheCurrentEra(t *testing.T) { t.Fatalf("the scan read through expectBuffered(1) on occupancy from a previous era: an arriving save would be judged against %s and accepted out of order", where) } } + +// intp is a scripted-status literal. +func intp(s int) *int { return &s } + +// TestDriverSeamClassificationIsPresenceKeyed: SPEC §6 fixes the adapter +// mapping the driver models — "a retryable outcome exhausted inside the seam +// whose last response carried a parsed Retry-After maps to +// throttled(retry_after) whatever its status, and one without maps to +// transient. A mapping keyed on status instead ... would honour the header at +// one retryable status and drop it at another, which is exactly the gate +// this section removes." And §6's parsing algorithm never yields zero +// ("zero is read as 'no usable value'"), so an unusable header is absence, +// not a throttled zero. +func TestDriverSeamClassificationIsPresenceKeyed(t *testing.T) { + d := &driver{h: newScenarioHarness()} + defer d.h.close() + + // A retryable non-429/503 status WITH a parsed Retry-After is throttled, + // carrying the value. + out := d.mintOutcomeFrom(mintRespond{Status: intp(502), Headers: map[string]string{"Retry-After": "3"}}) + var me *eventfeed.MintError + if !errors.As(out.err, &me) || me.Kind != eventfeed.MintThrottled || me.RetryAfter != 3*time.Second { + t.Fatalf("502 with Retry-After: 3 = %+v, want MintThrottled carrying 3s (presence-keyed, not status-keyed)", out.err) + } + // A 429 with NO usable header is transient: undefined falls to backoff. + out = d.mintOutcomeFrom(mintRespond{Status: intp(429)}) + if !errors.As(out.err, &me) || me.Kind != eventfeed.MintTransient { + t.Fatalf("bare 429 = %+v, want MintTransient (no parsed Retry-After)", out.err) + } + // Retry-After: 0 is undefined per the parsing algorithm (step 1 requires + // > 0) — transient, never a throttled zero. + out = d.mintOutcomeFrom(mintRespond{Status: intp(503), Headers: map[string]string{"Retry-After": "0"}}) + if !errors.As(out.err, &me) || me.Kind != eventfeed.MintTransient { + t.Fatalf("503 with Retry-After: 0 = %+v, want MintTransient (zero is 'no usable value')", out.err) + } + + // The poll lane, same mapping. + po, err := d.pollOutcomeFrom(pollRespond{Status: intp(502), Headers: map[string]string{"Retry-After": "2"}}) + if err != nil { + t.Fatalf("poll 502 with Retry-After: 2: %v", err) + } + var pe *eventfeed.PollError + if !errors.As(po.err, &pe) || pe.Kind != eventfeed.PollThrottled || pe.RetryAfter != 2*time.Second { + t.Fatalf("poll 502 with Retry-After: 2 = %+v, want PollThrottled carrying 2s", po.err) + } + po, err = d.pollOutcomeFrom(pollRespond{Status: intp(429)}) + if err != nil { + t.Fatalf("bare poll 429: %v", err) + } + if !errors.As(po.err, &pe) || pe.Kind != eventfeed.PollTransient { + t.Fatalf("bare poll 429 = %+v, want PollTransient", po.err) + } +} + +// TestDriverRetryAfterParseIsTheSpecAlgorithm: SPEC §6's parsing algorithm, +// which the driver claims to emulate — integer valid and > 0; else HTTP-date +// with max(0, date − now) rounded UP on a sub-second remainder, > 0; else +// undefined. Dates are judged against the harness's virtual clock. +func TestDriverRetryAfterParseIsTheSpecAlgorithm(t *testing.T) { + d := &driver{h: newScenarioHarness()} + defer d.h.close() + + // An HTTP-date carries whole seconds only, so the sub-second remainder + // must live in NOW: park the virtual clock exactly 90.3s before a + // whole-second instant, deterministically whatever the epoch's own + // fraction. + now0 := d.h.clock.Now() + dateInstant := now0.Truncate(time.Second).Add(93 * time.Second) + d.h.clock.Advance(dateInstant.Add(-90*time.Second - 300*time.Millisecond).Sub(now0)) + now := d.h.clock.Now() + + // A valid RFC 7231 HTTP-date 90.3s out parses, rounded UP to 91s. + date := dateInstant.UTC().Format(http.TimeFormat) + out := d.mintOutcomeFrom(mintRespond{Status: intp(429), Headers: map[string]string{"Retry-After": date}}) + var me *eventfeed.MintError + if !errors.As(out.err, &me) || me.Kind != eventfeed.MintThrottled || me.RetryAfter != 91*time.Second { + t.Fatalf("HTTP-date 90.3s out = %+v, want MintThrottled carrying 91s (sub-second remainder rounds UP)", out.err) + } + // A date already passed is undefined — max(0, ·) is not > 0. + past := now.Add(-time.Minute).UTC().Format(http.TimeFormat) + for name, header := range map[string]string{ + "past date": past, + "negative": "-5", + "malformed": "soon", + } { + out = d.mintOutcomeFrom(mintRespond{Status: intp(429), Headers: map[string]string{"Retry-After": header}}) + if !errors.As(out.err, &me) || me.Kind != eventfeed.MintTransient || me.RetryAfter != 0 { + t.Fatalf("%s Retry-After = %+v, want MintTransient with no value (undefined)", name, out.err) + } + } +} diff --git a/go/pkg/basecamp/eventfeed/seams.go b/go/pkg/basecamp/eventfeed/seams.go index 6109751f27..87759ff249 100644 --- a/go/pkg/basecamp/eventfeed/seams.go +++ b/go/pkg/basecamp/eventfeed/seams.go @@ -49,8 +49,10 @@ const ( // MintTransient — a retryable outcome exhausted inside the seam; rides // the reconnect cycle (Backoff). MintTransient MintErrorKind = iota + 1 - // MintThrottled — 429/503; RetryAfter honored as the floor of the next - // reconnect delay. + // MintThrottled — a retryable outcome whose last response carried a + // §6-parsed Retry-After, whatever its status (the same presence-keyed + // adapter mapping as PollThrottled; the parsed value is always > 0). + // RetryAfter is honored as the floor of the next reconnect delay. MintThrottled // MintUnauthorized — 401/403; increments the shared connection-level // authorization counter (terminal authorization_failed at threshold 3). @@ -148,7 +150,12 @@ const ( // PollTransient — a retryable outcome exhausted inside the seam; retries // on the poll-retry timer, never terminal. PollTransient PollErrorKind = iota + 1 - // PollThrottled — 429/503; RetryAfter waited exactly, cap-exempt. + // PollThrottled — a retryable outcome whose last response carried a + // §6-parsed Retry-After, whatever its status (SPEC §6 fixes the adapter + // mapping by presence of the parsed value, not by status — a status + // key would honour the header at one retryable status and drop it at + // another). The parsing algorithm yields only positive values, so + // RetryAfter is always > 0; it is waited exactly, cap-exempt. PollThrottled // PollPositionInvalid — 400-position: re-enter since= (or a present-class entry with none). From 222beed2310984fb1b4ec205826b0befe4d98ac8 Mon Sep 17 00:00:00 2001 From: Jeremy Daer Date: Sat, 22 Aug 2026 12:23:58 -0700 Subject: [PATCH 27/51] Event feed: the teardown's courtesy call, and three more places Close lost to an announcement MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit All seven of this round's findings taken — four in the loop's established families, three in the README, each verified against the code first. A superseded poll is now cancelled AT the grace deadline, in time and not merely in control flow. Disposal closes the socket gracefully before it cancels — deliberately, so the peer sees a close frame — but a peer that never acknowledges holds that close for the transport's full grace budget, and the abandoned seam call (carrying the caller's bearer) stayed live through it, stalling reconnection past §23's detection-plus-grace bound. The deadline branch cancels the attempt context itself; the disposal that follows finds its own cancel a no-op. The red proof captures the pump's read context and parks the teardown's close: pre-fix the context was still live at the park. AwaitingConfirmation's frame arm gains the latched-expiry drain the Streaming arm got last round — the same shape for a worse consequence: a frame received after the window fired could win the select and either admit a correlated message to the loop-wide buffer, which survives the teardown and resurfaces after reconnect, or hand a confirmation a live catch-up on a socket whose verdict was already in. Rounds red: 11/40 announced Confirmed past a latched expiry. Two Close-outranks-the-announcement gaps, both on the far side of observer callbacks that are documented Close sites. outcomeFailed announced Backoff and asked the product clock for a reconnect timer with runCtx already cancelled by a Close inside Disconnected or StaleConnection; finishDrain continued past CaughtUp into dispatchDeferred and stream, which re-arms the repair cadence through the same product seam and announces Streaming. Both now take the Closed edge first — a host clock that blocks in NewTimer would otherwise keep the iteration and Wait from ever reaching Closed. Red proofs: the state ledger carried "backoff" (and "streaming") after Close returned; green goes straight to closed. Three README corrections, each checked against the code. The terminal-vs- continuable section implied every continuable outcome waits a timer; rejected positions do not — reenterWalk re-polls a replacement cursor immediately on the same socket, and the text now says which failure continues how. The unwrap claim promised a generated error behind every poll_failed; the connector's own verdicts carry none (a 200 whose page has no position), so the claim is now conditional. And the consumer-surface section stated "Close stops the feed" without the lifecycle this branch actually ships: Close only cancels and can return with a save still in flight, Wait is the quiescence point, and a replacement connector opened straight after Close races the prior run's last save — the README now says so where the store guidance lives. --- go/README.md | 25 ++- go/pkg/basecamp/eventfeed/catchup.go | 22 +++ go/pkg/basecamp/eventfeed/catchup_test.go | 229 ++++++++++++++++++++++ go/pkg/basecamp/eventfeed/loop.go | 32 +++ go/pkg/basecamp/eventfeed/loop_test.go | 30 +++ 5 files changed, 332 insertions(+), 6 deletions(-) diff --git a/go/README.md b/go/README.md index a4fdb5121a..537b1c906a 100644 --- a/go/README.md +++ b/go/README.md @@ -867,6 +867,14 @@ attempts. breaking out of the loop all end iteration with **no** error element — a clean stop, and the feed is resumable by design. +`Close` only cancels: it can return while the run's final acts are still in flight — +including a checkpoint save, which deliberately outlives cancellation (an accepted +page's events were already delivered, and dropping the write would silently re-deliver +them). `Wait` is the quiescence point: it returns only once the run goroutine has +exited, so no save can still be in flight. Await the iterator's end — or `Wait` — +before opening a second connector over the same checkpoint store; a replacement opened +straight after `Close` can race the prior run's last save. + ### Checkpointing `FileCheckpointStore` is the built-in `CheckpointStore`: one JSON file holding every @@ -903,17 +911,22 @@ the disposition lives exclusively in the handler. ### Terminal versus continuable -Continuable failures — a dropped socket, a staleness expiry, a throttled or transient -mint or poll, a rejected position (400-position or 409) — never reach the consumer as -errors. They ride the reconnect cycle (full-jitter backoff) or the poll-retry timer, and -you see them through `Observer` callbacks if you register any. +Continuable failures never reach the consumer as errors; you see them through +`Observer` callbacks if you register any. How each continues differs: a dropped socket, +a staleness expiry, and a throttled or transient mint ride the reconnect cycle +(full-jitter backoff, fresh ticket); a throttled or transient poll waits the poll-retry +timer on the same socket; a rejected position (400-position or 409) re-enters the walk +**immediately** — a replacement cursor re-polled on the same socket, with no timer and +no reconnect between. A terminal condition ends the iteration with exactly one final `*eventfeed.TerminalError` element carrying a `TerminalReason`: `subscription_rejected`, `protocol_fatal`, `filter_invalid`, `authorization_failed`, `checkpoint_load`, `usage`, `buffer_overflow`, `feed_gap`, `invalid_continuation`, `poll_failed`, `mint_failed`, or `invalid_cable_url`. -Switch on `te.Reason` rather than on message text; `errors.Unwrap` reaches the generated -error behind `mint_failed` and `poll_failed`. +Switch on `te.Reason` rather than on message text. `errors.Unwrap` reaches the +generated error behind `mint_failed`, and behind `poll_failed` when one exists — the +connector's own poll verdicts carry none (a 200 whose page has no position is +`poll_failed` with no generated cause), so check the unwrapped error for nil. ## Pagination diff --git a/go/pkg/basecamp/eventfeed/catchup.go b/go/pkg/basecamp/eventfeed/catchup.go index b92ed3b709..6b4e2fa8a3 100644 --- a/go/pkg/basecamp/eventfeed/catchup.go +++ b/go/pkg/basecamp/eventfeed/catchup.go @@ -100,6 +100,16 @@ func (l *loop) walkThenDrain(at *attempt, cursor Cursor, presentClass bool) (cyc if l.cfg.observer.CaughtUp != nil { l.cfg.observer.CaughtUp() } + // And Close outranks what FOLLOWS the announcement too: CaughtUp is + // supported as a Close site like every callback, and past this point the + // path re-arms the repair cadence through the product clock's NewTimer + // and announces Streaming — reconnect-era acts a consumer who closed + // inside the callback must not observe (and a host clock that blocks in + // NewTimer would keep the iteration and Wait from reaching Closed). + if l.runCtx.Err() != nil { + l.disposeAttempt(at, nil) + return cycleOutcome{kind: outcomeClosed}, true + } // The walk's last dispatch point, and by construction a SOCKET outcome // only: nothing else is ever deferred — an overflowing admission is // dispatched where it is observed, and the drain's scan already took the @@ -630,6 +640,18 @@ func (l *loop) awaitSupersededPoll(at *attempt, done <-chan pollResult, deadline l.hooks.supersededWake() } if !l.cfg.clock.Now().Before(deadline) { + // Abandoned AT the deadline, in time and not merely in control + // flow: the teardown that dispatches the deferred outcome closes + // the socket gracefully FIRST (liveConn.dispose, deliberately), + // and a peer that never acknowledges holds that close for the + // transport's full grace budget — during which the abandoned + // seam call, carrying the caller's bearer, would still be live + // and the reconnect stalled past §23's published + // detection-plus-grace bound. The attempt context is the call's + // cancellation (Seam-Call Semantics), so it is cancelled here; + // the disposal that follows finds the pump already unblocked and + // its own cancel a no-op. + at.cancel() return pollAttempt{superseded: true} } } diff --git a/go/pkg/basecamp/eventfeed/catchup_test.go b/go/pkg/basecamp/eventfeed/catchup_test.go index 4c73e9e0e7..fdd0d3423f 100644 --- a/go/pkg/basecamp/eventfeed/catchup_test.go +++ b/go/pkg/basecamp/eventfeed/catchup_test.go @@ -9,6 +9,7 @@ package eventfeed_test import ( + "context" "errors" "fmt" "runtime" @@ -2875,3 +2876,231 @@ func TestGracePhaseEndsAtItsDeadlineNotAWindowAfterItsLastWake(t *testing.T) { } h.awaitTimer(timerBackoff) } + +// pollAbandonProbeTransport wraps the harness transport twice over: it +// captures the context the pump reads with (a live handle on the attempt +// context, which is also the poll seam call's cancellation), and it parks +// the FIRST socket close until released — the deterministic stand-in for a +// peer that never acknowledges a graceful close and holds wsConn.Close for +// its full grace budget. +type pollAbandonProbeTransport struct { + inner *feedtest.Transport + parked chan struct{} + release chan struct{} + once sync.Once + + mu sync.Mutex + readCtx context.Context +} + +func (p *pollAbandonProbeTransport) Dial(ctx context.Context, wsURL string, maxFrameBytes int64) (eventfeed.CableConn, error) { + conn, err := p.inner.Dial(ctx, wsURL, maxFrameBytes) + if err != nil { + return nil, err + } + return &pollAbandonProbeConn{CableConn: conn, tr: p}, nil +} + +type pollAbandonProbeConn struct { + eventfeed.CableConn + tr *pollAbandonProbeTransport +} + +func (c *pollAbandonProbeConn) ReadFrame(ctx context.Context) ([]byte, error) { + c.tr.mu.Lock() + if c.tr.readCtx == nil { + c.tr.readCtx = ctx + } + c.tr.mu.Unlock() + return c.CableConn.ReadFrame(ctx) +} + +func (c *pollAbandonProbeConn) Close(code int, reason string) error { + c.tr.once.Do(func() { close(c.tr.parked) }) + <-c.tr.release + return c.CableConn.Close(code, reason) +} + +// TestSupersededPollIsCancelledAtTheGraceDeadline: §23 abandons a superseded +// seam call at the grace deadline, and "abandoned to the deferred outcome's +// teardown (the teardown cancels it)" must mean cancelled AT the deadline — +// not after the teardown's graceful socket close, which a peer that never +// acknowledges holds for the transport's full close-grace budget. Until the +// cancel lands, the abandoned poll (carrying the caller's bearer) is still +// live and reconnection is stalled past the published +// detection-window-plus-grace bound. +func TestSupersededPollIsCancelledAtTheGraceDeadline(t *testing.T) { + pt := &pollAbandonProbeTransport{parked: make(chan struct{}), release: make(chan struct{})} + h := newHarness(t, eventfeed.WithTransport(pt)) + pt.inner = h.tr + var relOnce sync.Once + rel := func() { relOnce.Do(func() { close(pt.release) }) } + t.Cleanup(rel) + wakes := make(chan struct{}, 64) + deferred := make(chan struct{}, 1) + h.conn.OnSupersededWake(func() { + select { + case wakes <- struct{}{}: + default: + } + }) + h.conn.OnFrameDeferred(func() { + select { + case deferred <- struct{}{}: + default: + } + }) + h.minter.ScriptTicket(ticket(1)) + h.polls.StallNext() + h.start() + + conn := h.driveToSubscribed() + conn.Serve(frameConfirm(noFilterIdentifier)) + h.waitUntil("entry poll in flight", func() bool { return h.polls.CallCount() == 1 }) + conn.FailReads(errors.New("read failed mid-poll")) + select { + case <-deferred: + case <-time.After(watchdog): + t.Fatal("the read failure was not deferred") + } + h.fireTimer(timerStaleness) + select { + case <-wakes: + case <-time.After(watchdog): + t.Fatal("no wake followed the staleness firing") + } + // Reaching the deadline abandons the call; the teardown's socket close + // then parks in the wrapper, exactly where a slow peer would hold it. + h.clock.Advance(7500 * time.Millisecond) + select { + case <-pt.parked: + case <-time.After(watchdog): + t.Fatal("the teardown never reached the socket close") + } + pt.mu.Lock() + readCtx := pt.readCtx + pt.mu.Unlock() + if readCtx == nil { + t.Fatal("no attempt context was captured") + } + if readCtx.Err() == nil { + t.Fatal("the superseded poll's context is still live while disposal waits in the socket close: the abandoned call outlives the grace deadline by up to the close-grace budget") + } + rel() + h.awaitTimer(timerBackoff) +} + +// TestCloseFromCaughtUpDoesNotEnterStreaming: Close is supported from every +// observer callback, and CaughtUp is the last one before Streaming — a Close +// taken inside it must take the universal Closed edge, not announce +// Streaming and ask the product clock for a repair timer after Close has +// returned (a host clock that blocks in NewTimer would then keep the +// iteration and Wait from ever reaching Closed). +func TestCloseFromCaughtUpDoesNotEnterStreaming(t *testing.T) { + var mu sync.Mutex + var states []string + var h *harness + h = newHarness(t, eventfeed.WithObserver(eventfeed.Observer{ + CaughtUp: func() { h.conn.Close() }, //nolint:errcheck // asserted via states + })) + h.conn.OnStateChanged(func(s string) { mu.Lock(); states = append(states, s); mu.Unlock() }) + h.minter.ScriptTicket(ticket(1)) + h.polls.ScriptPage(eventfeed.PollPage{Position: "pos-1"}) + h.start() + + conn := h.driveToSubscribed() + conn.Serve(frameConfirm(noFilterIdentifier)) + h.join() + + mu.Lock() + defer mu.Unlock() + if slices.Contains(states, "streaming") { + t.Fatalf("states %v: Streaming was entered (and the repair timer armed through the product clock) after Close returned from CaughtUp", states) + } +} + +// TestLatchedStalenessOutranksAReadyConfirmation is the AwaitingConfirmation +// sibling of the Streaming frame-arm fix: a frame received after the window +// fired does not reset it (the expiry latches), and both the fired timer and +// that frame can be ready at the select. Handling the frame first hands a +// confirmation a live catch-up — Confirmed announced, the entry poll issued — +// on a socket whose staleness verdict is already in. Rounds-driven: the run +// goroutine parks in an overflow handler while the window fires and the +// confirmation arrives behind it; any round announcing Confirmed is a +// violation. +func TestLatchedStalenessOutranksAReadyConfirmation(t *testing.T) { + const rounds = 40 + violations := 0 + for i := range rounds { + func() { + parked := make(chan struct{}, 1) + release := make(chan struct{}) + var once sync.Once + rel := func() { once.Do(func() { close(release) }) } + t.Cleanup(rel) + confirmed := make(chan struct{}, 1) + disconnected := make(chan struct{}, 8) + h := newHarness(t, + eventfeed.WithLiveBufferCapacity(1), + eventfeed.WithSignalHandler(func(eventfeed.Signal) eventfeed.Disposition { + select { + case parked <- struct{}{}: + default: + } + <-release + return eventfeed.Accept + }), + eventfeed.WithObserver(eventfeed.Observer{ + Confirmed: func() { + select { + case confirmed <- struct{}{}: + default: + } + }, + Disconnected: func(string, error) { + select { + case disconnected <- struct{}{}: + default: + } + }, + })) + h.minter.ScriptTicket(ticket(1)) + h.start() + + conn := h.driveToSubscribed() + conn.Serve(frameMessage(noFilterIdentifier, 101)) + conn.Serve(frameMessage(noFilterIdentifier, 102)) + select { + case <-parked: + case <-time.After(watchdog): + t.Fatal("the overflow handler never parked the run goroutine") + } + h.fireTimer(timerStaleness) + conn.Serve(frameConfirm(noFilterIdentifier)) + h.waitUntil("the confirmation to reach the hand-off", func() bool { + return conn.Pending() == 0 + }) + rel() + + deadline := time.Now().Add(watchdog) + for { + select { + case <-confirmed: + violations++ + t.Logf("round %d: Confirmed announced past a latched staleness expiry", i) + return + case <-disconnected: + return + default: + } + if time.Now().After(deadline) { + t.Fatal("round undecided: neither Confirmed nor a teardown") + } + time.Sleep(time.Millisecond) + } + }() + } + if violations != 0 { + t.Errorf("%d/%d rounds confirmed a subscription past a latched staleness expiry; the frame arm must re-evaluate the verdict first (transition 15 before 11)", violations, rounds) + } +} diff --git a/go/pkg/basecamp/eventfeed/loop.go b/go/pkg/basecamp/eventfeed/loop.go index ed0e69575c..8dcc29ddab 100644 --- a/go/pkg/basecamp/eventfeed/loop.go +++ b/go/pkg/basecamp/eventfeed/loop.go @@ -699,6 +699,18 @@ func (l *loop) run(yield func(Event, error) bool) { l.emitTerminal(out.term) return case outcomeFailed: + // Close outranks the announcement here as everywhere: the failed + // cycle's own observer callbacks (Disconnected, StaleConnection) + // are supported Close sites, and runCtx is then already + // cancelled when this outcome arrives — announcing Backoff and + // asking the product clock for a reconnect timer would be + // reconnect acts performed after Close returned, and a host + // clock that blocks in NewTimer would keep the iteration and + // Wait from reaching Closed at all. + if l.runCtx.Err() != nil { + l.setState(stateClosed) + return + } // The failed attempt is fully disposed, so the exact // outstanding-timer set on entry to Backoff is {backoff}. l.failedCycles++ @@ -1046,6 +1058,26 @@ func (l *loop) awaitConfirmation(at *attempt, deadline Timer) cycleOutcome { l.observeDisconnected("", errors.New("event feed frame pump exited")) return cycleOutcome{kind: outcomeFailed} } + // A latched expiry outranks a ready frame, exactly as on the + // Streaming frame arm (catchup.go): a frame received after the + // window fired did not reset it, and handling it first admits a + // correlated message to the loop-wide buffer — which survives + // the teardown and resurfaces after reconnect — or hands a + // confirmation a live catch-up on a socket whose staleness + // verdict is already in. evaluate arbitrates: a frame the pump + // received first moved the generation and is handled as ever. + select { + case <-staleTimer.C(): + if age, authoritative := at.lc.stale.evaluate(staleGen); authoritative { + l.disposeAttempt(at, deadline) + if l.cfg.observer.StaleConnection != nil { + l.cfg.observer.StaleConnection(age) + } + l.observeDisconnected("", errStaleConnection) + return cycleOutcome{kind: outcomeFailed} + } + default: + } if out, done := l.handleFrame(at, &deadline, item); done { return out } diff --git a/go/pkg/basecamp/eventfeed/loop_test.go b/go/pkg/basecamp/eventfeed/loop_test.go index 24750935e1..2627e1471b 100644 --- a/go/pkg/basecamp/eventfeed/loop_test.go +++ b/go/pkg/basecamp/eventfeed/loop_test.go @@ -2400,3 +2400,33 @@ func TestStalledSubscribeWriteObservesStaleness(t *testing.T) { } h.awaitTimer(timerBackoff) } + +// TestCloseFromDisconnectedDoesNotAnnounceBackoff: Observer.Disconnected and +// StaleConnection run inside the failed cycle, and Close is supported from +// both — runCtx is then already cancelled when outcomeFailed reaches the run +// loop, and announcing Backoff plus asking the product clock for a reconnect +// timer is a reconnect act performed after Close returned (a host clock that +// blocks in NewTimer would keep the iteration and Wait from reaching Closed +// at all). Close outranks the announcement, exactly as it does for +// Connecting. +func TestCloseFromDisconnectedDoesNotAnnounceBackoff(t *testing.T) { + var mu sync.Mutex + var states []string + var h *harness + h = newHarness(t, eventfeed.WithObserver(eventfeed.Observer{ + Disconnected: func(string, error) { h.conn.Close() }, //nolint:errcheck // asserted via states + })) + h.conn.OnStateChanged(func(s string) { mu.Lock(); states = append(states, s); mu.Unlock() }) + h.minter.ScriptTicket(ticket(1)) + h.start() + + conn := h.liveConn() + conn.FailReads(errors.New("socket died")) + h.join() + + mu.Lock() + defer mu.Unlock() + if slices.Contains(states, "backoff") { + t.Fatalf("states %v announced Backoff after Close returned from Disconnected; the Closed edge must win", states) + } +} From cdb79661486e9138222f91204aa984576c8d702e Mon Sep 17 00:00:00 2001 From: Jeremy Daer Date: Sat, 22 Aug 2026 13:05:02 -0700 Subject: [PATCH 28/51] Event feed: a panic strands nothing, and Close wins the last both-ready race MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit All six of this round's findings taken. The three dispatch frame arms — AwaitingConfirmation, Streaming, and the poll-retry wait — now let the Closed edge win the both-ready race Close itself creates: cancellation makes the conforming ReadFrame return its cancellation error, the pump hands it off, and the frame case dispatching it reported the consumer's own Close to Observer.Disconnected as a socket failure, after Close had returned. Rounds red: 17/40 fired Disconnected(context canceled) post-Close. A panic in host code — an observer, the signal handler, a checkpoint store, the consumer's range body, all of which run on the run goroutine — unwound with only the run context cancelled: no deferred path closed the CableConn, and a compliant conn is only required to unblock reads on CLOSE, so under an outer recovery the socket stayed open with its pump parked and a later Connector.Close could not reach it (cancelRun cleared on unwind). runCycle now re-disposes the live attempt inside a recover and re-panics; disposal is idempotent, so paths that already disposed are unharmed, and phase timers held as locals are deliberately not chased — unfired, they fire into nothing. Red: the socket stayed open and the staleness timer stayed armed through a recovered panic. The repair-poll arm gains the family's expired-verdict drain: entering the walk made an already-rendered stale verdict look in-flight — pollPage deferred it into a grace phase, and the poll's page could be accepted first — where transition 25 fires directly from every socket-open state. Rounds red: repair polls issued on an expired socket. Both unauthorized paths forwarded a seam-supplied RetryAfter into the reconnect floor. The SPEC's loop table names `unauthorized` "a kind carrying no retry_after" and row 4 says the rest: "the backoff draw alone governs". Both arms drop the field; red on each showed the 5-minute floor verbatim. The tier-2 driver's redirect leg now runs the SHIPPED per-hop predicate. Fixture 30's description promised a sentinel listener the harness never binds, and the driver classified the scripted 302 by an ad-hoc origin comparison — synthesizing the refusal rather than exercising the decision, and turning an unreducible hostile Location into a SCENARIO error instead of a refusal. redirectRefusalFrom now classifies through checkContinuation (exported to the driver as a test shim), so the decision tier 2 pins is the one that ships. What tier 2 cannot see — an adapter's HTTP client auto-following inside one seam call — is below the poll seam by construction, and pretending otherwise was the finding: the fixture's description now claims exactly what the tier verifies (the refusal decision, the continuation terminal, the closed socket, no further seam calls) and names Layer-1 adapter conformance as the owner of the zero- egress property, where the pending adapter PR's sentinel can actually observe a request. The description edit is the only fixture change, one line, and the fixture gate stays green. --- ...30-continuation-redirect-cross-origin.json | 2 +- go/pkg/basecamp/eventfeed/catchup.go | 37 ++++- go/pkg/basecamp/eventfeed/catchup_test.go | 142 ++++++++++++++++++ go/pkg/basecamp/eventfeed/connector_test.go | 38 +++++ go/pkg/basecamp/eventfeed/export_test.go | 7 + go/pkg/basecamp/eventfeed/loop.go | 34 ++++- go/pkg/basecamp/eventfeed/loop_test.go | 14 ++ .../eventfeed/scenario_conformance_test.go | 25 ++- .../eventfeed/scenario_selftest_test.go | 39 +++++ 9 files changed, 331 insertions(+), 7 deletions(-) diff --git a/conformance/event-feed/fixtures/30-continuation-redirect-cross-origin.json b/conformance/event-feed/fixtures/30-continuation-redirect-cross-origin.json index cca380a8c8..aa7abaf3ee 100644 --- a/conformance/event-feed/fixtures/30-continuation-redirect-cross-origin.json +++ b/conformance/event-feed/fixtures/30-continuation-redirect-cross-origin.json @@ -1,6 +1,6 @@ { "name": "30-continuation-redirect-cross-origin", - "description": "A VALIDATED same-origin `next` answers 302 with a cross-origin Location: the poll seam suppresses automatic redirect-following, and the foreign Location is Terminal(invalid_continuation) with zero egress to the foreign origin. {{NEXT:1}} substitutes same-origin, so the pre-poll validation PASSES and the second poll seam call is made (contrast fixture 26, where no request reaches the URL at all) — the redirect answer is where the per-hop rule bites. The Location host is literal, never substituted, never served: the harness binds it to a sentinel listener whose any-request fails the scenario. Kills follow-cross-origin-redirect.", + "description": "A VALIDATED same-origin `next` answers 302 with a cross-origin Location: the poll seam suppresses automatic redirect-following, and the foreign Location is Terminal(invalid_continuation). {{NEXT:1}} substitutes same-origin, so the pre-poll validation PASSES and the second poll seam call is made (contrast fixture 26, where no request reaches the URL at all) — the redirect answer is where the per-hop rule bites. What tier 2 verifies is the per-hop refusal DECISION (the driver classifies the scripted Location through the shipped predicate) and the loop's response: the continuation terminal, the socket explicitly closed, and no further seam calls. That the adapter's HTTP client makes zero requests to the refused URL is below the poll seam and owned by Layer-1 adapter conformance; the Location host is literal, never substituted, and never served. Kills follow-cross-origin-redirect.", "config": { "position": "{{POS:0}}" }, diff --git a/go/pkg/basecamp/eventfeed/catchup.go b/go/pkg/basecamp/eventfeed/catchup.go index 6b4e2fa8a3..d1f3fc7608 100644 --- a/go/pkg/basecamp/eventfeed/catchup.go +++ b/go/pkg/basecamp/eventfeed/catchup.go @@ -1104,6 +1104,25 @@ func (l *loop) stream(at *attempt) cycleOutcome { l.disposeAttempt(at, repair) return cycleOutcome{kind: outcomeClosed} case <-repair.C(): + // The expired window outranks a ready repair cadence, as it does + // the poll-retry and live-frame arms: entering the walk makes the + // already-rendered stale verdict look in-flight — pollPage defers + // it and grants a grace phase, and the poll's page may even be + // accepted first — where transition 25 fires directly from every + // socket-open state, staleness among its triggers. evaluate + // arbitrates as ever. + select { + case <-staleTimer.C(): + if age, authoritative := at.lc.stale.evaluate(staleGen); authoritative { + l.disposeAttempt(at, repair) + if l.cfg.observer.StaleConnection != nil { + l.cfg.observer.StaleConnection(age) + } + l.observeDisconnected("", errStaleConnection) + return cycleOutcome{kind: outcomeFailed} + } + default: + } // Transition 24 → CatchingUp: one repair walk from the connector's // current position, returning here through Draining. The next // cadence is armed before Streaming is re-announced, for the same @@ -1134,6 +1153,13 @@ func (l *loop) stream(at *attempt) cycleOutcome { if !ok { return l.pumpExited(at, repair) } + // Close outranks a ready item — the cancellation error the pump + // hands off after Close must not be dispatched as a socket + // failure (see awaitConfirmation's frame arm). + if l.runCtx.Err() != nil { + l.disposeAttempt(at, repair) + return cycleOutcome{kind: outcomeClosed} + } // A latched expiry outranks a ready frame. The pump refuses the // reset for a frame received after the window fired // (staleHolder.arm latches instead), so the fired timer and that @@ -1315,7 +1341,11 @@ func (l *loop) recoverPoll(at *attempt, cursor Cursor, err error) (walkStep, cyc Err: err, }}, true } - return walkStep{}, cycleOutcome{kind: outcomeFailed, retryAfter: pe.RetryAfter}, true + // No retryAfter: `unauthorized` is specified to carry none (the + // backoff draw alone governs, bounded by the shared counter), so a + // custom PollSource populating the field must not floor the + // reconnect delay. + return walkStep{}, cycleOutcome{kind: outcomeFailed}, true case PollFilterInvalid: // Transition 20: a configuration error a position reset won't help; // the server's message naming the offending list is preserved. @@ -1417,6 +1447,11 @@ func (l *loop) waitPollRetry(at *attempt, d time.Duration) (cycleOutcome, bool) if !ok { return l.pumpExited(at, t), true } + // Close outranks a ready item (see awaitConfirmation's frame arm). + if l.runCtx.Err() != nil { + l.disposeAttempt(at, t) + return cycleOutcome{kind: outcomeClosed}, true + } if out, done := l.handleLiveFrame(at, t, item, false); done { return out, true } diff --git a/go/pkg/basecamp/eventfeed/catchup_test.go b/go/pkg/basecamp/eventfeed/catchup_test.go index fdd0d3423f..248a996986 100644 --- a/go/pkg/basecamp/eventfeed/catchup_test.go +++ b/go/pkg/basecamp/eventfeed/catchup_test.go @@ -3104,3 +3104,145 @@ func TestLatchedStalenessOutranksAReadyConfirmation(t *testing.T) { t.Errorf("%d/%d rounds confirmed a subscription past a latched staleness expiry; the frame arm must re-evaluate the verdict first (transition 15 before 11)", violations, rounds) } } + +// TestCloseRacingAPumpHandoffTakesTheClosedEdge: Close cancels the run +// context, the conforming ReadFrame returns its cancellation error promptly, +// and the pump hands that error off — so runCtx.Done() and the frame case +// can BOTH be ready at the next select. The frame arm winning must not +// dispatch the cancellation as a socket failure: §23's close() is the +// universal edge, and Observer.Disconnected firing after Close returned +// reports a teardown the consumer already ordered. Rounds-driven off a +// collector parked mid-delivery. +func TestCloseRacingAPumpHandoffTakesTheClosedEdge(t *testing.T) { + const rounds = 40 + violations := 0 + for i := range rounds { + func() { + disconnected := make(chan error, 4) + handedOffErr := make(chan struct{}, 4) + h := newHarness(t, eventfeed.WithObserver(eventfeed.Observer{ + Disconnected: func(_ string, err error) { + select { + case disconnected <- err: + default: + } + }, + })) + h.conn.OnPumpHandedOff(func(isErr bool) { + if isErr { + select { + case handedOffErr <- struct{}{}: + default: + } + } + }) + h.pauseAfter = 1 + h.minter.ScriptTicket(ticket(1)) + h.polls.ScriptPage(eventfeed.PollPage{Position: "pos-1"}) + h.start() + + conn := h.driveToSubscribed() + conn.Serve(frameConfirm(noFilterIdentifier)) + h.awaitStreaming() + conn.Serve(frameMessage(noFilterIdentifier, 101)) + h.waitUntil("the collector to park mid-delivery", func() bool { + return len(h.deliveredIDs()) == 1 + }) + h.conn.Close() //nolint:errcheck // asserted via the observer + select { + case <-handedOffErr: + case <-time.After(watchdog): + t.Fatal("the pump never handed off the cancellation error") + } + h.resume() + h.join() + select { + case err := <-disconnected: + violations++ + t.Logf("round %d: Disconnected(%v) fired after Close returned", i, err) + default: + } + }() + } + if violations != 0 { + t.Errorf("%d/%d rounds dispatched a post-Close pump item as a socket failure; the Closed edge must win the both-ready select", violations, rounds) + } +} + +// TestExpiredStalenessOutranksTheRepairPoll: the repair-poll sibling of the +// poll-retry and live-frame guards. With both `repair-poll` and `staleness` +// fired, taking the repair arm starts a walk whose first Poll makes the +// already-expired window look in-flight — granted a grace phase, and a page +// may even be accepted first — instead of Streaming's transition 25 firing +// directly (SPEC: every socket-open state's failure edge fires directly, +// staleness among its triggers). Rounds-driven off a parked delivery. +func TestExpiredStalenessOutranksTheRepairPoll(t *testing.T) { + const rounds = 40 + violations := 0 + for i := range rounds { + func() { + disconnected := make(chan struct{}, 4) + h := newHarness(t, eventfeed.WithObserver(eventfeed.Observer{ + Disconnected: func(string, error) { + select { + case disconnected <- struct{}{}: + default: + } + }, + })) + h.pauseAfter = 1 + h.minter.ScriptTicket(ticket(1)) + h.polls.ScriptPage(eventfeed.PollPage{Position: "pos-1"}) + h.start() + + conn := h.driveToSubscribed() + conn.Serve(frameConfirm(noFilterIdentifier)) + h.awaitStreaming() + conn.Serve(frameMessage(noFilterIdentifier, 101)) + h.waitUntil("the collector to park mid-delivery", func() bool { + return len(h.deliveredIDs()) == 1 + }) + h.fireTimer(timerStaleness) + h.fireTimer(timerRepairPoll) + h.resume() + + deadline := time.Now().Add(watchdog) + for { + if h.polls.CallCount() >= 2 { + violations++ + t.Logf("round %d: a repair poll was issued on an already-expired socket", i) + return + } + select { + case <-disconnected: + return + default: + } + if time.Now().After(deadline) { + t.Fatal("round undecided: neither a repair poll nor a teardown") + } + time.Sleep(time.Millisecond) + } + }() + } + if violations != 0 { + t.Errorf("%d/%d rounds began a repair walk past an expired staleness window; the repair arm must re-evaluate the verdict first (transition 25 before 24)", violations, rounds) + } +} + +// TestUnauthorizedPollCarriesNoRetryAfterFloor: SPEC pins `unauthorized` as +// "a kind carrying no retry_after; authorization recovery bounded by the +// shared counter" — the backoff draw alone governs. A custom PollSource +// populating the field anyway must not floor the reconnect delay. +func TestUnauthorizedPollCarriesNoRetryAfterFloor(t *testing.T) { + h := newHarness(t) + h.minter.ScriptTicket(ticket(1)) + h.polls.ScriptError(&eventfeed.PollError{Kind: eventfeed.PollUnauthorized, RetryAfter: 5 * time.Minute}) + h.start() + + conn := h.driveToSubscribed() + conn.Serve(frameConfirm(noFilterIdentifier)) + if d := h.fireTimer(timerBackoff); d >= 5*time.Minute { + t.Fatalf("backoff armed for %s: an unauthorized poll's seam-supplied RetryAfter floored the reconnect delay; unauthorized carries no retry_after and the local draw alone governs", d) + } +} diff --git a/go/pkg/basecamp/eventfeed/connector_test.go b/go/pkg/basecamp/eventfeed/connector_test.go index ce593f4ac2..5f26b1ff1f 100644 --- a/go/pkg/basecamp/eventfeed/connector_test.go +++ b/go/pkg/basecamp/eventfeed/connector_test.go @@ -6,6 +6,7 @@ package eventfeed_test import ( "context" "errors" + "runtime" "strings" "sync" "sync/atomic" @@ -1588,3 +1589,40 @@ func TestAcceptedPositionSavesAgainstAContextHonoringStore(t *testing.T) { "the run's values and drop only its cancellation", got) } } + +// TestPanicInHostCodeDisposesTheLiveAttempt: observers, signal handlers, +// checkpoint stores, and the consumer's own range body all run on the run +// goroutine, and any of them can panic. When an outer recovery keeps the +// process alive, the unwound iteration must not strand the live attempt: the +// socket must be closed (a compliant CableConn is only required to unblock +// reads on Close), the pump joined, and the staleness window stopped — while +// the panic itself still propagates to the consumer's recovery. +func TestPanicInHostCodeDisposesTheLiveAttempt(t *testing.T) { + h := newHarness(t, eventfeed.WithObserver(eventfeed.Observer{ + Confirmed: func() { panic("host code panicked") }, + })) + h.minter.ScriptTicket(ticket(1)) + base := runtime.NumGoroutine() + recovered := make(chan any, 1) + go func() { + defer func() { recovered <- recover() }() + for range h.conn.Events(context.Background()) { + } + }() + + conn := h.driveToSubscribed() + conn.Serve(frameConfirm(noFilterIdentifier)) + select { + case r := <-recovered: + if r == nil { + t.Fatal("the panic did not propagate out of the iteration") + } + case <-time.After(watchdog): + t.Fatal("the iteration neither panicked nor returned") + } + if !conn.Closed() { + t.Fatal("the live socket was not disposed during panic unwinding") + } + assertTimers(t, h.clock, map[string]int{}) + assertGoroutinesSettle(t, base) +} diff --git a/go/pkg/basecamp/eventfeed/export_test.go b/go/pkg/basecamp/eventfeed/export_test.go index e0306eb53b..dd847608be 100644 --- a/go/pkg/basecamp/eventfeed/export_test.go +++ b/go/pkg/basecamp/eventfeed/export_test.go @@ -160,3 +160,10 @@ func (c *Connector) OnSubscribeWritten(f func()) { c.hooks.subscribeWritten = f // context, and any proxy for it (a nil cancel func, a closed latch) is exactly // the state a broken Close sets too early. func (c *Connector) OnRunContext(f func(context.Context)) { c.hooks.runContext = f } + +// ExportCheckContinuation exposes the per-hop continuation/redirect predicate +// to the tier-2 driver, which models the Layer-1 adapter and must run the +// SHIPPED rule rather than re-implement the decision. +func ExportCheckContinuation(baseOrigin, pageURL string) *TerminalError { + return checkContinuation(baseOrigin, pageURL) +} diff --git a/go/pkg/basecamp/eventfeed/loop.go b/go/pkg/basecamp/eventfeed/loop.go index 8dcc29ddab..7661f3194c 100644 --- a/go/pkg/basecamp/eventfeed/loop.go +++ b/go/pkg/basecamp/eventfeed/loop.go @@ -793,6 +793,24 @@ func (l *loop) runCycle(delay time.Duration) cycleOutcome { at := &attempt{} at.ctx, at.cancel = context.WithCancel(l.runCtx) defer at.cancel() + // Host code — observers, the signal handler, the checkpoint store, and + // the consumer's own range body — runs on this goroutine, and any of it + // can panic. An outer recovery that keeps the process alive must not + // inherit a stranded attempt: a compliant CableConn is only required to + // unblock reads when CLOSED, so an undisposed socket leaves its pump + // parked forever, and a later Connector.Close cannot reach it (cancelRun + // is cleared on unwind). Disposal is idempotent, so a panic past a path + // that already disposed re-disposes harmlessly; the panic itself + // propagates untouched. Phase timers held as locals are deliberately not + // chased: unfired, they fire into nothing. + defer func() { + if r := recover(); r != nil { + if at.lc != nil { + l.disposeAttempt(at, nil) + } + panic(r) + } + }() // Transitions 1/2 → Minting. No timers are armed here (per-state // invariant: Minting's set is {}); mint cancellation rides the attempt @@ -1002,7 +1020,11 @@ func (l *loop) classifyMintFailure(err error) cycleOutcome { Err: err, }} } - return cycleOutcome{kind: outcomeFailed, retryAfter: me.RetryAfter} + // No retryAfter: SPEC pins `unauthorized` as "a kind carrying no + // retry_after" (row 4: "the backoff draw alone governs"), so a + // seam-supplied value is a contract violation and must not floor + // the reconnect delay. + return cycleOutcome{kind: outcomeFailed} case MintUnrecoverable: return cycleOutcome{kind: outcomeTerminal, term: &TerminalError{Reason: ReasonMintFailed, Err: err}} default: @@ -1058,6 +1080,16 @@ func (l *loop) awaitConfirmation(at *attempt, deadline Timer) cycleOutcome { l.observeDisconnected("", errors.New("event feed frame pump exited")) return cycleOutcome{kind: outcomeFailed} } + // Close outranks a ready item: cancellation makes the conforming + // ReadFrame return its cancellation error, the pump hands it + // off, and both this case and runCtx.Done() are then ready — + // dispatching the item would report the consumer's own Close to + // Observer.Disconnected as a socket failure, after Close + // returned. The universal edge wins the both-ready race. + if l.runCtx.Err() != nil { + l.disposeAttempt(at, deadline) + return cycleOutcome{kind: outcomeClosed} + } // A latched expiry outranks a ready frame, exactly as on the // Streaming frame arm (catchup.go): a frame received after the // window fired did not reset it, and handling it first admits a diff --git a/go/pkg/basecamp/eventfeed/loop_test.go b/go/pkg/basecamp/eventfeed/loop_test.go index 2627e1471b..7cfb3b2f6c 100644 --- a/go/pkg/basecamp/eventfeed/loop_test.go +++ b/go/pkg/basecamp/eventfeed/loop_test.go @@ -2430,3 +2430,17 @@ func TestCloseFromDisconnectedDoesNotAnnounceBackoff(t *testing.T) { t.Fatalf("states %v announced Backoff after Close returned from Disconnected; the Closed edge must win", states) } } + +// TestUnauthorizedMintCarriesNoRetryAfterFloor: row 4 pins it — "unauthorized +// carries no retry_after, so the backoff draw alone governs". A seam that +// populates MintError.RetryAfter on a 401/403 anyway must not floor the +// reconnect delay. +func TestUnauthorizedMintCarriesNoRetryAfterFloor(t *testing.T) { + h := newHarness(t) + h.minter.ScriptError(&eventfeed.MintError{Kind: eventfeed.MintUnauthorized, RetryAfter: 5 * time.Minute}) + h.start() + + if d := h.fireTimer(timerBackoff); d >= 5*time.Minute { + t.Fatalf("backoff armed for %s: an unauthorized mint's seam-supplied RetryAfter floored the reconnect delay; unauthorized carries no retry_after and the local draw alone governs", d) + } +} diff --git a/go/pkg/basecamp/eventfeed/scenario_conformance_test.go b/go/pkg/basecamp/eventfeed/scenario_conformance_test.go index 41f95fc7a1..d4f6ddc4c2 100644 --- a/go/pkg/basecamp/eventfeed/scenario_conformance_test.go +++ b/go/pkg/basecamp/eventfeed/scenario_conformance_test.go @@ -1131,21 +1131,38 @@ func goneOutcome(body json.RawMessage) (pollOutcome, error) { // same-origin/no-downgrade validation is refused inside the call, with the // Location redacted to its origin. No request is ever issued to it — the // driver is the seam, so the foreign origin is unreachable by construction. +// redirectRefusalFrom models the Layer-1 adapter's per-hop redirect rule, +// and the rule it runs is the SHIPPED one: eventfeed.CheckContinuation is +// the per-hop same-origin/no-downgrade validation the adapter is specified +// to apply to a redirect Location ("a redirect Location failing per-hop +// validation inside a poll seam call"). An earlier revision re-implemented +// the decision as an ad-hoc origin comparison, which diverged at the edges — +// a Location that cannot be reduced to an origin at all failed the SCENARIO, +// as if the fixture were malformed rather than the peer hostile. What tier 2 +// verifies through this is the refusal DECISION and the loop's response to +// it; that the adapter's HTTP client makes zero requests to the refused URL +// is a below-the-seam property, owned by Layer-1 adapter conformance. func (d *driver) redirectRefusalFrom(headers map[string]string) (pollOutcome, error) { location := headers["Location"] if location == "" { return pollOutcome{}, errors.New("a 302 poll response needs a Location header") } - origin, err := eventfeed.CanonicalOrigin(location) + base, err := eventfeed.CanonicalOrigin(d.h.apiOrigin) if err != nil { - return pollOutcome{}, fmt.Errorf("302 Location: %w", err) + return pollOutcome{}, fmt.Errorf("canonicalizing the API origin: %w", err) } - if origin == d.h.apiOrigin { + if terr := eventfeed.ExportCheckContinuation(base, location); terr == nil { return pollOutcome{}, errors.New("a same-origin 302 is not modeled: the seam would follow it, and no fixture scripts that hop") } + // Redacted to its origin best-effort, exactly as the seam contract + // requires of the adapter; an unreducible Location carries none. + locationOrigin := "" + if origin, oerr := eventfeed.CanonicalOrigin(location); oerr == nil { + locationOrigin = origin + } return pollOutcome{err: &eventfeed.PollError{ Kind: eventfeed.PollRedirectRefused, - LocationOrigin: origin, + LocationOrigin: locationOrigin, Err: errors.New("poll refused a redirect"), }}, nil } diff --git a/go/pkg/basecamp/eventfeed/scenario_selftest_test.go b/go/pkg/basecamp/eventfeed/scenario_selftest_test.go index 0e9bc7d278..3d6b52dee3 100644 --- a/go/pkg/basecamp/eventfeed/scenario_selftest_test.go +++ b/go/pkg/basecamp/eventfeed/scenario_selftest_test.go @@ -771,3 +771,42 @@ func TestDriverRetryAfterParseIsTheSpecAlgorithm(t *testing.T) { } } } + +// TestDriverRedirectRefusalUsesTheRealPredicate: the driver models the +// Layer-1 poll adapter, whose per-hop redirect rule IS checkContinuation — +// "a redirect Location failing per-hop validation inside a poll seam call +// (Continuation and Resume URL Validation)". Classifying by an ad-hoc origin +// comparison instead re-implements the predicate, and diverges at its edges: +// a Location that cannot be reduced to an origin at all ("https://", a +// relative path) is a REFUSAL under the real rule — the seam refuses the +// hop, zero requests to the failing URL — where the ad-hoc parse turned it +// into a scenario error, as if the fixture were malformed rather than the +// peer hostile. +func TestDriverRedirectRefusalUsesTheRealPredicate(t *testing.T) { + d := &driver{h: newScenarioHarness()} + defer d.h.close() + + for name, location := range map[string]string{ + "unreducible location": "https://", + "relative location": "/events.json", + } { + po, err := d.redirectRefusalFrom(map[string]string{"Location": location}) + if err != nil { + t.Fatalf("%s: a hostile Location must classify as a refusal, not fail the scenario: %v", name, err) + } + var pe *eventfeed.PollError + if !errors.As(po.err, &pe) || pe.Kind != eventfeed.PollRedirectRefused { + t.Fatalf("%s = %+v, want PollRedirectRefused", name, po.err) + } + } + // And the refusal decision agrees with the predicate on the ordinary + // cross-origin case. + po, err := d.redirectRefusalFrom(map[string]string{"Location": "https://attacker.example.com/events.json"}) + if err != nil { + t.Fatalf("cross-origin Location: %v", err) + } + var pe *eventfeed.PollError + if !errors.As(po.err, &pe) || pe.Kind != eventfeed.PollRedirectRefused { + t.Fatalf("cross-origin Location = %+v, want PollRedirectRefused", po.err) + } +} From 4c2f7d956eab2f1adf080f73fa02281b44c0f2ff Mon Sep 17 00:00:00 2001 From: Jeremy Daer Date: Mon, 24 Aug 2026 01:16:35 -0700 Subject: [PATCH 29/51] Event feed: every deferral gets the latch, not just the one that fired MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Copilot's exact-head review carried one suppressed finding, and it is a real residual of the round-5 grace fix: the latched, deadline-remainder wake was armed only when the DEFERRED OUTCOME was a staleness expiry. A deferred disconnect or invalid frame left the phase's wakes riding the ordinary staleness window — which the pump still re-arms on every inbound frame — so a peer whose last frame landed just short of the deadline pushed the only remaining wake to frameTime + staleAfter: the rearm's own wake ran the deadline check a hair early, then nothing fired until almost a full window past the deadline, and the abandoned poll outlived the one-window phase §23 measures from the deferral. The SPEC's immunity is unconditional — "a frame arriving inside the phase re-arms staleness in the ordinary way and MUST NOT move the deadline" — and a deadline nothing wakes is moved in effect. The existing TestGracePhaseIsImmuneToFrameResets could not see it: its staleDeferralHarness drives a silent socket, so the deferral is always the stale kind, the one path that already had the latch. pollPage now arms graceWake at the deferral for EVERY deferred kind. The latch is what buys both published immunities — pump resets are refused, a latched window cannot be suspended — and the deferred outcome is untouched: the walk still dispatches the PARKED item, never a substituted staleness verdict (the new test pins that too — the teardown reports the disconnect's redacted "other", not staleness's empty reason). With the wake armed at the deferral, awaitSupersededPoll's firing arm collapses to one case: every firing is a wake re-armed for the remainder, the two-regime split and its evaluate call are gone, and the `stale` parameter with them. Red proof, the frame-deferral sibling of the existing immunity test: a deferred remote disconnect, the peer's last ping 1ms short of the deadline, then silence — pre-fix the wait sat parked past the deadline until the watchdog; green tears down at the deadline exactly, dispatching the parked disconnect, with Backoff arming after. --- go/pkg/basecamp/eventfeed/catchup.go | 64 ++++++++---------- go/pkg/basecamp/eventfeed/catchup_test.go | 79 +++++++++++++++++++++++ 2 files changed, 105 insertions(+), 38 deletions(-) diff --git a/go/pkg/basecamp/eventfeed/catchup.go b/go/pkg/basecamp/eventfeed/catchup.go index d1f3fc7608..5647a8d0a0 100644 --- a/go/pkg/basecamp/eventfeed/catchup.go +++ b/go/pkg/basecamp/eventfeed/catchup.go @@ -481,7 +481,6 @@ func (l *loop) pollPage(at *attempt, cursor Cursor) pollAttempt { }() for { staleTimer, staleGen := at.lc.stale.current() - stale := false select { case r := <-done: return pollAttempt{page: r.page, err: r.err} @@ -498,7 +497,7 @@ func (l *loop) pollPage(at *attempt, cursor Cursor) pollAttempt { // window evaluate just re-armed. continue } - l.deferred, stale = &deferredFrame{stale: true, age: age}, true + l.deferred = &deferredFrame{stale: true, age: age} case item, ok := <-at.lc.frames: if !ok { l.deferred = &deferredFrame{closed: true} @@ -519,20 +518,24 @@ func (l *loop) pollPage(at *attempt, cursor Cursor) pollAttempt { // read before the hook fires so nothing observing the deferral can // race the deadline into existence behind it. deadline := l.cfg.clock.Now().Add(l.cfg.staleAfter) - if stale { - // The firing this branch consumed was the wait's own wake source, - // and an authoritative expiry latches — so without this there is - // nothing left to wake the grace phase at all, and the bounded wait - // becomes unbounded again. Armed BEFORE the hook, so a test that - // rendezvouses on the deferral cannot advance past a wake that is - // not yet armed. The distance is the remainder to the deadline — - // here, at the deferral itself, the full window. - at.lc.stale.graceWake(l.cfg.staleAfter) - } + // The grace wake is armed HERE, at the deferral, for EVERY deferred + // kind — not only the stale one. graceWake's latch is what buys both + // of the phase's immunities: it refuses every later pump reset, so a + // peer whose last frame lands just short of the deadline cannot push + // the only remaining wake to frameTime + staleAfter (a deferred + // disconnect used to ride the ordinary window and outlive the phase + // by almost another full one), and a latched window cannot be + // suspended. The deferred outcome is untouched — the walk still + // dispatches the PARKED item, never a substituted staleness verdict. + // Armed BEFORE the hook, so a test that rendezvouses on the deferral + // cannot advance past a wake that is not yet armed; the distance is + // the remainder to the deadline — here, at the deferral itself, the + // full window. + at.lc.stale.graceWake(l.cfg.staleAfter) if l.hooks.frameDeferred != nil { l.hooks.frameDeferred() } - return l.awaitSupersededPoll(at, done, deadline, stale) + return l.awaitSupersededPoll(at, done, deadline) } } @@ -602,9 +605,9 @@ func (l *loop) pollPage(at *attempt, cursor Cursor) pollAttempt { // window that a peer CAN push, but the post-select check runs on every wake, // and a re-armed window under a blocked pump still delivers one within a window // — so a peer that keeps sending can move a wake, never the verdict. -func (l *loop) awaitSupersededPoll(at *attempt, done <-chan pollResult, deadline time.Time, stale bool) pollAttempt { +func (l *loop) awaitSupersededPoll(at *attempt, done <-chan pollResult, deadline time.Time) pollAttempt { for { - staleTimer, staleGen := at.lc.stale.current() + staleTimer, _ := at.lc.stale.current() select { case r := <-done: return pollAttempt{page: r.page, err: r.err} @@ -612,29 +615,14 @@ func (l *loop) awaitSupersededPoll(at *attempt, done <-chan pollResult, deadline return pollAttempt{superseded: true} case <-at.lc.stale.rearmed(): case <-staleTimer.C(): - if stale { - // The verdict is already rendered and parked; this firing is a - // wake and nothing else, so nothing re-evaluates it. Re-arming - // keeps a wake in hand for the next turn — for the REMAINDER - // to the deadline, so the last wake lands at the deadline - // itself rather than a window past it. - at.lc.stale.graceWake(deadline.Sub(l.cfg.clock.Now())) - } else if _, ok := at.lc.stale.evaluate(staleGen); ok { - // An authoritative expiry during another outcome's grace - // phase is ALSO only a wake. The deferral slot already holds - // the socket's verdict — a read error or a disconnect frame, - // which does not reset staleness — so the pre-existing - // window, mostly spent by the time that outcome landed, can - // fire here almost immediately, and abandoning the call on it - // granted the poll whatever sliver of the old window remained - // instead of the phase §23 measures from the deferral. The - // expiry adds nothing the deferred outcome's teardown will - // not do; graceWake latches it and keeps the next wake in - // hand — armed for the remainder to the deadline, which is - // what decides. A firing that lands late in the phase must - // not push the only remaining wake a full window past it. - at.lc.stale.graceWake(deadline.Sub(l.cfg.clock.Now())) - } + // The wait's own latched wake, armed at the deferral for every + // deferred kind — or, in a narrow race, the old window's final + // firing before the latch replaced it. Either way the verdict is + // already parked in the deferral slot, so a firing here is a + // wake and nothing else; re-arming for the REMAINDER keeps the + // next wake in hand and lands the last one at the deadline + // itself, never a window past it. The deadline below decides. + at.lc.stale.graceWake(deadline.Sub(l.cfg.clock.Now())) } if l.hooks.supersededWake != nil { l.hooks.supersededWake() diff --git a/go/pkg/basecamp/eventfeed/catchup_test.go b/go/pkg/basecamp/eventfeed/catchup_test.go index 248a996986..77c014a886 100644 --- a/go/pkg/basecamp/eventfeed/catchup_test.go +++ b/go/pkg/basecamp/eventfeed/catchup_test.go @@ -3246,3 +3246,82 @@ func TestUnauthorizedPollCarriesNoRetryAfterFloor(t *testing.T) { t.Fatalf("backoff armed for %s: an unauthorized poll's seam-supplied RetryAfter floored the reconnect delay; unauthorized carries no retry_after and the local draw alone governs", d) } } + +// TestFrameDeferralGracePhaseIsImmuneToFrameResets is the frame-deferral +// sibling of TestGracePhaseIsImmuneToFrameResets, which covers only the +// stale deferral — there, graceWake's latch is armed at the deferral and +// refuses every later reset. A deferred DISCONNECT (or invalid frame) had no +// latch: its wakes rode the ordinary staleness window, and a peer whose last +// frame lands just short of the deadline re-armed that window a full +// staleAfter out — the rearm's wake ran the deadline check a hair early, +// and the next wake did not exist until frameTime + staleAfter, so the +// abandoned poll outlived the one-window grace phase by almost another +// window. SPEC's immunity is unconditional: "a frame arriving inside the +// phase re-arms staleness in the ordinary way and MUST NOT move the +// deadline" — and a deadline nothing wakes is moved in effect. The deferred +// outcome itself must stay the dispatched verdict: the teardown reports the +// parked disconnect, not staleness. +func TestFrameDeferralGracePhaseIsImmuneToFrameResets(t *testing.T) { + deferred := make(chan struct{}, 1) + wakes := make(chan struct{}, 8) + disconnected := make(chan string, 4) + h := newHarness(t, eventfeed.WithObserver(eventfeed.Observer{ + Disconnected: func(reason string, _ error) { + select { + case disconnected <- reason: + default: + } + }, + })) + h.conn.OnFrameDeferred(func() { + select { + case deferred <- struct{}{}: + default: + } + }) + h.conn.OnSupersededWake(func() { + select { + case wakes <- struct{}{}: + default: + } + }) + h.minter.ScriptTicket(ticket(1)) + h.polls.StallNext() + h.start() + + conn := h.driveToSubscribed() + conn.Serve(frameConfirm(noFilterIdentifier)) + h.waitUntil("entry poll in flight", func() bool { return h.polls.CallCount() == 1 }) + // A raw disconnect lands mid-poll and is deferred: the grace deadline is + // Now + staleAfter, and the socket stays open with the pump reading. + conn.Serve(frameDisconnect("remote", true)) + select { + case <-deferred: + case <-time.After(watchdog): + t.Fatal("the disconnect frame was not deferred") + } + + // The peer's LAST frame lands 1ms short of the deadline. + h.clock.Advance(7499 * time.Millisecond) + conn.Serve(framePing()) + h.waitUntil("the ping to reach the pump", func() bool { return conn.Pending() == 0 }) + + // One more millisecond reaches the deadline exactly; the wait must end + // there and dispatch the PARKED disconnect — not sit until the re-armed + // window fires a full staleAfter after the ping. + h.clock.Advance(time.Millisecond) + select { + case reason := <-disconnected: + // The parked disconnect is the dispatched verdict. Its reason rides + // the closed observer vocabulary — "remote" is not one of the two + // acted-on reasons named exactly, so it renders as "other" — while a + // substituted staleness teardown would report an EMPTY reason (with + // errStaleConnection as the cause). + if reason != "other" { + t.Fatalf("teardown reported reason %q, want %q — the parked disconnect's redacted spelling, not a substituted staleness verdict", reason, "other") + } + case <-time.After(watchdog): + t.Fatal("the frame-deferral grace phase was extended past its deadline by an ordinary staleness reset: the only remaining wake sits at frameTime + staleAfter, almost a full window past the deadline") + } + h.awaitTimer(timerBackoff) +} From 97c600558cd86493680e3e01f192be262fc16cee Mon Sep 17 00:00:00 2001 From: Jeremy Daer Date: Mon, 24 Aug 2026 10:04:06 -0700 Subject: [PATCH 30/51] Event feed: the ring the buffer needed, the parser parity it claimed, and two honest declines MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Five findings; three taken, two declined with the mechanism named — and one of the takes ships with a no-red account rather than a proof, stated plainly. The subscribe write's result arm gains the family's Closed-edge check: cancellation surfaces through a conforming WriteFrame as its own error, so `written` and runCtx.Done() can be ready together and the result arm dispatched the consumer's own Close to Observer.Disconnected. No red proof discriminates this one, and two attempts taught why. The written channel's readiness is causally downstream of runCtx's, so a machine PARKED in the select always takes the Closed arm first — the race lives only in the spawn-to-select entry window, which no hook can hold open. A staged version that raced a genuine socket failure against Close fired on nearly every round on BOTH sides of the fix, because a real failure dispatched before the cancel lands is legitimately Disconnected and no assertion can tell the orderings apart. The check is kept because the family closed every sibling arm and the entry window is real; the terminal-claim fix set the precedent for saying so instead of shipping a test that proves nothing. Declined: the pump-exit (!ok) racing runCtx.Done. The interleaving is structurally unreachable: handOff's FIRST send is non-blocking, so with queue room the error item always lands and close-without-item requires a full queue — whose drained items hit the ready-item Closed check first, one round's fix ago. Forty staged rounds produced zero occurrences; the test was dropped rather than kept as decoration. Declined: redacting the generated error out of Terminal(poll_failed). The SPEC pins the opposite twice — the out-of-inventory edge list and the terminal-reason table both say poll_failed is "passed through with the generated error attached" — and the URL it may render is the VALIDATED same-origin continuation: the consumer's own configured origin and a feed cursor, carrying no credential (the bearer rides the header; tickets never touch the poll lane). The redirect arm redacts a different trust class — its URL is the hostile Location that FAILED validation. The observer surface is already closed-vocabulary either way. The live buffer is now a head-indexed ring. The finding's magnitude was wrong — reslice-and-append amortizes to O(1), not an O(capacity) copy per event — but its direction was right: each growth step copied the full backing and transiently retained a SECOND buffer's worth of payload, the exact class add's zeroing exists to prevent. The store still grows lazily while filling, wraps only at capacity, zeroes vacated slots, and exposes logical order through a snapshot the hand-off boundary now uses. Red: a full window of sustained overflow made 10004 allocations against the 10000 the dropped-ids slices account for; green makes exactly 10000, and the eviction/zeroing/order pins all hold. The driver's Retry-After parse now matches the SDK's parseRetryAfter exactly: RFC 9110's 1*DIGIT checked before ParseInt (which honours a sign Atoi also accepted — a schema-valid "+5" classified as throttled), int64 so the verdict cannot vary with the platform's int width, digits beyond int64 malformed, and a representable value clamped to the portable MaxInt32-seconds ceiling instead of overflowing the duration multiply into garbage. Red: "+5" → throttled(5s); green: the sign, width, overflow and saturation rows all land where client.go's parser puts them. --- go/pkg/basecamp/eventfeed/buffer_test.go | 57 +++++++++--- go/pkg/basecamp/eventfeed/export_test.go | 3 +- go/pkg/basecamp/eventfeed/loop.go | 88 ++++++++++++++----- .../eventfeed/scenario_conformance_test.go | 30 ++++++- .../eventfeed/scenario_selftest_test.go | 34 +++++++ 5 files changed, 176 insertions(+), 36 deletions(-) diff --git a/go/pkg/basecamp/eventfeed/buffer_test.go b/go/pkg/basecamp/eventfeed/buffer_test.go index 6928536261..72b310bd9b 100644 --- a/go/pkg/basecamp/eventfeed/buffer_test.go +++ b/go/pkg/basecamp/eventfeed/buffer_test.go @@ -15,22 +15,59 @@ func TestLiveBufferAddClearsEvictedSlots(t *testing.T) { b := newLiveBuffer(2, nil) b.add(Event{ID: 1, Kind: "message", EventType: "message.created"}) b.add(Event{ID: 2, Kind: "message", EventType: "message.created"}) - // Captured before the eviction: base still spans the backing array the - // live slice is resliced from, so base[0] IS the vacated slot. - base := b.events dropped := b.add(Event{ID: 3, Kind: "message", EventType: "message.created"}) if len(dropped) != 1 || dropped[0] != 1 { t.Fatalf("dropped = %v, want [1]", dropped) } - if base[0] != (Event{}) { - t.Errorf("evicted slot = %+v, want the zero Event — a bare reslice keeps its payload reachable", base[0]) + // Under the ring the evicted slot is zeroed and then reused by the very + // insert that evicted it, so the hazard this test was born for — a + // vacated slot pinning its payload in the backing — cannot arise on add + // at all. The pin survives as a scan: NO physical slot may retain the + // evicted event. + for i, slot := range b.events { + if slot.ID == 1 { + t.Errorf("physical slot %d still holds the evicted event %+v", i, slot) + } } - // The eviction itself is unchanged. - if got := len(b.events); got != 2 { - t.Fatalf("occupancy = %d, want 2", got) + if b.size != 2 { + t.Fatalf("occupancy = %d, want 2", b.size) } - if b.events[0].ID != 2 || b.events[1].ID != 3 { - t.Fatalf("buffer = %d,%d, want 2,3", b.events[0].ID, b.events[1].ID) + // Logical order survives the wrap. + ev, ok := b.shift() + if !ok || ev.ID != 2 { + t.Fatalf("first shift = %+v (%t), want id 2", ev, ok) + } + ev, ok = b.shift() + if !ok || ev.ID != 3 { + t.Fatalf("second shift = %+v (%t), want id 3", ev, ok) + } +} + +func TestSustainedOverflowRetainsTheBacking(t *testing.T) { + // At capacity, the reslice-and-append shape burns one slot of slice + // capacity per admit, so sustained overflow periodically reallocates and + // copies the entire backing — a full-capacity copy plus a transient + // SECOND buffer's worth of retained payload at every growth step, the + // exact retention class add's zeroing exists to prevent. (Not, as first + // reported, an O(capacity) copy on EVERY admit — the copies amortize — + // but the spikes and the doubled retention are real.) A full capacity's + // worth of sustained overflow must allocate nothing beyond the one + // dropped-ids slice each admit legitimately makes: any backing + // reallocation shows up as extra allocations here. + const capacity = 10000 + b := newLiveBuffer(capacity, nil) + for i := range capacity { + b.add(Event{ID: int64(i)}) + } + id := int64(capacity) + allocs := testing.AllocsPerRun(1, func() { + for range capacity { + b.add(Event{ID: id}) + id++ + } + }) + if allocs > capacity { + t.Fatalf("a full window of sustained overflow made %.0f allocations, want at most %d (one dropped-ids slice per admit): the backing was reallocated instead of retained", allocs, capacity) } } diff --git a/go/pkg/basecamp/eventfeed/export_test.go b/go/pkg/basecamp/eventfeed/export_test.go index dd847608be..4d427ea311 100644 --- a/go/pkg/basecamp/eventfeed/export_test.go +++ b/go/pkg/basecamp/eventfeed/export_test.go @@ -3,7 +3,6 @@ package eventfeed import ( "context" "errors" - "slices" "time" ) @@ -30,7 +29,7 @@ func (c *Connector) OnCatchUpEntered(f func(CatchUpBoundary)) { f(CatchUpBoundary{ Entry: h.entry, PresentClass: h.presentClass, - Buffered: slices.Clone(h.buffer.events), + Buffered: h.buffer.snapshot(), }) } } diff --git a/go/pkg/basecamp/eventfeed/loop.go b/go/pkg/basecamp/eventfeed/loop.go index 7661f3194c..8fa5b6dadf 100644 --- a/go/pkg/basecamp/eventfeed/loop.go +++ b/go/pkg/basecamp/eventfeed/loop.go @@ -480,7 +480,20 @@ func (h *staleHolder) stop() { // denominated in events and every dropped entry has an id. type liveBuffer struct { capacity int - events []Event + // events is the physical store, a head-indexed ring once it has grown to + // capacity. It grows by append while filling (an idle consumer's buffer + // stays small) and is never reallocated after that: the reslice-and- + // append shape this replaced burned one slot of slice capacity per + // overflow admit, so sustained overflow periodically reallocated and + // copied the whole backing — an O(capacity) copy spike, and a transient + // SECOND buffer's worth of retained payload at each growth step, the + // exact retention class add's zeroing exists to prevent. Entries live at + // [head, head+size) modulo len(events); a wrap can exist only once the + // store has reached capacity, because drops happen only at capacity and + // pre-capacity inserts always land at or before len(events). + events []Event + head int + size int // onChange reports the buffer's occupancy after every change. Test-only // (testHooks.bufferOccupancy); nil in production. onChange func(int) @@ -491,22 +504,30 @@ func newLiveBuffer(capacity int, onChange func(int)) *liveBuffer { } // add admits ev, returning the ids of any events dropped to make room — -// oldest first. +// oldest first. Vacated and evicted slots are zeroed for shift's reason: a +// slot that no longer counts toward occupancy must not keep its event's +// strings reachable — under sustained overflow that is a second buffer's +// worth of payload held beyond the ceiling §23 publishes. func (b *liveBuffer) add(ev Event) []int64 { var dropped []int64 - for len(b.events) >= b.capacity { - dropped = append(dropped, b.events[0].ID) - // Zeroed before the reslice, for shift's reason: the reslice removes - // the evicted event logically, but the slice that results still - // points into the same backing array, whose prefix keeps that - // event's strings reachable until a later reallocation. Under - // sustained overflow that is a second buffer's worth of payload held - // beyond the ceiling §23 publishes — retained by events that no - // longer count toward occupancy. - b.events[0] = Event{} - b.events = b.events[1:] - } - b.events = append(b.events, ev) + for b.size >= b.capacity { + dropped = append(dropped, b.events[b.head].ID) + b.events[b.head] = Event{} + b.head = (b.head + 1) % len(b.events) + b.size-- + } + switch pos := b.head + b.size; { + case pos < len(b.events): + b.events[pos] = ev + case pos-len(b.events) < b.head: + // Wrapped: the store is at capacity and the tail circles back below + // the head. + b.events[pos-len(b.events)] = ev + default: + // Still filling: pos == len(events) < capacity, so the store grows. + b.events = append(b.events, ev) + } + b.size++ b.changed() return dropped } @@ -526,21 +547,32 @@ func (b *liveBuffer) add(ev Event) []int64 { // during a drain are precisely the taken ones), which is this method with a // second container in front of it. func (b *liveBuffer) shift() (Event, bool) { - if len(b.events) == 0 { + if b.size == 0 { return Event{}, false } - ev := b.events[0] - // Re-slicing alone would pin the whole backing array through the - // drain; zeroing the vacated slot lets it go. - b.events[0] = Event{} - b.events = b.events[1:] + ev := b.events[b.head] + // Zeroing the vacated slot is what lets the payload go while the ring + // retains its backing. + b.events[b.head] = Event{} + b.head = (b.head + 1) % len(b.events) + b.size-- b.changed() return ev, true } +// snapshot returns the buffered events in logical, oldest-first order — +// the ring's physical layout is an implementation detail no reader may see. +func (b *liveBuffer) snapshot() []Event { + out := make([]Event, 0, b.size) + for i := range b.size { + out = append(out, b.events[(b.head+i)%len(b.events)]) + } + return out +} + func (b *liveBuffer) changed() { if b.onChange != nil { - b.onChange(len(b.events)) + b.onChange(b.size) } } @@ -1152,6 +1184,18 @@ func (l *loop) writeSubscribe(at *attempt, deadline *Timer) (cycleOutcome, bool) staleTimer, staleGen := at.lc.stale.current() select { case werr := <-written: + // Close outranks the write's result: cancellation surfaces + // through a conforming WriteFrame as its own error, so `written` + // and runCtx.Done() can be ready together — and dispatching the + // result reported the consumer's own Close to + // Observer.Disconnected as a socket failure, after Close + // returned. The Closed edge wins the both-ready race, for a + // clean completion exactly as for an error: either is post-Close + // work. + if l.runCtx.Err() != nil { + l.disposeAttempt(at, *deadline) + return cycleOutcome{kind: outcomeClosed}, true + } if werr != nil { l.disposeAttempt(at, *deadline) l.observeDisconnected("", werr) diff --git a/go/pkg/basecamp/eventfeed/scenario_conformance_test.go b/go/pkg/basecamp/eventfeed/scenario_conformance_test.go index d4f6ddc4c2..4e7ab18e0c 100644 --- a/go/pkg/basecamp/eventfeed/scenario_conformance_test.go +++ b/go/pkg/basecamp/eventfeed/scenario_conformance_test.go @@ -22,6 +22,7 @@ import ( "errors" "fmt" "maps" + "math" "net/http" "os" "path/filepath" @@ -1181,8 +1182,19 @@ func (d *driver) retryAfterFrom(headers map[string]string) (value time.Duration, if !ok { return 0, false } - if seconds, err := strconv.Atoi(raw); err == nil { - if seconds > 0 { + // The delta-seconds form is RFC 9110's 1*DIGIT — no sign, no space — so + // the digits are checked before ParseInt, which would honour a leading + // "+" or "-". Parsed through int64 so the verdict cannot vary with the + // platform's int width (Atoi's range is int's), with digits beyond int64 + // malformed, and a representable value clamped to the same portable + // MaxInt32-seconds ceiling the SDK's parseRetryAfter uses — the naive + // duration multiply would overflow into garbage for anything past + // ~292 years. + if isDigits(raw) { + if seconds, err := strconv.ParseInt(raw, 10, 64); err == nil && seconds > 0 { + if seconds > math.MaxInt32 { + seconds = math.MaxInt32 + } return time.Duration(seconds) * time.Second, true } return 0, false @@ -1204,6 +1216,20 @@ func (d *driver) retryAfterFrom(headers map[string]string) (value time.Duration, // --- server frames ------------------------------------------------------- +// isDigits reports RFC 9110's 1*DIGIT and nothing else, mirroring the SDK +// parser's isDelaySeconds (go/pkg/basecamp/client.go). +func isDigits(value string) bool { + if value == "" { + return false + } + for _, r := range value { + if r < '0' || r > '9' { + return false + } + } + return true +} + // serveFrameBytes renders one scripted server frame. confirm/reject/message // auto-echo the captured subscribe identifier unless the fixture overrides it. func serveFrameBytes(step *serveStep, identifier string) ([]byte, error) { diff --git a/go/pkg/basecamp/eventfeed/scenario_selftest_test.go b/go/pkg/basecamp/eventfeed/scenario_selftest_test.go index 3d6b52dee3..330c018a13 100644 --- a/go/pkg/basecamp/eventfeed/scenario_selftest_test.go +++ b/go/pkg/basecamp/eventfeed/scenario_selftest_test.go @@ -13,6 +13,7 @@ import ( "encoding/json" "errors" "fmt" + "math" "net/http" "os" "path/filepath" @@ -810,3 +811,36 @@ func TestDriverRedirectRefusalUsesTheRealPredicate(t *testing.T) { t.Fatalf("cross-origin Location = %+v, want PollRedirectRefused", po.err) } } + +// TestDriverRetryAfterParseMatchesTheSDKParser: the SDK's parseRetryAfter +// (go/pkg/basecamp/client.go) pins the delta-seconds form to RFC 9110's +// 1*DIGIT — no sign — parses through int64 so the verdict cannot vary with +// the platform's int width, and clamps a representable-but-unschedulable +// value to the portable MaxInt32-seconds ceiling. The driver models the same +// §6 algorithm and must match: a signed "+5" is not a delay at all, digits +// too large for int64 are malformed, and a huge representable value +// saturates instead of overflowing the duration multiply into garbage. +func TestDriverRetryAfterParseMatchesTheSDKParser(t *testing.T) { + d := &driver{h: newScenarioHarness()} + defer d.h.close() + var me *eventfeed.MintError + + // Signed values are not 1*DIGIT: undefined, never throttled. + for _, header := range []string{"+5", "-5", " 5", "5 "} { + out := d.mintOutcomeFrom(mintRespond{Status: intp(429), Headers: map[string]string{"Retry-After": header}}) + if !errors.As(out.err, &me) || me.Kind != eventfeed.MintTransient || me.RetryAfter != 0 { + t.Fatalf("Retry-After %q = %+v, want MintTransient with no value (not 1*DIGIT)", header, out.err) + } + } + // Digits too large for int64 are malformed — undefined on every platform. + out := d.mintOutcomeFrom(mintRespond{Status: intp(429), Headers: map[string]string{"Retry-After": "18446744073709551616"}}) + if !errors.As(out.err, &me) || me.Kind != eventfeed.MintTransient || me.RetryAfter != 0 { + t.Fatalf("over-int64 digits = %+v, want MintTransient (malformed)", out.err) + } + // Representable but unschedulable saturates at the SDK's portable + // ceiling; the naive duration multiply would overflow into garbage. + out = d.mintOutcomeFrom(mintRespond{Status: intp(429), Headers: map[string]string{"Retry-After": "99999999999"}}) + if !errors.As(out.err, &me) || me.Kind != eventfeed.MintThrottled || me.RetryAfter != time.Duration(math.MaxInt32)*time.Second { + t.Fatalf("huge representable value = %+v (retryAfter %v), want MintThrottled clamped to MaxInt32 seconds", out.err, me.RetryAfter) + } +} From cf47163124f588630f2302d39e8d98d300d07337 Mon Sep 17 00:00:00 2001 From: Jeremy Daer Date: Mon, 24 Aug 2026 19:26:34 -0700 Subject: [PATCH 31/51] Event feed: the wedge's hand-off signal aliased the fatal's MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Race Detection job on the conformance-driver branch failed TestProtocolFatalBehindADeferredRecoverableOutranksIt/page_boundary with mint_failed where protocol_fatal must win — while the same loop code passed this branch's own CI, and the driver branch's eventfeed diff is comment-only. The defect is the TEST's rendezvous, latent since the wedge was added, and a loaded two-core runner is what finally scheduled it. The choreography serves the wedge ping without awaiting it, drains the hand-off signal channel, serves the fatal, and waits for ONE signal. When the ping's hand-off lands after the drain, that one signal is the PING's: the fatal is still between the pump's read and its hand-off — where §23's normative boundary says no scan need find it — when the poll returns. The entry cut spends its capacity-1 budget on the ping, the probe scans a queue the fatal has not reached, the deferred recoverable's teardown wins, and the reconnect lands on the terminally scripted second mint: mint_failed, exactly the CI signature. The loop is correct; the test counted one signal for two frames. The rendezvous now owes one awaited signal per frame served after the drained baseline (the deferral rendezvous proves every earlier signal had landed), so the fatal is handed off — the state the guarantee is stated over — before the poll is allowed to return. The CI failure is the red capture; the interleaving cannot be forced locally without a hook into the pump's hand-off, and 20 through 100 stress runs under GOMAXPROCS=2 never hit it here. Post-fix: 100 race runs green under the same constraints. --- go/pkg/basecamp/eventfeed/catchup_test.go | 35 +++++++++++++++++------ 1 file changed, 27 insertions(+), 8 deletions(-) diff --git a/go/pkg/basecamp/eventfeed/catchup_test.go b/go/pkg/basecamp/eventfeed/catchup_test.go index 77c014a886..552470ad5e 100644 --- a/go/pkg/basecamp/eventfeed/catchup_test.go +++ b/go/pkg/basecamp/eventfeed/catchup_test.go @@ -1745,18 +1745,37 @@ func TestProtocolFatalBehindADeferredRecoverableOutranksIt(t *testing.T) { t.Error("the recoverable disconnect was never deferred") return } + // The baseline is drained BEFORE anything else is served: + // every earlier frame's hand-off signal has landed (the + // deferral rendezvous above proves the recoverable was + // consumed, which is downstream of its hand-off), so from + // here each serve owes exactly one signal. + drain(queued) + frames := 0 if tc.wedge != nil { tc.wedge(conn) + frames++ } - // Fatal LAST, and only once the pump has handed it off: - // the guarantee is about a frame the pump has READ, so a - // test that raced the read would prove nothing. - drain(queued) + // Fatal LAST, and only once the pump has handed EVERY + // served frame off — the wedge's included: the guarantee + // is about a frame the pump has handed over, and a single + // awaited signal could be the WEDGE's, leaving the fatal + // still between the pump's read and its hand-off, where + // SPEC's normative boundary says no scan need find it. + // That aliasing is exactly the interleaving a loaded CI + // runner produced: the entry cut then spent its budget on + // the wedge, the probe scanned a queue the fatal had not + // reached, and the deferred recoverable's teardown + // reconnected into the terminally scripted second mint. conn.Serve(frameDisconnect("invalid_event_stream_command", false)) - select { - case <-queued: - case <-time.After(watchdog): - t.Error("the fatal frame never reached the hand-off queue") + frames++ + for range frames { + select { + case <-queued: + case <-time.After(watchdog): + t.Error("a served frame never reached the hand-off queue") + return + } } if tc.stalled { // Lapse the superseded-poll bound so the walk From 4e643ad8407d0d8c183afd65b7b2c6e57658bc64 Mon Sep 17 00:00:00 2001 From: Jeremy Daer Date: Mon, 24 Aug 2026 19:32:48 -0700 Subject: [PATCH 32/51] Event feed: the phase timer a panic left behind, and three more verdicts that lost their turn MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Five findings from the exact-head review; four taken, one declined. The panic disposal now stops the attempt's active phase timer. The round's own fix handed disposal a nil deadline — its commit even said phase timers were "deliberately not chased" because an unfired timer fires into nothing — and that reasoning holds for the system clock only: the clock is a product seam, and a handshake deadline left registered in a host's custom clock outlives the run indefinitely, contaminating every later user. The attempt retains its currently armed phase timer (handshake and confirmation deadlines, repair cadence, poll-retry), the recover stops it, and ordinary teardowns double-stop harmlessly. Red: a panic from Observer.Connected left map[handshake-deadline:1] outstanding. The in-flight poll's done arm now parks an authoritative expiry before returning the result. Both can be ready, and accepting a FAILED verdict let recoverPoll's disposal discard the socket's fired window — an unauthorized verdict incrementing the shared counter, a 410 raising FeedGap, where transition 21 should have torn the dead socket down. The walk's existing precedence (probe, then deferral, then recoverPoll) dispatches the parked expiry ahead of any failed poll, and a succeeded page keeps finish-the-page ordering exactly as if the expiry had been observed mid-flight. Rounds red: 20/40 dispatched the unauthorized verdict over the fired window, with StaleConnection never reported. Declined: arbitrating the superseded wait's done arm against the grace deadline. The deadline bounds WAITING — "nothing waits longer than their sum" — and a result that is already ready costs no wait to accept: the page's deliveries and save are §23's own finish-the-page preference, and the deferred outcome still dispatches at the very next boundary. Discarding a served page because its acceptance raced the final wake would trade data for nothing the bound asks for. Both semantic dispatch sites now let Close outrank the handler. Observer.BufferOverflow and Observer.Gap are supported Close sites, and the handler that followed them is host code that may block — run after Close returned, it kept the iteration and Wait from terminating, and its disposition would govern a feed the consumer already ended. Red, both sites: the handler ran after Close returned. The unauthorized-threshold and filter-invalid terminals rebuild their causes without the generated error. §23's terminal table mandates the attachment for poll_failed and mint_failed ONLY — the previously declined poll_failed finding stands on exactly that mandate — while these two reasons' contracts are the counter message and the server's verbatim filter message, and a generated error routinely renders the full request URL: on this lane the server-controlled continuation, path and query included. Red, verbatim: both terminals rendered "?page_token=SECRET-CURSOR" through Error and the unwrap chain; green keeps the reason, the counter text, and the verbatim filter message, on a connector-authored cause. --- go/pkg/basecamp/eventfeed/catchup.go | 36 ++++- go/pkg/basecamp/eventfeed/catchup_test.go | 159 ++++++++++++++++++++ go/pkg/basecamp/eventfeed/connector_test.go | 96 ++++++++++++ go/pkg/basecamp/eventfeed/loop.go | 23 ++- go/pkg/basecamp/eventfeed/recovery.go | 8 + 5 files changed, 319 insertions(+), 3 deletions(-) diff --git a/go/pkg/basecamp/eventfeed/catchup.go b/go/pkg/basecamp/eventfeed/catchup.go index 5647a8d0a0..211824dbf4 100644 --- a/go/pkg/basecamp/eventfeed/catchup.go +++ b/go/pkg/basecamp/eventfeed/catchup.go @@ -483,6 +483,24 @@ func (l *loop) pollPage(at *attempt, cursor Cursor) pollAttempt { staleTimer, staleGen := at.lc.stale.current() select { case r := <-done: + // The socket's verdict outranks the poll's: both can be ready, + // and returning a FAILED result unchecked lets recoverPoll's + // disposal discard the fired expiry — an unauthorized verdict + // incrementing the shared counter, a 410 raising FeedGap, where + // transition 21 should have torn the dead socket down. An + // authoritative expiry is parked in the deferral slot (empty by + // construction here — an occupied slot means this select was + // left for the superseded wait); the walk's existing precedence + // dispatches it ahead of any failed poll, and a SUCCEEDED page + // keeps transition 21's finish-the-page ordering exactly as if + // the expiry had been observed mid-flight. + select { + case <-staleTimer.C(): + if age, ok := at.lc.stale.evaluate(staleGen); ok { + l.deferred = &deferredFrame{stale: true, age: age} + } + default: + } return pollAttempt{page: r.page, err: r.err} case <-at.lc.stale.rearmed(): // The pump swapped the window out; a parked select holds one timer, @@ -1084,6 +1102,7 @@ func protocolFatalFrame(item pumpItem) (frame, bool) { // from the first instant an observer can see the state. func (l *loop) stream(at *attempt) cycleOutcome { repair := l.cfg.clock.NewTimer(repairJitter(l.cfg.repairInterval, l.cfg.rand), timerRepairPoll) + at.phase = repair l.setState(stateStreaming) for { staleTimer, staleGen := at.lc.stale.current() @@ -1120,6 +1139,7 @@ func (l *loop) stream(at *attempt) cycleOutcome { return out } repair = l.cfg.clock.NewTimer(repairJitter(l.cfg.repairInterval, l.cfg.rand), timerRepairPoll) + at.phase = repair l.setState(stateStreaming) case <-at.lc.stale.rearmed(): continue @@ -1323,10 +1343,17 @@ func (l *loop) recoverPoll(at *attempt, cursor Cursor, err error) (walkStep, cyc l.disposeAttempt(at, nil) l.observeDisconnected("", err) if l.authFailures >= authFailureThreshold { + // The cause is rebuilt WITHOUT the generated error: §23's + // terminal table mandates the attachment for poll_failed and + // mint_failed only, and a generated error routinely renders the + // full request URL — on this lane the server-controlled + // continuation, path and query included (the redirect arm's + // reasoning, applied to the reason whose contract is its counter + // message alone). return walkStep{}, cycleOutcome{kind: outcomeTerminal, term: &TerminalError{ Reason: ReasonAuthorizationFailed, Msg: fmt.Sprintf("%d consecutive connection-level authorization failures", l.authFailures), - Err: err, + Err: &PollError{Kind: PollUnauthorized}, }}, true } // No retryAfter: `unauthorized` is specified to carry none (the @@ -1338,8 +1365,12 @@ func (l *loop) recoverPoll(at *attempt, cursor Cursor, err error) (walkStep, cyc // Transition 20: a configuration error a position reset won't help; // the server's message naming the offending list is preserved. l.disposeAttempt(at, nil) + // Same sanitized-cause rule as the unauthorized threshold above: + // filter_invalid's contract is the server's verbatim message naming + // the offending list — carried in Msg — never the generated error's + // URL-bearing rendering. return walkStep{}, cycleOutcome{kind: outcomeTerminal, term: &TerminalError{ - Reason: ReasonFilterInvalid, Msg: pe.Msg, Err: err, + Reason: ReasonFilterInvalid, Msg: pe.Msg, Err: &PollError{Kind: PollFilterInvalid, Msg: pe.Msg}, }}, true case PollRedirectRefused: // A 3xx whose Location failed the seam's per-hop validation: the @@ -1385,6 +1416,7 @@ func (l *loop) recoverPoll(at *attempt, cursor Cursor, err error) (walkStep, cyc // close still ends the cycle promptly. func (l *loop) waitPollRetry(at *attempt, d time.Duration) (cycleOutcome, bool) { t := l.cfg.clock.NewTimer(d, timerPollRetry) + at.phase = t for { staleTimer, staleGen := at.lc.stale.current() select { diff --git a/go/pkg/basecamp/eventfeed/catchup_test.go b/go/pkg/basecamp/eventfeed/catchup_test.go index 552470ad5e..70f7662b75 100644 --- a/go/pkg/basecamp/eventfeed/catchup_test.go +++ b/go/pkg/basecamp/eventfeed/catchup_test.go @@ -3344,3 +3344,162 @@ func TestFrameDeferralGracePhaseIsImmuneToFrameResets(t *testing.T) { } h.awaitTimer(timerBackoff) } + +// TestFiredStalenessOutranksAReadyPollVerdict: the family at the poll-result +// acceptance site. With the in-flight select's done and staleness cases both +// ready, accepting a FAILED poll dispatches its verdict — unauthorized +// incrementing the shared counter, gone raising FeedGap — while the socket's +// own already-fired expiry is discarded by recoverPoll's disposal, though +// the walk explicitly gives a deferred socket outcome precedence over any +// failed poll. The done arm must park an authoritative expiry first; the +// walk's existing precedence then dispatches it. Rounds-driven off a parked +// overflow handler. +func TestFiredStalenessOutranksAReadyPollVerdict(t *testing.T) { + const rounds = 40 + violations := 0 + for i := range rounds { + func() { + parked := make(chan struct{}, 1) + release := make(chan struct{}) + var once sync.Once + rel := func() { once.Do(func() { close(release) }) } + t.Cleanup(rel) + stale := make(chan time.Duration, 4) + backoffSeen := make(chan struct{}, 4) + h := newHarness(t, + eventfeed.WithLiveBufferCapacity(1), + eventfeed.WithSignalHandler(func(eventfeed.Signal) eventfeed.Disposition { + select { + case parked <- struct{}{}: + default: + } + <-release + return eventfeed.Accept + }), + eventfeed.WithObserver(eventfeed.Observer{ + StaleConnection: func(age time.Duration) { + select { + case stale <- age: + default: + } + }, + })) + h.conn.OnStateChanged(func(s string) { + if s == "backoff" { + select { + case backoffSeen <- struct{}{}: + default: + } + } + }) + h.minter.ScriptTicket(ticket(1)) + h.polls.ScriptError(&eventfeed.PollError{Kind: eventfeed.PollUnauthorized}) + var conn *feedtest.Conn + var armOnce sync.Once + h.polls.OnCall(func(feedtest.PollCall) { + armOnce.Do(func() { + // Two events against capacity 1: the drop's handler parks + // the run goroutine inside the in-flight servicing. + conn.Serve(frameMessage(noFilterIdentifier, 101)) + conn.Serve(frameMessage(noFilterIdentifier, 102)) + select { + case <-parked: + case <-time.After(watchdog): + t.Error("the overflow handler never parked the run goroutine") + return + } + // The window fires while the machine is parked; the + // release and this callback's return then land together — + // the handler's Accept resumes the machine as the poll + // verdict reaches done, and the next select sees both + // ready. + h.fireTimer(timerStaleness) + rel() + }) + }) + h.start() + conn = h.driveToSubscribed() + conn.Serve(frameConfirm(noFilterIdentifier)) + select { + case <-backoffSeen: + case <-time.After(watchdog): + t.Fatal("the round never settled in Backoff") + } + select { + case <-stale: + default: + violations++ + t.Logf("round %d: the poll's unauthorized verdict was dispatched over the socket's fired staleness window", i) + } + }() + } + if violations != 0 { + t.Errorf("%d/%d rounds accepted a poll verdict past an authoritative staleness expiry; the done arm must park the socket's verdict first (transition 21 precedence)", violations, rounds) + } +} + +// TestAuthAndFilterTerminalsCarryNoContinuationURL: §23's terminal table +// mandates the generated-error attachment for poll_failed and mint_failed +// ONLY; authorization_failed's contract is its reason and counter message, +// filter_invalid's is the server's message naming the offending list — and a +// generated error routinely renders the full request URL, which on the poll +// lane is the server-controlled continuation, path and query included. Both +// terminals must rebuild their causes without it. +func TestAuthAndFilterTerminalsCarryNoContinuationURL(t *testing.T) { + const secret = "page_token=SECRET-CURSOR" + genErr := func() error { + return fmt.Errorf("GET %s/999/events.json?%s: request failed", testOrigin, secret) + } + assertClean := func(t *testing.T, terminal *eventfeed.TerminalError) { + t.Helper() + if terminal == nil { + t.Fatal("no terminal element") + } + if strings.Contains(terminal.Error(), secret) { + t.Fatalf("the terminal's rendering carries the continuation URL: %v", terminal) + } + for err := error(terminal); err != nil; err = errors.Unwrap(err) { + if strings.Contains(err.Error(), secret) { + t.Fatalf("the terminal's unwrap chain carries the continuation URL: %v", err) + } + } + } + t.Run("filter_invalid", func(t *testing.T) { + h := newHarness(t) + h.minter.ScriptTicket(ticket(1)) + h.polls.ScriptError(&eventfeed.PollError{Kind: eventfeed.PollFilterInvalid, Msg: "types contains an unknown kind", Err: genErr()}) + h.start() + conn := h.driveToSubscribed() + conn.Serve(frameConfirm(noFilterIdentifier)) + h.join() + _, terminal, _ := h.snapshot() + if terminal == nil || terminal.Reason != eventfeed.ReasonFilterInvalid { + t.Fatalf("terminal = %v, want filter_invalid", terminal) + } + if !strings.Contains(terminal.Msg, "unknown kind") { + t.Fatalf("the server's verbatim filter message must survive: %q", terminal.Msg) + } + assertClean(t, terminal) + }) + t.Run("authorization_failed", func(t *testing.T) { + h := newHarness(t) + for range 3 { + h.minter.ScriptTicket(ticket(1)) + h.polls.ScriptError(&eventfeed.PollError{Kind: eventfeed.PollUnauthorized, Err: genErr()}) + } + h.start() + for cycle := range 3 { + if cycle > 0 { + h.fireTimer(timerBackoff) + } + conn := h.driveToSubscribed() + conn.Serve(frameConfirm(noFilterIdentifier)) + } + h.join() + _, terminal, _ := h.snapshot() + if terminal == nil || terminal.Reason != eventfeed.ReasonAuthorizationFailed { + t.Fatalf("terminal = %v, want authorization_failed", terminal) + } + assertClean(t, terminal) + }) +} diff --git a/go/pkg/basecamp/eventfeed/connector_test.go b/go/pkg/basecamp/eventfeed/connector_test.go index 5f26b1ff1f..64ce74bbaa 100644 --- a/go/pkg/basecamp/eventfeed/connector_test.go +++ b/go/pkg/basecamp/eventfeed/connector_test.go @@ -1626,3 +1626,99 @@ func TestPanicInHostCodeDisposesTheLiveAttempt(t *testing.T) { assertTimers(t, h.clock, map[string]int{}) assertGoroutinesSettle(t, base) } + +// TestPanicMidHandshakeStopsThePhaseTimer: the panic-disposal fix closed the +// socket, joined the pump, and stopped staleness — but handed disposal a nil +// deadline, so a phase timer active at the panic (here the handshake +// deadline, armed through Observer.Connected) stayed registered. With the +// product clock seam that is an outstanding timer contaminating the host's +// clock indefinitely; the attempt must retain its active phase timer so +// panic unwinding can stop it. +func TestPanicMidHandshakeStopsThePhaseTimer(t *testing.T) { + h := newHarness(t, eventfeed.WithObserver(eventfeed.Observer{ + Connected: func() { panic("host code panicked mid-handshake") }, + })) + h.minter.ScriptTicket(ticket(1)) + base := runtime.NumGoroutine() + recovered := make(chan any, 1) + go func() { + defer func() { recovered <- recover() }() + for range h.conn.Events(context.Background()) { + } + }() + + select { + case r := <-recovered: + if r == nil { + t.Fatal("the panic did not propagate out of the iteration") + } + case <-time.After(watchdog): + t.Fatal("the iteration neither panicked nor returned") + } + if sock := h.tr.LastConn(); sock == nil || !sock.Closed() { + t.Fatal("the live socket was not disposed during panic unwinding") + } + assertTimers(t, h.clock, map[string]int{}) + assertGoroutinesSettle(t, base) +} + +// TestCloseFromSignalObserverSkipsTheHandler: BufferOverflow and Gap each +// fire an observability callback before their semantic handler, and Close is +// supported from every observer callback — runCtx is then already cancelled +// when the callback returns, and dispatching the handler anyway runs host +// code (which may block indefinitely) after Close returned, keeping the +// iteration and Wait from terminating. The Closed edge wins between the +// callback and the dispatch. +func TestCloseFromSignalObserverSkipsTheHandler(t *testing.T) { + t.Run("buffer overflow", func(t *testing.T) { + var invoked atomic.Bool + var h *harness + h = newHarness(t, + eventfeed.WithLiveBufferCapacity(1), + eventfeed.WithObserver(eventfeed.Observer{ + BufferOverflow: func(int) { h.conn.Close() }, //nolint:errcheck // asserted below + }), + eventfeed.WithSignalHandler(func(eventfeed.Signal) eventfeed.Disposition { + invoked.Store(true) + return eventfeed.Accept + })) + h.minter.ScriptTicket(ticket(1)) + h.start() + + conn := h.driveToSubscribed() + conn.Serve(frameMessage(noFilterIdentifier, 101)) + conn.Serve(frameMessage(noFilterIdentifier, 102)) + h.join() + if invoked.Load() { + t.Fatal("the signal handler ran after Close returned from Observer.BufferOverflow") + } + if _, terminal, elements := h.snapshot(); terminal != nil || elements != 0 { + t.Fatalf("clean close: got %d elements, terminal %v", elements, terminal) + } + }) + t.Run("feed gap", func(t *testing.T) { + var invoked atomic.Bool + var h *harness + h = newHarness(t, + eventfeed.WithObserver(eventfeed.Observer{ + Gap: func(int64, string) { h.conn.Close() }, //nolint:errcheck // asserted below + }), + eventfeed.WithSignalHandler(func(eventfeed.Signal) eventfeed.Disposition { + invoked.Store(true) + return eventfeed.Terminate + })) + h.minter.ScriptTicket(ticket(1)) + h.polls.ScriptError(&eventfeed.PollError{Kind: eventfeed.PollGone, EpochAfterID: 50, ResumeURL: testOrigin + "/999/events.json?since=now"}) + h.start() + + conn := h.driveToSubscribed() + conn.Serve(frameConfirm(noFilterIdentifier)) + h.join() + if invoked.Load() { + t.Fatal("the signal handler ran after Close returned from Observer.Gap") + } + if _, terminal, elements := h.snapshot(); terminal != nil || elements != 0 { + t.Fatalf("clean close: got %d elements, terminal %v", elements, terminal) + } + }) +} diff --git a/go/pkg/basecamp/eventfeed/loop.go b/go/pkg/basecamp/eventfeed/loop.go index 8fa5b6dadf..cef2030224 100644 --- a/go/pkg/basecamp/eventfeed/loop.go +++ b/go/pkg/basecamp/eventfeed/loop.go @@ -117,6 +117,14 @@ type attempt struct { ctx context.Context cancel context.CancelFunc lc *liveConn + // phase is the attempt's currently armed phase timer — handshake or + // confirmation deadline, repair cadence, or poll-retry — retained so + // panic unwinding can stop it. Ordinary teardown paths stop these + // through disposal's own parameter; this field exists for the one + // unwinding that cannot name them: a host-code panic, where a timer + // left registered contaminates a product clock indefinitely. Stop is + // idempotent, so double-stopping a normally disposed timer is free. + phase Timer } // pumpDepth is the frame pump's bounded hand-off queue depth (SPEC.md §23 @@ -838,7 +846,9 @@ func (l *loop) runCycle(delay time.Duration) cycleOutcome { defer func() { if r := recover(); r != nil { if at.lc != nil { - l.disposeAttempt(at, nil) + l.disposeAttempt(at, at.phase) + } else if at.phase != nil { + at.phase.Stop() } panic(r) } @@ -888,6 +898,7 @@ func (l *loop) runCycle(delay time.Duration) cycleOutcome { // The handshake deadline arms on entry to Connecting, BEFORE dial — it // spans dial-to-welcome, so a stalled dial expires it (transition 7). hs := l.cfg.clock.NewTimer(handshakeDeadline, timerHandshakeDeadline) + at.phase = hs type dialResult struct { conn CableConn err error @@ -1289,6 +1300,7 @@ func (l *loop) handleFrame(at *attempt, deadline *Timer, item pumpItem) (cycleOu return cycleOutcome{kind: outcomeFailed}, true } *deadline = l.cfg.clock.NewTimer(l.cfg.confirmationDeadline, timerConfirmationDeadline) + at.phase = *deadline l.setState(stateAwaitingConfirmation) } return cycleOutcome{}, false @@ -1428,6 +1440,15 @@ func (l *loop) admitLive(at *attempt, deadline Timer, ev Event) (cycleOutcome, b if l.cfg.observer.BufferOverflow != nil { l.cfg.observer.BufferOverflow(len(dropped)) } + // Close outranks the semantic dispatch: the observer above is a + // supported Close site, and the handler is host code that may block — + // run after Close returned, it keeps the iteration and Wait from + // terminating, and its disposition would govern a feed the consumer + // already ended. + if l.runCtx.Err() != nil { + l.disposeAttempt(at, deadline) + return cycleOutcome{kind: outcomeClosed}, true + } if l.cfg.handler != nil && l.cfg.handler(signal) == Accept { // Accept: the consumer owns the acknowledged incompleteness; the // feed continues (acceptance is not license to skip retained diff --git a/go/pkg/basecamp/eventfeed/recovery.go b/go/pkg/basecamp/eventfeed/recovery.go index 1d3fdb7910..ebf7a709a9 100644 --- a/go/pkg/basecamp/eventfeed/recovery.go +++ b/go/pkg/basecamp/eventfeed/recovery.go @@ -117,6 +117,14 @@ func (l *loop) recoverGone(at *attempt, pe *PollError) (walkStep, cycleOutcome, // follows the full URL after §8 validation. l.cfg.observer.Gap(pe.EpochAfterID, redactURL(l.cfg.origin, pe.ResumeURL)) } + // Close outranks the semantic dispatch, exactly as at the overflow site: + // Observer.Gap is a supported Close site, and the handler is host code + // that may block — run after Close returned, it keeps the iteration and + // Wait from terminating. + if l.runCtx.Err() != nil { + l.disposeAttempt(at, nil) + return walkStep{}, cycleOutcome{kind: outcomeClosed}, true + } if l.cfg.handler != nil && l.cfg.handler(signal) == Accept { if pe.ResumeURL == "" { // Accepting a 410 that carried no resume URL cannot be honored: From 1d87d28fa284ed15a298d640b25c240b84bbcada Mon Sep 17 00:00:00 2001 From: Jeremy Daer Date: Mon, 24 Aug 2026 21:15:19 -0700 Subject: [PATCH 33/51] Invert the two continuation-terminal pins to the value-free contract Foundations' round-12 P1 made every rejected-continuation rendering zero-interpolation closed vocabulary; these two end-to-end pins still required the rejected origin in the terminal message and went red on the merge. They now assert the absence of every server-authored component, matching the checkContinuation pin that owns the exact phrase. --- go/pkg/basecamp/eventfeed/catchup_test.go | 12 +++++++----- go/pkg/basecamp/eventfeed/recovery_test.go | 12 +++++++----- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/go/pkg/basecamp/eventfeed/catchup_test.go b/go/pkg/basecamp/eventfeed/catchup_test.go index 70f7662b75..8424836325 100644 --- a/go/pkg/basecamp/eventfeed/catchup_test.go +++ b/go/pkg/basecamp/eventfeed/catchup_test.go @@ -451,11 +451,13 @@ func TestHostileContinuationIsTerminal(t *testing.T) { if terminal == nil || terminal.Reason != eventfeed.ReasonInvalidContinuation { t.Fatalf("terminal = %v, want reason %q", terminal, eventfeed.ReasonInvalidContinuation) } - if !strings.Contains(terminal.Msg, "https://attacker.example.com") { - t.Fatalf("terminal message %q should name the rejected origin", terminal.Msg) - } - if strings.Contains(terminal.Msg, "token=secret") || strings.Contains(terminal.Msg, "/999/") { - t.Fatalf("terminal message %q must be redacted to the origin", terminal.Msg) + // The rejection is value-free: a rejected URL's every component is hostile + // text a bearer-holding server can reflect, so nothing server-authored may + // render (the checkContinuation pin owns the exact phrase; this asserts + // the absence end-to-end). + if strings.Contains(terminal.Msg, "attacker.example.com") || + strings.Contains(terminal.Msg, "token=secret") || strings.Contains(terminal.Msg, "/999/") { + t.Fatalf("terminal message %q echoes the rejected URL", terminal.Msg) } if got := h.polls.CallCount(); got != 1 { t.Fatalf("poll seam calls = %d, want 1 (zero requests to the foreign origin)", got) diff --git a/go/pkg/basecamp/eventfeed/recovery_test.go b/go/pkg/basecamp/eventfeed/recovery_test.go index cfe6801ddf..56ec09735a 100644 --- a/go/pkg/basecamp/eventfeed/recovery_test.go +++ b/go/pkg/basecamp/eventfeed/recovery_test.go @@ -276,11 +276,13 @@ func TestFeedGapAcceptedHostileResumeIsInvalidContinuation(t *testing.T) { if terminal == nil || terminal.Reason != eventfeed.ReasonInvalidContinuation { t.Fatalf("terminal = %v, want reason %q", terminal, eventfeed.ReasonInvalidContinuation) } - if !strings.Contains(terminal.Msg, "https://attacker.example.com") { - t.Fatalf("terminal message %q should name the rejected origin", terminal.Msg) - } - if strings.Contains(terminal.Msg, "token=secret") || strings.Contains(terminal.Msg, "/999/") { - t.Fatalf("terminal message %q must be redacted to the origin", terminal.Msg) + // The rejection is value-free: a rejected URL's every component is hostile + // text a bearer-holding server can reflect, so nothing server-authored may + // render (the checkContinuation pin owns the exact phrase; this asserts + // the absence end-to-end). + if strings.Contains(terminal.Msg, "attacker.example.com") || + strings.Contains(terminal.Msg, "token=secret") || strings.Contains(terminal.Msg, "/999/") { + t.Fatalf("terminal message %q echoes the rejected URL", terminal.Msg) } if got := h.polls.CallCount(); got != 1 { t.Fatalf("poll seam calls = %d, want 1 (zero requests to the foreign origin)", got) From 2f7c26082e2fc08a5f9da6420163292da512762e Mon Sep 17 00:00:00 2001 From: Jeremy Daer Date: Mon, 24 Aug 2026 22:29:30 -0700 Subject: [PATCH 34/51] Event feed: the redirect terminal renders a phrase, never the peer's spelling MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The routed half of foundations' round: PollError.LocationOrigin is now data-not-rendering by contract — a hostile redirect can reflect the caller's bearer into a host label, the CloseError.Reason precedent — and PollError.Error already omits it, but this branch's refused-redirect terminal still concatenated the origin into TerminalError.Msg, undoing the field's demotion at the one surface consumers read first. The terminal now renders the fixed violation-class phrase the continuation rejections converged on ("the poll refused a cross-origin redirect"); the origin survives as a field on the retained sanitized cause for a caller that reads it, which no rendering includes. Red proof, bearer-canary form: a LocationOrigin of https://bearer-canary-abc123.invalid appeared verbatim in the terminal's Error() pre-fix and appears nowhere in the rendering or the unwrap chain after. Two tests pinned the old origin-bearing message — TestRedirectRefusalExposesOnlyTheLocationOrigin (now TestRedirectRefusalRendersNoServerValue, inverted into the canary proof) and TestPollFailureClassification's redirect row, which now expects the fixed phrase and points at the negative pin. --- go/pkg/basecamp/eventfeed/catchup.go | 18 +++++++----- go/pkg/basecamp/eventfeed/catchup_test.go | 36 +++++++++++++++-------- 2 files changed, 34 insertions(+), 20 deletions(-) diff --git a/go/pkg/basecamp/eventfeed/catchup.go b/go/pkg/basecamp/eventfeed/catchup.go index 211824dbf4..efcdfa6ffd 100644 --- a/go/pkg/basecamp/eventfeed/catchup.go +++ b/go/pkg/basecamp/eventfeed/catchup.go @@ -1374,17 +1374,19 @@ func (l *loop) recoverPoll(at *attempt, cursor Cursor, err error) (walkStep, cyc }}, true case PollRedirectRefused: // A 3xx whose Location failed the seam's per-hop validation: the - // continuation edge, NEVER poll_failed. This edge exposes the rejected - // Location redacted to its ORIGIN — a hostile continuation's path and - // query are exactly what must not be echoed — so the seam error is not - // retained as the cause: PollError.Error renders its underlying Err and - // Unwrap hands it out, and the generated error's text routinely carries - // the request URL in full. The classification survives on a sanitized - // cause carrying the same redaction the terminal promises. + // continuation edge, NEVER poll_failed — and the terminal renders a + // FIXED violation-class phrase, the continuation rejections' own + // contract. The refused origin is DATA, not a rendering: a hostile + // redirect can reflect the caller's bearer into a host label (the + // CloseError.Reason precedent), so no rendering may carry + // LocationOrigin. It survives as a field on the sanitized cause for a + // caller that reads it — PollError.Error deliberately omits it — and + // the generated error is not retained at all, its text routinely + // carrying the request URL in full. l.disposeAttempt(at, nil) return walkStep{}, cycleOutcome{kind: outcomeTerminal, term: &TerminalError{ Reason: ReasonInvalidContinuation, - Msg: "the poll refused a redirect to " + pe.LocationOrigin, + Msg: "the poll refused a cross-origin redirect", Err: &PollError{Kind: PollRedirectRefused, LocationOrigin: pe.LocationOrigin}, }}, true case PollUnrecoverable: diff --git a/go/pkg/basecamp/eventfeed/catchup_test.go b/go/pkg/basecamp/eventfeed/catchup_test.go index 8424836325..76a1cc55b2 100644 --- a/go/pkg/basecamp/eventfeed/catchup_test.go +++ b/go/pkg/basecamp/eventfeed/catchup_test.go @@ -612,10 +612,14 @@ func TestPollFailureClassification(t *testing.T) { "unknown poll error kind 0", }, { + // The terminal renders a fixed violation-class phrase; the + // refused origin is data on the retained cause, never a + // rendering (TestRedirectRefusalRendersNoServerValue pins the + // negative). "redirect refused is invalid_continuation", &eventfeed.PollError{Kind: eventfeed.PollRedirectRefused, LocationOrigin: "https://attacker.example.com"}, eventfeed.ReasonInvalidContinuation, - "https://attacker.example.com", + "the poll refused a cross-origin redirect", }, } for _, tc := range cases { @@ -1460,20 +1464,23 @@ func TestDrainHoldsNoMoreThanTheLiveBufferCapacity(t *testing.T) { assertIDs(t, dropped, 11, 12, 13, 14, 21, 22, 23, 24) } -// TestRedirectRefusalExposesOnlyTheLocationOrigin: the refused-redirect edge -// promises the rejected Location redacted to its ORIGIN — a hostile -// continuation's path and query are exactly what must not be echoed — so the -// terminal cannot retain the seam error as its cause. PollError.Error and -// PollError.Unwrap both reach the underlying generated error, whose text -// routinely carries the request URL in full. -func TestRedirectRefusalExposesOnlyTheLocationOrigin(t *testing.T) { +// TestRedirectRefusalRendersNoServerValue: the refused-redirect terminal +// renders a FIXED violation-class phrase and nothing of the server's — the +// continuation rejections' own contract, applied to the redirect edge. +// LocationOrigin is DATA, never a rendering: a hostile redirect can reflect +// the caller's bearer into a host label (the CloseError.Reason precedent), +// so the canary below stands in for exactly that, and it must appear in no +// rendering and nowhere in the unwrap chain — the retained PollError carries +// the field for a caller that reads it, and PollError.Error omits it. +func TestRedirectRefusalRendersNoServerValue(t *testing.T) { + const canary = "bearer-canary-abc123" const secret = "/steal?ticket=abc123&next=%2Fadmin" h := newHarness(t) h.minter.ScriptTicket(ticket(1)) h.polls.ScriptError(&eventfeed.PollError{ Kind: eventfeed.PollRedirectRefused, - LocationOrigin: "https://attacker.example.com", - Err: errors.New("302 Location: https://attacker.example.com" + secret), + LocationOrigin: "https://" + canary + ".invalid", + Err: errors.New("302 Location: https://" + canary + ".invalid" + secret), }) h.start() @@ -1485,8 +1492,13 @@ func TestRedirectRefusalExposesOnlyTheLocationOrigin(t *testing.T) { if terminal == nil || terminal.Reason != eventfeed.ReasonInvalidContinuation { t.Fatalf("terminal = %v, want reason %q", terminal, eventfeed.ReasonInvalidContinuation) } - if !strings.Contains(terminal.Msg, "https://attacker.example.com") { - t.Fatalf("terminal message %q should name the refused origin", terminal.Msg) + if strings.Contains(terminal.Error(), canary) { + t.Fatalf("terminal rendering %q carries the refused Location's origin; the terminal renders a fixed phrase, never a server value", terminal.Error()) + } + for err := error(terminal); err != nil; err = errors.Unwrap(err) { + if strings.Contains(err.Error(), canary) { + t.Fatalf("the terminal's cause chain renders the refused origin: %v", err) + } } // The whole rendering, not just Msg: Error walks the cause chain. if rendered := terminal.Error(); strings.Contains(rendered, secret) { From 40a7d4f509e7f0f0f262ca69c3a31f107acc809d Mon Sep 17 00:00:00 2001 From: Jeremy Daer Date: Wed, 9 Sep 2026 20:45:01 -0700 Subject: [PATCH 35/51] Annotate the attempt's cancel for gosec G118 --- go/pkg/basecamp/eventfeed/loop.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/pkg/basecamp/eventfeed/loop.go b/go/pkg/basecamp/eventfeed/loop.go index cef2030224..4292474146 100644 --- a/go/pkg/basecamp/eventfeed/loop.go +++ b/go/pkg/basecamp/eventfeed/loop.go @@ -831,7 +831,7 @@ func (l *loop) emitTerminal(term *TerminalError) { func (l *loop) runCycle(delay time.Duration) cycleOutcome { at := &attempt{} - at.ctx, at.cancel = context.WithCancel(l.runCtx) + at.ctx, at.cancel = context.WithCancel(l.runCtx) //nolint:gosec // G118: deferred on the next line; the field is what disposeAttempt cancels early defer at.cancel() // Host code — observers, the signal handler, the checkpoint store, and // the consumer's own range body — runs on this goroutine, and any of it From f3e64f437f08b8f677bf2ba25280abc77c17f4b7 Mon Sep 17 00:00:00 2001 From: Jeremy Daer Date: Wed, 9 Sep 2026 23:47:44 -0700 Subject: [PATCH 36/51] State the Clock seam's promptness obligation on NewTimer --- go/pkg/basecamp/eventfeed/clock.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/go/pkg/basecamp/eventfeed/clock.go b/go/pkg/basecamp/eventfeed/clock.go index 71300b0497..8a46e6914f 100644 --- a/go/pkg/basecamp/eventfeed/clock.go +++ b/go/pkg/basecamp/eventfeed/clock.go @@ -19,7 +19,12 @@ type Clock interface { // only for deltas, never persisted. Now() time.Time // NewTimer arms a one-shot named timer. Firing removes it from the - // registry. + // registry. It is called under the connector's own locks — on every + // state transition, on every inbound frame, and during teardown, where + // the pump's last hand-off can re-arm staleness a moment before the + // holder is stopped — so it must return promptly: a NewTimer that blocks + // stalls the transition that called it, and a teardown that stalls holds + // the iteration and Wait open after Close. NewTimer(d time.Duration, name string) Timer // Outstanding returns the names of live (unfired, unstopped) timers, in // creation order. From 300b97494b902bc8adcd9dfd7b92c3339ca3fdc9 Mon Sep 17 00:00:00 2001 From: Jeremy Daer Date: Thu, 10 Sep 2026 00:34:04 -0700 Subject: [PATCH 37/51] Event feed: the ring grows only while it is contiguous MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The live buffer's ring wrapped an insert whenever the tail's index fell below the head, and grew the backing whenever it did not. Those are the right two moves in the wrong order. A drain's shifts move the head off zero long before the store reaches capacity, so a pre-capacity store whose tail has reached len(events) satisfied the wrap test first: the insert circled to index 0, and the next insert — now past the head — appended, changing the modulus every existing index is taken against. With capacity 3, add 1 and 2, shift 1, add 3, add 4 read back as 2, 4, 3. fatalScan interleaves admissions with the drain's shifts, so this was the drain's own delivery order. The insert now asks whether the store has reached capacity, which is the invariant the layout actually rests on: below it the occupied range is contiguous and ends at or before len(events), so growth is the only move that keeps every index valid, and a wrap can exist only at capacity, where the modulus never changes again. The regression test drives the three partial-wrap shapes and reads the buffer back in logical order. --- go/pkg/basecamp/eventfeed/buffer_test.go | 66 +++++++++++++++++++++++- go/pkg/basecamp/eventfeed/loop.go | 28 +++++++--- 2 files changed, 85 insertions(+), 9 deletions(-) diff --git a/go/pkg/basecamp/eventfeed/buffer_test.go b/go/pkg/basecamp/eventfeed/buffer_test.go index 72b310bd9b..62fc0d1792 100644 --- a/go/pkg/basecamp/eventfeed/buffer_test.go +++ b/go/pkg/basecamp/eventfeed/buffer_test.go @@ -1,6 +1,9 @@ package eventfeed -import "testing" +import ( + "fmt" + "testing" +) // TestLiveBufferAddClearsEvictedSlots pins the eviction half of the live // buffer's memory ceiling. SPEC.md §23 publishes the connector's worst case @@ -71,3 +74,64 @@ func TestSustainedOverflowRetainsTheBacking(t *testing.T) { t.Fatalf("a full window of sustained overflow made %.0f allocations, want at most %d (one dropped-ids slice per admit): the backing was reallocated instead of retained", allocs, capacity) } } + +// TestLiveBufferGrowsAfterAPartialWrapInOrder pins FIFO order across the one +// layout the ring's growth rule did not account for: a store that has not yet +// reached capacity, whose head a drain's shifts have moved off zero, and +// whose tail has since circled back below the head. Growing the backing then +// changes the modulus every existing index is taken against, and a wrapped +// entry that was logically last becomes physically stranded — with capacity +// 3, add 1 and 2, shift 1, add 3 (wraps to index 0), add 4 (appends index 2) +// read back as 2, 4, 3. fatalScan interleaves admissions with drain shifts, +// so this is the drain's own delivery order, not a contrived one. +func TestLiveBufferGrowsAfterAPartialWrapInOrder(t *testing.T) { + for _, tc := range []struct { + name string + capacity int + adds int + shifts int + more int + }{ + {"capacity 3: fill 2, shift 1, add 2", 3, 2, 1, 2}, + {"capacity 5: fill 3, shift 2, add 3", 5, 3, 2, 3}, + {"capacity 4: fill 1, shift 1, add 4", 4, 1, 1, 4}, + } { + t.Run(tc.name, func(t *testing.T) { + b := newLiveBuffer(tc.capacity, nil) + var next int64 = 1 + var want []int64 + for range tc.adds { + b.add(Event{ID: next}) + want = append(want, next) + next++ + } + for range tc.shifts { + if ev, ok := b.shift(); !ok || ev.ID != want[0] { + t.Fatalf("shift = %+v (%t), want id %d", ev, ok, want[0]) + } + want = want[1:] + } + for range tc.more { + if dropped := b.add(Event{ID: next}); len(dropped) != 0 { + t.Fatalf("add %d dropped %v below capacity %d", next, dropped, tc.capacity) + } + want = append(want, next) + next++ + } + got := b.snapshot() + ids := make([]int64, len(got)) + for i, ev := range got { + ids[i] = ev.ID + } + if fmt.Sprint(ids) != fmt.Sprint(want) { + t.Fatalf("logical order = %v, want %v", ids, want) + } + for i, id := range want { + ev, ok := b.shift() + if !ok || ev.ID != id { + t.Fatalf("shift %d = %+v (%t), want id %d", i, ev, ok, id) + } + } + }) + } +} diff --git a/go/pkg/basecamp/eventfeed/loop.go b/go/pkg/basecamp/eventfeed/loop.go index 4292474146..a8023a5ca4 100644 --- a/go/pkg/basecamp/eventfeed/loop.go +++ b/go/pkg/basecamp/eventfeed/loop.go @@ -496,9 +496,17 @@ type liveBuffer struct { // copied the whole backing — an O(capacity) copy spike, and a transient // SECOND buffer's worth of retained payload at each growth step, the // exact retention class add's zeroing exists to prevent. Entries live at - // [head, head+size) modulo len(events); a wrap can exist only once the - // store has reached capacity, because drops happen only at capacity and - // pre-capacity inserts always land at or before len(events). + // [head, head+size) modulo len(events), and the layout invariant that + // makes growth safe is that a wrap exists ONLY once the store has reached + // capacity: below it the occupied range is contiguous and ends at or + // before len(events), so an insert that lands at len(events) appends and + // leaves every existing index valid under the grown modulus. A drain's + // shifts move head off zero long before capacity, which is why the + // insert has to ask "has the store reached capacity?" and never "does the + // tail's index fall below the head?" — the latter is true of a + // pre-capacity store whose tail has reached len(events), and wrapping + // there and growing afterwards strands the wrapped entry under a modulus + // it was not written against (2, 3, 4 read back as 2, 4, 3). events []Event head int size int @@ -527,13 +535,17 @@ func (b *liveBuffer) add(ev Event) []int64 { switch pos := b.head + b.size; { case pos < len(b.events): b.events[pos] = ev - case pos-len(b.events) < b.head: + case len(b.events) < b.capacity: + // Still filling: the occupied range is contiguous below capacity, so + // pos == len(events) here and the store grows. Growing is the ONLY + // move that keeps order once head has moved off zero — see the + // layout invariant on the events field. + b.events = append(b.events, ev) + default: // Wrapped: the store is at capacity and the tail circles back below - // the head. + // the head (size < capacity == len(events) after the drops above, so + // pos-len(events) < head). b.events[pos-len(b.events)] = ev - default: - // Still filling: pos == len(events) < capacity, so the store grows. - b.events = append(b.events, ev) } b.size++ b.changed() From 1987fc4cb06dd341e833240f9b622d0f2e08fe4f Mon Sep 17 00:00:00 2001 From: Jeremy Daer Date: Thu, 10 Sep 2026 00:44:20 -0700 Subject: [PATCH 38/51] Event feed: Close outranks the admission, wherever the frame was received MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three receiving arms were missed when the others learned that a ready item does not outrank a Close: the in-flight-poll servicing, the bounded admission pass, and the drain's protocol-fatal scan. Each dequeued past a cancelled run context, and each did post-Close work the Closed edge exists to rule out. The admission pass dispatched the pump's own cancellation error through the live-frame path, so Observer.Disconnected reported the consumer's Close as a socket failure after Close returned. The other two admitted frames into a buffer the next run never reads, and when that admission dropped, called Observer.BufferOverflow after Close returned — a registered handler that closed the connector and returned Accept saw the pass it returned into keep admitting. The check is added at the three receives, and once more at the one place every admission passes: admitLive refuses an admission under a cancelled run context before it touches the buffer. That is what covers the accepting-handler case wherever the pass that dispatched it receives next, rather than only at that pass's own dispatch point. Three tests pin the three arms, each red against the previous revision: the handler queues a third frame before closing and asserts no overflow observer call follows; the pass is parked by a frame-handled callback while Close lands and the cancellation queues behind the parked frame; and the drain's loop body closes with a protocol-fatal disconnect and two events already queued behind the event in hand — the fatal queued first, because it bypasses admitLive's guard and so pins the scan's own check. --- go/pkg/basecamp/eventfeed/catchup.go | 26 ++ go/pkg/basecamp/eventfeed/connector_test.go | 266 ++++++++++++++++++++ go/pkg/basecamp/eventfeed/loop.go | 13 + 3 files changed, 305 insertions(+) diff --git a/go/pkg/basecamp/eventfeed/catchup.go b/go/pkg/basecamp/eventfeed/catchup.go index efcdfa6ffd..7c665d1931 100644 --- a/go/pkg/basecamp/eventfeed/catchup.go +++ b/go/pkg/basecamp/eventfeed/catchup.go @@ -520,6 +520,15 @@ func (l *loop) pollPage(at *attempt, cursor Cursor) pollAttempt { if !ok { l.deferred = &deferredFrame{closed: true} } else { + // Close outranks a ready item, as on every other arm that + // receives: the item may be the cancellation the pump handed + // off after Close, and servicing it — or admitting a frame + // queued ahead of it — is post-Close work. Disposing here is + // what returns the abandoned call. + if l.runCtx.Err() != nil { + l.disposeAttempt(at, nil) + return pollAttempt{ended: true, out: cycleOutcome{kind: outcomeClosed}} + } handled, out, ended := l.admitDuringPoll(at, item) if ended { // The drop's own disposition. Disposing the attempt is what @@ -805,6 +814,15 @@ func (l *loop) admissionPass(at *attempt) (cycleOutcome, bool) { if !ok { return l.pumpExited(at, nil), true } + // Close outranks a ready item. The pass is entered under a check, + // but it dequeues up to liveBufferCapacity items after it, and a + // Close landing mid-pass is followed by the pump's cancellation + // error — which handleLiveFrame would report to + // Observer.Disconnected as a socket failure, after Close returned. + if l.runCtx.Err() != nil { + l.disposeAttempt(at, nil) + return cycleOutcome{kind: outcomeClosed}, true + } if out, done := l.handleLiveFrame(at, nil, item, false); done { return out, true } @@ -992,6 +1010,14 @@ func (l *loop) fatalScan(at *attempt, budget *int) (cycleOutcome, bool) { return cycleOutcome{}, false } *budget-- + // Close outranks a ready item here as in every receive: a consumer + // that closed from the drain's own loop body continues the range, + // and the next scan must not admit — or dispatch — what the pump + // queued after that Close. + if l.runCtx.Err() != nil { + l.disposeAttempt(at, nil) + return cycleOutcome{kind: outcomeClosed}, true + } handled, out, ended := l.admitDuringPoll(at, item) switch { case ended: diff --git a/go/pkg/basecamp/eventfeed/connector_test.go b/go/pkg/basecamp/eventfeed/connector_test.go index 64ce74bbaa..6acc918ce1 100644 --- a/go/pkg/basecamp/eventfeed/connector_test.go +++ b/go/pkg/basecamp/eventfeed/connector_test.go @@ -1722,3 +1722,269 @@ func TestCloseFromSignalObserverSkipsTheHandler(t *testing.T) { } }) } + +// TestCloseFromAnAcceptingHandlerRefusesTheNextAdmission: a registered +// SignalHandler that closes the connector and returns Accept is host code +// ending the feed from inside a dispatch. The in-flight-poll servicing that +// dispatched it keeps receiving until the seam call returns, and a frame +// already queued behind the overflowing one used to be admitted anyway — +// manufacturing a second drop, and with it a second Observer.BufferOverflow +// call, after Close returned. Admission is refused once runCtx is cancelled, +// wherever the pass that dispatched the handler receives it. Rounds-driven: +// the abandoned call's own return races the queued frame at the select. +func TestCloseFromAnAcceptingHandlerRefusesTheNextAdmission(t *testing.T) { + const rounds = 10 + violations := 0 + for i := range rounds { + func() { + var overflows atomic.Int32 + var afterClose atomic.Int32 + closed := make(chan struct{}) + handedOff := make(chan struct{}, 8) + var h *harness + h = newHarness(t, + eventfeed.WithStart(eventfeed.StartPresent()), + eventfeed.WithLiveBufferCapacity(1), + eventfeed.WithObserver(eventfeed.Observer{ + BufferOverflow: func(int) { + overflows.Add(1) + select { + case <-closed: + afterClose.Add(1) + default: + } + }, + }), + eventfeed.WithSignalHandler(func(eventfeed.Signal) eventfeed.Disposition { + // A third frame is queued BEFORE the handler closes, so the + // servicing loop it returns into finds it ready. + drain(handedOff) + h.tr.LastConn().Serve(frameMessage(noFilterIdentifier, 103)) + select { + case <-handedOff: + case <-time.After(watchdog): + t.Error("the third frame never reached the hand-off queue") + } + h.conn.Close() //nolint:errcheck // asserted via the observer + close(closed) + return eventfeed.Accept + })) + h.conn.OnPumpHandedOff(func(bool) { + select { + case handedOff <- struct{}{}: + default: + } + }) + h.minter.ScriptTicket(ticket(1)) + h.polls.StallNext() + h.start() + + conn := h.driveToSubscribed() + conn.Serve(frameConfirm(noFilterIdentifier)) + h.polls.OnCall(func(feedtest.PollCall) {}) + h.waitUntil("the entry poll to be in flight", func() bool { return h.polls.CallCount() == 1 }) + drain(handedOff) + conn.Serve(frameMessage(noFilterIdentifier, 101)) + conn.Serve(frameMessage(noFilterIdentifier, 102)) + h.join() + if n := afterClose.Load(); n != 0 { + violations++ + t.Logf("round %d: %d BufferOverflow observer call(s) after Close returned from the handler", i, n) + } + if _, terminal, elements := h.snapshot(); terminal != nil || elements != 0 { + t.Fatalf("clean close: got %d elements, terminal %v", elements, terminal) + } + }() + } + if violations != 0 { + t.Errorf("%d/%d rounds admitted a frame after the accepting handler closed the connector", violations, rounds) + } +} + +// TestCloseMidAdmissionPassTakesTheClosedEdge: the bounded admission pass is +// entered under a cancellation check but dequeues up to liveBufferCapacity +// items after it, and a Close landing mid-pass is followed by the pump's own +// cancellation error — which the pass dispatched through the ordinary +// live-frame path, reporting the consumer's Close to Observer.Disconnected as +// a socket failure after Close returned. The state machine is parked inside +// the pass by a frame-handled callback, Close lands, and the cancellation is +// queued behind the parked frame before the pass resumes. +func TestCloseMidAdmissionPassTakesTheClosedEdge(t *testing.T) { + disconnected := make(chan error, 4) + handedOff := make(chan struct{}, 8) + handedOffErr := make(chan struct{}, 4) + var conn *feedtest.Conn + var deliveredOnce sync.Once + h := newHarness(t, + eventfeed.WithStart(eventfeed.StartPresent()), + eventfeed.WithObserver(eventfeed.Observer{ + PageDelivered: func(int, string) { + deliveredOnce.Do(func() { + // Queued AFTER the entry page returned and BEFORE the + // cut's pass runs, so the pass — not the in-flight + // servicing — is what dequeues them. + drain(handedOff) + conn.Serve(framePing()) + conn.Serve(framePing()) + for range 2 { + select { + case <-handedOff: + case <-time.After(watchdog): + t.Error("a ping never reached the hand-off queue") + } + } + }) + }, + Disconnected: func(_ string, err error) { + select { + case disconnected <- err: + default: + } + }, + })) + h.conn.OnPumpHandedOff(func(isErr bool) { + ch := handedOff + if isErr { + ch = handedOffErr + } + select { + case ch <- struct{}{}: + default: + } + }) + parked := make(chan struct{}) + gate := make(chan struct{}) + var parkOnce sync.Once + h.conn.OnFrameHandled(func(kind string) { + if kind != "ping" { + return + } + parkOnce.Do(func() { + close(parked) + <-gate + }) + }) + h.minter.ScriptTicket(ticket(1)) + h.polls.ScriptPage(eventfeed.PollPage{Position: "pos-1"}) + h.start() + conn = h.driveToSubscribed() + conn.Serve(frameConfirm(noFilterIdentifier)) + select { + case <-parked: + case <-time.After(watchdog): + t.Fatal("the state machine never parked inside the admission pass") + } + h.conn.Close() //nolint:errcheck // asserted via the observer + select { + case <-handedOffErr: + case <-time.After(watchdog): + t.Fatal("the pump never handed off the cancellation error") + } + close(gate) + h.join() + select { + case err := <-disconnected: + t.Fatalf("Disconnected(%v) fired after Close returned: the pass dispatched the consumer's own cancellation as a socket failure", err) + default: + } + if _, terminal, elements := h.snapshot(); terminal != nil || elements != 0 { + t.Fatalf("clean close: got %d elements, terminal %v", elements, terminal) + } +} + +// TestCloseFromTheDrainBodyStopsTheScan: a consumer that closes the connector +// from the loop body while a drained event is in hand, and then continues the +// range, re-enters the drain's protocol-fatal scan. The scan received every +// frame the pump had queued since — dispatching a queued protocol-fatal +// disconnect to Observer.Disconnected, admitting events into a buffer the +// next run never reads and, when that admission dropped, calling +// Observer.BufferOverflow — all after Close returned. The scan's receive now +// takes the Closed edge like every other. The fatal frame is queued FIRST: it +// bypasses admitLive's own guard, so it is what pins the scan's check rather +// than the admission's. +func TestCloseFromTheDrainBodyStopsTheScan(t *testing.T) { + var overflows atomic.Int32 + disconnected := make(chan error, 4) + handedOff := make(chan struct{}, 8) + handedOffErr := make(chan struct{}, 4) + var conn *feedtest.Conn + h := newHarness(t, + eventfeed.WithStart(eventfeed.StartPresent()), + eventfeed.WithLiveBufferCapacity(1), + eventfeed.WithObserver(eventfeed.Observer{ + BufferOverflow: func(int) { overflows.Add(1) }, + Disconnected: func(_ string, err error) { + select { + case disconnected <- err: + default: + } + }, + })) + h.conn.OnPumpHandedOff(func(isErr bool) { + ch := handedOff + if isErr { + ch = handedOffErr + } + select { + case ch <- struct{}{}: + default: + } + }) + h.pauseAfter = 1 + h.minter.ScriptTicket(ticket(1)) + var once sync.Once + h.polls.OnCall(func(feedtest.PollCall) { + once.Do(func() { + // One event admitted while the entry poll is in flight: the drain + // has something to deliver, and the collector parks holding it. + drain(handedOff) + conn.Serve(frameMessage(noFilterIdentifier, 101)) + select { + case <-handedOff: + case <-time.After(watchdog): + t.Error("the straggler never reached the hand-off queue") + } + }) + }) + h.polls.ScriptPage(eventfeed.PollPage{Position: "pos-1"}) + h.start() + conn = h.driveToSubscribed() + conn.Serve(frameConfirm(noFilterIdentifier)) + h.waitUntil("the collector to park mid-drain", func() bool { return len(h.deliveredIDs()) == 1 }) + drain(handedOff) + // A fatal disconnect and two more events queued while the collector + // holds the drained one: dispatching the fatal calls Disconnected, and + // admitting both events into a capacity-1 buffer drops the first — each + // the post-Close work the scan must not do. (Queued before the Close, + // because a cancelled read returns its error ahead of any pending frame — + // nothing served after Close reaches the queue.) + conn.Serve(frameDisconnect("invalid_event_stream_command", false)) + conn.Serve(frameMessage(noFilterIdentifier, 102)) + conn.Serve(frameMessage(noFilterIdentifier, 103)) + for range 3 { + select { + case <-handedOff: + case <-time.After(watchdog): + t.Fatal("a queued frame never reached the hand-off queue") + } + } + h.conn.Close() //nolint:errcheck // asserted via the observer + select { + case <-handedOffErr: + case <-time.After(watchdog): + t.Fatal("the pump never handed off the cancellation error") + } + h.resume() + h.join() + select { + case err := <-disconnected: + t.Fatalf("Disconnected(%v) fired after Close returned from the drain's loop body: the scan dispatched a queued verdict", err) + default: + } + if n := overflows.Load(); n != 0 { + t.Fatalf("BufferOverflow fired %d time(s) after Close returned from the drain's loop body", n) + } + if _, terminal, elements := h.snapshot(); terminal != nil || elements != 1 { + t.Fatalf("clean close after one delivery: got %d elements, terminal %v", elements, terminal) + } +} diff --git a/go/pkg/basecamp/eventfeed/loop.go b/go/pkg/basecamp/eventfeed/loop.go index a8023a5ca4..1bb35048ca 100644 --- a/go/pkg/basecamp/eventfeed/loop.go +++ b/go/pkg/basecamp/eventfeed/loop.go @@ -1441,6 +1441,19 @@ func (l *loop) dispatchDisconnect(at *attempt, deadline Timer, f frame) cycleOut // something to park until a later cut — a cut a stalled or failing call may // never reach. func (l *loop) admitLive(at *attempt, deadline Timer, ev Event) (cycleOutcome, bool) { + // Close outranks the admission itself. Every path that admits is a + // receive from the pump's queue, and after Close the queue is being + // drained toward a buffer the next run never reads — an admission then + // can only manufacture a drop, and with it a BufferOverflow observer call + // and a handler dispatch that run after Close returned. Checked HERE, at + // the one place every admission passes, rather than only at the arms that + // happen to receive: an accepting handler that closed the connector is + // then refused its next admission wherever the pass it returned into + // receives it, instead of continuing until that pass's own dispatch point. + if l.runCtx.Err() != nil { + l.disposeAttempt(at, deadline) + return cycleOutcome{kind: outcomeClosed}, true + } dropped := l.buffer.add(ev) if len(dropped) == 0 { return cycleOutcome{}, false From c26f490b4ed97abc7606bf5c6d799a9d4247f759 Mon Sep 17 00:00:00 2001 From: Jeremy Daer Date: Thu, 10 Sep 2026 00:50:44 -0700 Subject: [PATCH 39/51] Event feed: a verdict already in outranks the act that would hide it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Four places acted on a socket whose verdict was already rendered, and each made that verdict look like something else. The walk's page boundary probed the socket before dispatching the outcome parked during the page's own poll. A deferral latches a grace wake on the staleness timer, so a page whose delivery ran past the grace deadline reached the boundary with that wake fired; the probe read it as an expiry, disposed the attempt — which clears the slot — and reported staleness where the parked verdict was the server's own invalid_event_stream_command. The boundary now dispatches a parked verdict first, and only when one is parked: the fatal probe admits what it dequeues, so with nothing parked the ordinary order — the expired window before any admission — stands, or an overflow the probe manufactured would end the cycle ahead of a staleness verdict already in. The walk issued its first poll without probing staleness, so a window that expired during Observer.CatchUpStarted, or during a re-entry's callbacks, was deferred as if it had fired mid-call and granted a grace phase, with the page possibly accepted first. The repair cadence and the poll-retry arms already probe before entering the walk; the walk's own entries do now. The subscribe write's wait has no frame arm, because a write in flight cannot dispatch frames without re-entering the handshake it belongs to. A conforming transport is only required to unblock a write on Close or cancellation, so a verdict the pump queued behind a blocked write died with the socket when the wait ended for its own reason, and the connector reconnected into a rejection §23 says is terminal with zero reconnects: a raw invalid_event_stream_command, or a correlated reject_subscription queued behind a duplicate welcome's blocked retransmit. The same loss turned out to exist wherever the connector tears the attempt down for a LOCAL reason with the server's last word already queued behind it — a confirmation deadline against a queued rejection, an expired window against a queued fatal, a garbled frame ahead of a queued fatal, a lapse ahead of a queued pre-welcome `unauthorized` that the shared counter needed — and which arm won the select, or which frame arrived first, decided it. So the rule is stated once: every recoverable teardown (failSocket, lapse, staleTeardown) scans the queue for the server's own verdict before disposing — any disconnect frame, dispatched by reason, or a correlated rejection — and a frame arm holding one dispatches it ahead of its expiry probe. The non-blocking probe those arms each spelled out is one helper now, expiredStaleness, and the StaleConnection-then-Disconnected pair every staleness edge reports is observeStale; the five inline copies and seven pairs are gone. Eight tests pin the defects, each red against the previous revision. The written-arm probe carries no test of its own: the fake transport's write and the fired window cannot be made ready together from outside the select, and a test that passes for either behavior is worse than none. --- go/pkg/basecamp/eventfeed/catchup.go | 136 +++++------ go/pkg/basecamp/eventfeed/catchup_test.go | 239 +++++++++++++++++++ go/pkg/basecamp/eventfeed/loop.go | 271 +++++++++++++++++----- go/pkg/basecamp/eventfeed/loop_test.go | 207 +++++++++++++++++ 4 files changed, 713 insertions(+), 140 deletions(-) diff --git a/go/pkg/basecamp/eventfeed/catchup.go b/go/pkg/basecamp/eventfeed/catchup.go index 7c665d1931..5b2c898ccc 100644 --- a/go/pkg/basecamp/eventfeed/catchup.go +++ b/go/pkg/basecamp/eventfeed/catchup.go @@ -168,6 +168,15 @@ func (l *loop) walk(at *attempt, cursor Cursor, presentClass bool) (out cycleOut return cycleOutcome{kind: outcomeTerminal, term: terr}, "", true } } + // An expiry already in before the poll is issued is transition 21 + // now, not a deferral: issued anyway, the call would make the rendered + // verdict look in-flight — granted a grace phase, its page possibly + // accepted first. The repair cadence and the poll-retry arms make the + // same check before entering the walk; this covers the walk's own + // entries — after CatchUpStarted, and after a re-entry's callbacks. + if out, done := l.expiredStaleness(at, nil); done { + return out, "", true + } p := l.pollPage(at, cursor) if p.ended { // Servicing the socket during the call ended the cycle: an @@ -367,6 +376,26 @@ func (l *loop) walk(at *attempt, cursor Cursor, presentClass bool) (out cycleOut // observed. Following `next` on a dead socket would walk the whole // frozen head before noticing, delaying the reconnect cycle by the // length of the walk. + // + // A verdict PARKED during that call is dispatched before the socket is + // probed afresh. The deferral latched a grace wake on the staleness + // timer, and a page whose delivery ran past that deadline finds the + // wake fired: probing first reads it as an expiry, disposes the + // attempt — which clears the slot — and reports staleness where the + // parked verdict was the server's own, a protocol-fatal disconnect + // reconnected into. Only an OCCUPIED slot takes this path: the fatal + // probe admits what it dequeues, and with nothing parked the ordinary + // order — the expired window before any admission — is the right one, + // or an overflow the scan manufactured would end the cycle ahead of a + // staleness verdict that was already in. + if l.deferred != nil { + if out, done := l.probeFatal(at); done { + return out, "", true + } + if out, done := l.dispatchDeferred(at); done { + return out, "", true + } + } if out, done := l.socketCheck(at); done { return out, "", true } @@ -750,12 +779,7 @@ func (l *loop) dispatchDeferred(at *attempt) (cycleOutcome, bool) { l.deferred = nil switch { case d.stale: - l.disposeAttempt(at, nil) - if l.cfg.observer.StaleConnection != nil { - l.cfg.observer.StaleConnection(d.age) - } - l.observeDisconnected("", errStaleConnection) - return cycleOutcome{kind: outcomeFailed}, true + return l.staleTeardown(at, nil, d.age), true case d.closed: return l.pumpExited(at, nil), true } @@ -785,18 +809,8 @@ func (l *loop) ownershipCut(at *attempt) (cycleOutcome, bool) { // pump queued — a socket failure, a disconnect frame, or live events for the // buffer. func (l *loop) socketCheck(at *attempt) (cycleOutcome, bool) { - staleTimer, staleGen := at.lc.stale.current() - select { - case <-staleTimer.C(): - if age, ok := at.lc.stale.evaluate(staleGen); ok { - l.disposeAttempt(at, nil) - if l.cfg.observer.StaleConnection != nil { - l.cfg.observer.StaleConnection(age) - } - l.observeDisconnected("", errStaleConnection) - return cycleOutcome{kind: outcomeFailed}, true - } - default: + if out, done := l.expiredStaleness(at, nil); done { + return out, true } return l.admissionPass(at) } @@ -987,7 +1001,7 @@ func (l *loop) fatalScan(at *attempt, budget *int) (cycleOutcome, bool) { if d := l.deferred; d != nil && !d.closed && !d.stale { if f, ok := protocolFatalFrame(d.item); ok { l.deferred = nil - return l.terminateProtocolFatal(at, f), true + return l.terminateProtocolFatal(at, nil, f), true } } // An occupied slot no longer ENDS the scan (#760). It used to, on the @@ -1025,7 +1039,7 @@ func (l *loop) fatalScan(at *attempt, budget *int) (cycleOutcome, bool) { case handled: default: if f, ok := protocolFatalFrame(item); ok { - return l.terminateProtocolFatal(at, f), true + return l.terminateProtocolFatal(at, nil, f), true } l.deferForDrain(&deferredFrame{item: item}) } @@ -1100,11 +1114,11 @@ func (l *loop) deferForDrain(d *deferredFrame) { // terminateProtocolFatal takes the raw disconnect's state-generic terminal // through the ordinary dispatch, so the frame is counted as handled and the // disconnect observation carries the server's reason verbatim. -func (l *loop) terminateProtocolFatal(at *attempt, f frame) cycleOutcome { +func (l *loop) terminateProtocolFatal(at *attempt, pending Timer, f frame) cycleOutcome { if l.hooks.frameHandled != nil { l.hooks.frameHandled(f.kind) } - return l.dispatchDisconnect(at, nil, f) + return l.dispatchDisconnect(at, pending, f) } // protocolFatalFrame reports whether one pump item is a raw @@ -1144,17 +1158,8 @@ func (l *loop) stream(at *attempt) cycleOutcome { // accepted first — where transition 25 fires directly from every // socket-open state, staleness among its triggers. evaluate // arbitrates as ever. - select { - case <-staleTimer.C(): - if age, authoritative := at.lc.stale.evaluate(staleGen); authoritative { - l.disposeAttempt(at, repair) - if l.cfg.observer.StaleConnection != nil { - l.cfg.observer.StaleConnection(age) - } - l.observeDisconnected("", errStaleConnection) - return cycleOutcome{kind: outcomeFailed} - } - default: + if out, done := l.expiredStaleness(at, repair); done { + return out } // Transition 24 → CatchingUp: one repair walk from the connector's // current position, returning here through Draining. The next @@ -1177,12 +1182,7 @@ func (l *loop) stream(at *attempt) cycleOutcome { continue } // Transition 25's staleness trigger. - l.disposeAttempt(at, repair) - if l.cfg.observer.StaleConnection != nil { - l.cfg.observer.StaleConnection(age) - } - l.observeDisconnected("", errStaleConnection) - return cycleOutcome{kind: outcomeFailed} + return l.staleTeardown(at, repair, age) case item, ok := <-at.lc.frames: if !ok { return l.pumpExited(at, repair) @@ -1202,21 +1202,15 @@ func (l *loop) stream(at *attempt) cycleOutcome { // already in, where §23 pins the fired deadline as authoritative // over frames that did not reset it. evaluate is the arbiter: a // frame the pump received FIRST moved the generation, so its - // firing is not evidence and the frame delivers as ever. - select { - case <-staleTimer.C(): - if age, authoritative := at.lc.stale.evaluate(staleGen); authoritative { - // Transition 25 first; the frame is discarded with the - // socket — the reconnect walk repairs anything the - // stream carried. - l.disposeAttempt(at, repair) - if l.cfg.observer.StaleConnection != nil { - l.cfg.observer.StaleConnection(age) - } - l.observeDisconnected("", errStaleConnection) - return cycleOutcome{kind: outcomeFailed} - } - default: + // firing is not evidence and the frame delivers as ever. Transition + // 25 first; the frame is discarded with the socket — the reconnect + // walk repairs anything the stream carried — unless the frame IS + // the server's own terminal verdict, which outranks the expiry. + if f, ok := l.serverVerdict(item); ok { + return l.dispatchServerVerdict(at, repair, f) + } + if out, done := l.expiredStaleness(at, repair); done { + return out } if out, done := l.handleLiveFrame(at, repair, item, true); done { return out @@ -1274,17 +1268,13 @@ func (l *loop) handleLiveFrame(at *attempt, pending Timer, item pumpItem, delive if item.err != nil { // Socket failure: peer close, read error, or the transport's // frame-size rejection (transitions 21/25). - l.disposeAttempt(at, pending) - l.observeDisconnected("", item.err) - return cycleOutcome{kind: outcomeFailed}, true + return l.failSocket(at, pending, item.err), true } f, err := parseFrame(item.data) if err != nil { // Invalid-frame class, parse shape: a socket-failure dispatch, never // terminal, never a silent skip. - l.disposeAttempt(at, pending) - l.observeDisconnected("", err) - return cycleOutcome{kind: outcomeFailed}, true + return l.failSocket(at, pending, err), true } if l.hooks.frameHandled != nil { l.hooks.frameHandled(f.kind) @@ -1302,9 +1292,7 @@ func (l *loop) handleLiveFrame(at *attempt, pending Timer, item pumpItem, delive ev, derr := decodeMessageEvent(f.message) if derr != nil { // Invalid-frame class, decode shape. - l.disposeAttempt(at, pending) - l.observeDisconnected("", derr) - return cycleOutcome{kind: outcomeFailed}, true + return l.failSocket(at, pending, derr), true } if !deliverLive { return l.admitLive(at, pending, ev) @@ -1462,19 +1450,8 @@ func (l *loop) waitPollRetry(at *attempt, d time.Duration) (cycleOutcome, bool) // expired window takes transition 21 now, exactly as it would // have had its case won the select. The retry timer has fired // and is no longer outstanding, so disposal is handed nil. - select { - case <-staleTimer.C(): - if age, ok := at.lc.stale.evaluate(staleGen); ok { - l.disposeAttempt(at, nil) - if l.cfg.observer.StaleConnection != nil { - l.cfg.observer.StaleConnection(age) - } - l.observeDisconnected("", errStaleConnection) - return cycleOutcome{kind: outcomeFailed}, true - } - // Superseded or suspended: not evidence, and evaluate has - // re-armed the window — proceed with the retry. - default: + if out, done := l.expiredStaleness(at, nil); done { + return out, true } return cycleOutcome{}, false case <-at.lc.stale.rearmed(): @@ -1485,12 +1462,7 @@ func (l *loop) waitPollRetry(at *attempt, d time.Duration) (cycleOutcome, bool) continue } // Transition 21's staleness trigger. - l.disposeAttempt(at, t) - if l.cfg.observer.StaleConnection != nil { - l.cfg.observer.StaleConnection(age) - } - l.observeDisconnected("", errStaleConnection) - return cycleOutcome{kind: outcomeFailed}, true + return l.staleTeardown(at, t, age), true case item, ok := <-at.lc.frames: if !ok { return l.pumpExited(at, t), true diff --git a/go/pkg/basecamp/eventfeed/catchup_test.go b/go/pkg/basecamp/eventfeed/catchup_test.go index 76a1cc55b2..8d055c4b74 100644 --- a/go/pkg/basecamp/eventfeed/catchup_test.go +++ b/go/pkg/basecamp/eventfeed/catchup_test.go @@ -16,6 +16,7 @@ import ( "slices" "strings" "sync" + "sync/atomic" "testing" "time" @@ -3517,3 +3518,241 @@ func TestAuthAndFilterTerminalsCarryNoContinuationURL(t *testing.T) { assertClean(t, terminal) }) } + +// TestParkedFatalSurvivesALateGraceWakeAtThePageBoundary: a protocol-fatal +// disconnect deferred while a poll was in flight is parked in the slot, and +// the deferral latches a grace wake on the staleness timer. When the page +// returns before the grace deadline but its delivery runs past it, the wake +// has fired by the time the walk reaches the page boundary — and probing the +// socket there BEFORE dispatching the parked verdict read the wake as an +// expiry, disposed the attempt (clearing the slot), and reconnected into the +// rejection the server had already declared terminal. +func TestParkedFatalSurvivesALateGraceWakeAtThePageBoundary(t *testing.T) { + store := feedtest.NewStore() + store.Stored("pos-0") + var mu sync.Mutex + var ages []time.Duration + var h *harness + var delivered sync.Once + h = storedHarness(t, store, eventfeed.WithObserver(eventfeed.Observer{ + StaleConnection: func(age time.Duration) { + mu.Lock() + ages = append(ages, age) + mu.Unlock() + }, + PageDelivered: func(int, string) { + delivered.Do(func() { + // The page's delivery runs past the grace deadline: the + // latched wake fires before the walk reaches the boundary. + h.clock.Advance(staleAfter) + }) + }, + })) + h.minter.ScriptTicket(ticket(1)) + // A reconnecting connector ends at Terminal(mint_failed) with two mint + // calls; a correct one at Terminal(protocol_fatal) with one. + h.minter.ScriptError(&eventfeed.MintError{Kind: eventfeed.MintUnrecoverable}) + h.polls.ScriptPage(eventfeed.PollPage{Position: "pos-1", Next: testOrigin + "/999/events.json?after=1"}) + h.polls.ScriptPage(eventfeed.PollPage{Position: "pos-2"}) + h.drainHandled() + deferral := &deferralWatch{ch: make(chan struct{}, 4)} + h.conn.OnFrameDeferred(func() { deferral.ch <- struct{}{} }) + t.Cleanup(func() { deferral.report(t) }) + + var conn *feedtest.Conn + var once sync.Once + h.polls.OnCall(func(feedtest.PollCall) { + once.Do(func() { + // The fatal is parked while the first page is still in flight; + // the call then returns normally, inside the grace phase. + conn.Serve(frameDisconnect("invalid_event_stream_command", false)) + deferral.await() + }) + }) + h.start() + conn = h.driveToSubscribed() + conn.Serve(frameConfirm(noFilterIdentifier)) + h.awaitEndOrReconnect() + h.join() + + _, terminal, _ := h.snapshot() + if terminal == nil || terminal.Reason != eventfeed.ReasonProtocolFatal { + t.Fatalf("terminal = %v, want reason %q — the parked verdict was the server's own", terminal, eventfeed.ReasonProtocolFatal) + } + if got := h.minter.Calls(); got != 1 { + t.Fatalf("mint calls = %d, want 1: a protocol-fatal rejection must not reconnect", got) + } + assertPositions(t, store.Saves(), "pos-1") + mu.Lock() + defer mu.Unlock() + if len(ages) != 0 { + t.Fatalf("StaleConnection ages = %v, want none — the grace wake is not an expiry", ages) + } +} + +// TestExpiredStalenessBeforeTheEntryPollIsNotDeferred: the walk's own entry +// is the third place a rendered verdict could be made to look in-flight. A +// window that expires while Observer.CatchUpStarted runs is already in when +// the walk reaches its first poll; issuing the call anyway parked the expiry +// as a deferral, granted the call a grace phase, and accepted its page on a +// socket whose death was decided before the request was made. The walk +// probes before it polls, exactly as the repair cadence and the poll-retry +// arms do before entering it. +func TestExpiredStalenessBeforeTheEntryPollIsNotDeferred(t *testing.T) { + var mu sync.Mutex + var ages []time.Duration + var h *harness + var started sync.Once + h = newHarness(t, eventfeed.WithObserver(eventfeed.Observer{ + StaleConnection: func(age time.Duration) { + mu.Lock() + ages = append(ages, age) + mu.Unlock() + }, + CatchUpStarted: func(eventfeed.Cursor) { + started.Do(func() { h.clock.Advance(staleAfter) }) + }, + })) + h.minter.ScriptTicket(ticket(1)) + h.minter.ScriptError(&eventfeed.MintError{Kind: eventfeed.MintUnrecoverable}) + h.polls.ScriptPage(eventfeed.PollPage{Position: "pos-1"}) + h.start() + conn := h.driveToSubscribed() + conn.Serve(frameConfirm(noFilterIdentifier)) + h.awaitEndOrReconnect() + h.join() + + if got := h.polls.CallCount(); got != 0 { + t.Fatalf("poll calls = %d, want 0: the expired window was in before the entry poll, so transition 21 fires directly", got) + } + if !conn.Closed() { + t.Fatal("the stale socket was not torn down") + } + mu.Lock() + defer mu.Unlock() + if len(ages) != 1 || ages[0] < staleAfter { + t.Fatalf("StaleConnection ages = %v, want one at or past the window", ages) + } +} + +// TestExpiredStalenessAtThePageBoundaryOutranksAQueuedOverflow: with nothing +// parked, the page boundary's order is the expired window first, then the +// admission pass. Running the fatal probe unconditionally ahead of it admitted +// what the pump had queued before the window was honoured, and an admission +// that dropped — capacity 1, one event still buffered from the poll — ended +// the cycle at Terminal(buffer_overflow) where transition 21 was already due: +// a recoverable staleness turned terminal by the probe's own admission. +func TestExpiredStalenessAtThePageBoundaryOutranksAQueuedOverflow(t *testing.T) { + store := feedtest.NewStore() + store.Stored("pos-0") + var mu sync.Mutex + var ages []time.Duration + // Rendezvous by ABSOLUTE hand-off count, not by draining anonymous + // notifications: the pump publishes a notification after its send, so a + // late one for an earlier frame could satisfy a wait meant for a later + // one. Welcome, confirm, 101 and 102 are the four frames this socket + // ever carries. + var handedOff atomic.Int32 + var conn *feedtest.Conn + var h *harness + var delivered sync.Once + h = storedHarness(t, store, + eventfeed.WithLiveBufferCapacity(1), + eventfeed.WithObserver(eventfeed.Observer{ + StaleConnection: func(age time.Duration) { + mu.Lock() + ages = append(ages, age) + mu.Unlock() + }, + PageDelivered: func(int, string) { + delivered.Do(func() { + // The window expires during the page's delivery, and a + // second event reaches the queue after it: the boundary + // must honour the expiry before admitting anything. + h.clock.Advance(staleAfter) + conn.Serve(frameMessage(noFilterIdentifier, 102)) + h.waitUntil("the second event to be handed off", func() bool { return handedOff.Load() >= 4 }) + }) + }, + })) + h.conn.OnPumpHandedOff(func(isErr bool) { + if !isErr { + handedOff.Add(1) + } + }) + h.minter.ScriptTicket(ticket(1)) + h.minter.ScriptError(&eventfeed.MintError{Kind: eventfeed.MintUnrecoverable}) + h.polls.ScriptPage(eventfeed.PollPage{Position: "pos-1", Next: testOrigin + "/999/events.json?after=1"}) + h.drainHandled() + var once sync.Once + h.polls.OnCall(func(feedtest.PollCall) { + once.Do(func() { + // One event admitted while the page is in flight, so a second + // admission at capacity 1 is a drop. + conn.Serve(frameMessage(noFilterIdentifier, 101)) + h.waitUntil("the first event to be handed off", func() bool { return handedOff.Load() >= 3 }) + }) + }) + h.start() + conn = h.driveToSubscribed() + conn.Serve(frameConfirm(noFilterIdentifier)) + h.awaitEndOrReconnect() + h.join() + + _, terminal, _ := h.snapshot() + if terminal == nil || terminal.Reason != eventfeed.ReasonMintFailed { + t.Fatalf("terminal = %v, want the reconnect's scripted %q — a recoverable staleness must not become buffer_overflow", terminal, eventfeed.ReasonMintFailed) + } + mu.Lock() + defer mu.Unlock() + if len(ages) != 1 { + t.Fatalf("StaleConnection ages = %v, want exactly one: the expired window was the boundary's verdict", ages) + } +} + +// TestQueuedFatalOutranksARecoverableFrameAheadOfIt: a malformed frame is a +// recoverable socket failure, and dispatching it used to end the cycle with +// the pump's queue never inspected — so an invalid_event_stream_command +// queued right behind it died with the socket and the connector reconnected +// into the rejection. The recoverable teardown scans first. +func TestQueuedFatalOutranksARecoverableFrameAheadOfIt(t *testing.T) { + h := newHarness(t) + h.minter.ScriptTicket(ticket(1)) + h.minter.ScriptError(&eventfeed.MintError{Kind: eventfeed.MintUnrecoverable}) + h.polls.ScriptPage(eventfeed.PollPage{Position: "pos-1"}) + handedOff := make(chan struct{}, 8) + h.conn.OnPumpHandedOff(func(bool) { + select { + case handedOff <- struct{}{}: + default: + } + }) + h.pauseAfter = 1 + h.start() + conn := h.driveToSubscribed() + conn.Serve(frameConfirm(noFilterIdentifier)) + h.awaitStreaming() + conn.Serve(frameMessage(noFilterIdentifier, 101)) + h.waitUntil("the collector to park mid-delivery", func() bool { return len(h.deliveredIDs()) == 1 }) + drain(handedOff) + conn.Serve([]byte("{not json")) + conn.Serve(frameDisconnect("invalid_event_stream_command", false)) + for range 2 { + select { + case <-handedOff: + case <-time.After(watchdog): + t.Fatal("a frame never reached the hand-off queue") + } + } + h.resume() + h.awaitEndOrReconnect() + h.join() + + _, terminal, _ := h.snapshot() + if terminal == nil || terminal.Reason != eventfeed.ReasonProtocolFatal { + t.Fatalf("terminal = %v, want reason %q — the fatal was queued behind the garbled frame", terminal, eventfeed.ReasonProtocolFatal) + } + if got := h.minter.Calls(); got != 1 { + t.Fatalf("mint calls = %d, want 1", got) + } +} diff --git a/go/pkg/basecamp/eventfeed/loop.go b/go/pkg/basecamp/eventfeed/loop.go index 1bb35048ca..d85a7f78e4 100644 --- a/go/pkg/basecamp/eventfeed/loop.go +++ b/go/pkg/basecamp/eventfeed/loop.go @@ -1105,11 +1105,12 @@ func (l *loop) awaitConfirmation(at *attempt, deadline Timer) cycleOutcome { case <-deadline.C(): // Transition 9 (handshake lapse) or 14 (confirmation lapse): // full teardown — conn, pump, and ALL the attempt's timers — - // then a jittered fresh-ticket retry. - lapsed := errDeadlineLapsed(l.state) - l.disposeAttempt(at, nil) - l.observeDisconnected("", lapsed) - return cycleOutcome{kind: outcomeFailed} + // then a jittered fresh-ticket retry. A verdict the pump has + // already queued outranks the lapse: the frame arm would have + // dispatched it, but a deadline firing in the same instant is a + // both-ready select, and losing that coin flip must not turn a + // terminal rejection into a reconnect. + return l.lapse(at) case <-at.lc.stale.rearmed(): continue case <-staleTimer.C(): @@ -1120,12 +1121,7 @@ func (l *loop) awaitConfirmation(at *attempt, deadline Timer) cycleOutcome { continue } // Staleness expiry — rows 9/15's staleness trigger. - l.disposeAttempt(at, deadline) - if l.cfg.observer.StaleConnection != nil { - l.cfg.observer.StaleConnection(age) - } - l.observeDisconnected("", errStaleConnection) - return cycleOutcome{kind: outcomeFailed} + return l.staleTeardown(at, deadline, age) case item, ok := <-at.lc.frames: if !ok { // The pump exited without a terminating error — only @@ -1152,18 +1148,14 @@ func (l *loop) awaitConfirmation(at *attempt, deadline Timer) cycleOutcome { // the teardown and resurfaces after reconnect — or hands a // confirmation a live catch-up on a socket whose staleness // verdict is already in. evaluate arbitrates: a frame the pump - // received first moved the generation and is handled as ever. - select { - case <-staleTimer.C(): - if age, authoritative := at.lc.stale.evaluate(staleGen); authoritative { - l.disposeAttempt(at, deadline) - if l.cfg.observer.StaleConnection != nil { - l.cfg.observer.StaleConnection(age) - } - l.observeDisconnected("", errStaleConnection) - return cycleOutcome{kind: outcomeFailed} - } - default: + // received first moved the generation and is handled as ever — + // except a terminal verdict in hand, which outranks the expiry as + // it outranks every recoverable teardown. + if f, ok := l.serverVerdict(item); ok { + return l.dispatchServerVerdict(at, deadline, f) + } + if out, done := l.expiredStaleness(at, deadline); done { + return out } if out, done := l.handleFrame(at, &deadline, item); done { return out @@ -1219,10 +1211,19 @@ func (l *loop) writeSubscribe(at *attempt, deadline *Timer) (cycleOutcome, bool) l.disposeAttempt(at, *deadline) return cycleOutcome{kind: outcomeClosed}, true } + // The expired window outranks the write's result too: both can be + // ready, and accepting a completed subscribe on a socket whose + // staleness verdict is already in announces AwaitingConfirmation + // and arms a confirmation deadline for one select turn on a dead + // socket — or reports a failed write as the failure, where the + // verdict was the window's. The teardown it takes is the stale + // arm's, queued-verdict scan included: which arm won the select + // must not decide whether a queued rejection is honoured. + if out, done := l.expiredStaleness(at, *deadline); done { + return out, true + } if werr != nil { - l.disposeAttempt(at, *deadline) - l.observeDisconnected("", werr) - return cycleOutcome{kind: outcomeFailed}, true + return l.failSocket(at, *deadline, werr), true } if l.hooks.subscribeWritten != nil { l.hooks.subscribeWritten() @@ -1232,27 +1233,23 @@ func (l *loop) writeSubscribe(at *attempt, deadline *Timer) (cycleOutcome, bool) l.disposeAttempt(at, *deadline) return cycleOutcome{kind: outcomeClosed}, true case <-(*deadline).C(): - // The deadline is spent, so disposal is handed nil: the attempt's - // timer set is empty from here, as on every other lapse. - lapsed := errDeadlineLapsed(l.state) - l.disposeAttempt(at, nil) - l.observeDisconnected("", lapsed) - return cycleOutcome{kind: outcomeFailed}, true + // The server's own verdict outranks the lapse: this wait has no + // frame arm — a write in flight cannot dispatch frames without + // re-entering the handshake it is part of — so a disconnect or a + // correlated rejection the pump queued while a conforming + // transport kept the write blocked would otherwise die with the + // socket, and the connector would reconnect into a verdict §23 + // says is terminal with zero reconnects. lapse scans first. + return l.lapse(at), true case <-at.lc.stale.rearmed(): case <-staleTimer.C(): age, ok := at.lc.stale.evaluate(staleGen) if !ok { continue } - // Rows 9/15's staleness trigger, observed mid-write: full - // teardown, and disposal's cancel is what returns the abandoned - // write. - l.disposeAttempt(at, *deadline) - if l.cfg.observer.StaleConnection != nil { - l.cfg.observer.StaleConnection(age) - } - l.observeDisconnected("", errStaleConnection) - return cycleOutcome{kind: outcomeFailed}, true + // Rows 9/15's staleness trigger, observed mid-write; disposal's + // cancel is what returns the abandoned write. + return l.staleTeardown(at, *deadline, age), true } } } @@ -1265,6 +1262,84 @@ func errDeadlineLapsed(s connState) error { return errors.New("event feed confirmation deadline lapsed before confirm_subscription") } +// queuedServerVerdict scans what the pump has already handed off — plus the +// one frame a blocked hand-off is holding, the same pumpDepth+1 bound the +// drain's scan carries — for the server's own verdict on the socket +// (serverVerdict), and dispatches it when one is found. It runs wherever the +// connector is about to tear the attempt down for a LOCAL reason — a lapsed +// deadline, an expired window, a read or write error, an invalid frame — +// because a queued disconnect or rejection is the server's last word and +// outranks every one of those: a garbled frame ahead of a queued +// invalid_event_stream_command reconnected into the rejection, a lapse +// ahead of a queued pre-welcome `unauthorized` skipped the shared counter. +// Everything else it dequeues is discarded: every other frame is either +// liveness, which the teardown makes moot, or a socket outcome that ends the +// attempt exactly as the teardown is about to. +func (l *loop) queuedServerVerdict(at *attempt, pending Timer) (cycleOutcome, bool) { + for range pumpDepth + 1 { + select { + case item, ok := <-at.lc.frames: + if !ok { + return cycleOutcome{}, false + } + if f, ok := l.serverVerdict(item); ok { + return l.dispatchServerVerdict(at, pending, f), true + } + default: + return cycleOutcome{}, false + } + } + return cycleOutcome{}, false +} + +// serverVerdict classifies one pump item as the server's own verdict on the +// socket, which outranks any local reason for tearing it down: a raw +// disconnect frame of ANY reason — the reason-string dispatch decides what +// it means (protocol-fatal terminal from every state; a pre-welcome +// `unauthorized` increments the shared counter and is terminal at its +// threshold; `remote` and the rest a socket drop), and only the dispatch may +// decide it, since a scan that recognised the fatal alone would discard an +// `unauthorized` the counter needed — or a correlated `reject_subscription` +// once a subscribe has been sent (transition 12 is drawn from +// AwaitingConfirmation alone, exactly as handleFrame gates it). +func (l *loop) serverVerdict(item pumpItem) (frame, bool) { + if item.err != nil { + return frame{}, false + } + f, err := parseFrame(item.data) + if err != nil { + return frame{}, false + } + switch { + case f.kind == frameDisconnect: + return f, true + case f.kind == frameReject && l.state == stateAwaitingConfirmation && f.identifier == l.identifier: + return f, true + } + return frame{}, false +} + +// dispatchServerVerdict takes a verdict serverVerdict classified: a +// disconnect through the ordinary reason dispatch, the rejection as +// transition 12 — deadline cancelled, socket explicitly closed, zero +// reconnects. +func (l *loop) dispatchServerVerdict(at *attempt, pending Timer, f frame) cycleOutcome { + if f.kind == frameDisconnect { + if l.hooks.frameHandled != nil { + l.hooks.frameHandled(f.kind) + } + return l.dispatchDisconnect(at, pending, f) + } + if l.hooks.frameHandled != nil { + l.hooks.frameHandled(f.kind) + } + l.disposeAttempt(at, pending) + return cycleOutcome{kind: outcomeTerminal, term: &TerminalError{ + Reason: ReasonSubscriptionRejected, + Msg: "the server rejected the EventsChannel subscription", + }} +} + // handleFrame dispatches one pump item in AwaitingWelcome / // AwaitingConfirmation. It returns done=true when the cycle is over — // including the confirmed handoff's own outcome flowing back through it. @@ -1272,18 +1347,14 @@ func (l *loop) handleFrame(at *attempt, deadline *Timer, item pumpItem) (cycleOu if item.err != nil { // Socket failure: peer close, read error, or the transport's // frame-size rejection (rows 9/15). - l.disposeAttempt(at, *deadline) - l.observeDisconnected("", item.err) - return cycleOutcome{kind: outcomeFailed}, true + return l.failSocket(at, *deadline, item.err), true } f, err := parseFrame(item.data) if err != nil { // Invalid-frame class, parse shape: a peer protocol violation // dispatched as a socket failure — never terminal, never a silent // skip (SPEC.md §23 "Cable Protocol Details"). - l.disposeAttempt(at, *deadline) - l.observeDisconnected("", err) - return cycleOutcome{kind: outcomeFailed}, true + return l.failSocket(at, *deadline, err), true } if l.hooks.frameHandled != nil { l.hooks.frameHandled(f.kind) @@ -1306,10 +1377,7 @@ func (l *loop) handleFrame(at *attempt, deadline *Timer, item pumpItem) (cycleOu // confirmation window and accept a welcome that arrived past // the deadline — transition 9's lapse, silently converted // into a live handshake half the time it happens. - lapsed := errDeadlineLapsed(l.state) - l.disposeAttempt(at, nil) - l.observeDisconnected("", lapsed) - return cycleOutcome{kind: outcomeFailed}, true + return l.lapse(at), true } *deadline = l.cfg.clock.NewTimer(l.cfg.confirmationDeadline, timerConfirmationDeadline) at.phase = *deadline @@ -1335,10 +1403,7 @@ func (l *loop) handleFrame(at *attempt, deadline *Timer, item pumpItem) (cycleOu // subscription whose confirmation arrived past transition 14's // deadline, so the lapse is not merely swallowed but overwritten // by a successful handoff. - lapsed := errDeadlineLapsed(l.state) - l.disposeAttempt(at, nil) - l.observeDisconnected("", lapsed) - return cycleOutcome{kind: outcomeFailed}, true + return l.lapse(at), true } l.failedCycles = 0 if l.cfg.observer.Confirmed != nil { @@ -1376,9 +1441,7 @@ func (l *loop) handleFrame(at *attempt, deadline *Timer, item pumpItem) (cycleOu if derr != nil { // Invalid-frame class, decode shape: same socket-failure // disposition as the parse shape. - l.disposeAttempt(at, *deadline) - l.observeDisconnected("", derr) - return cycleOutcome{kind: outcomeFailed}, true + return l.failSocket(at, *deadline, derr), true } return l.admitLive(at, *deadline, ev) } @@ -1579,6 +1642,98 @@ func (l *loop) observeDisconnected(reason string, err error) { } } +// observeStale reports one staleness teardown: StaleConnection carrying the +// silence the verdict measured, then the same errStaleConnection disconnect +// every staleness edge reports. One helper so the pair cannot drift apart +// between the arms that fire it. +func (l *loop) observeStale(age time.Duration) { + if l.cfg.observer.StaleConnection != nil { + l.cfg.observer.StaleConnection(age) + } + l.observeDisconnected("", errStaleConnection) +} + +// expiredStaleness probes the staleness timer WITHOUT waiting on it, and takes +// the current state's staleness edge when an authoritative expiry is already +// in: full teardown (pending is the state's own timer, stopped with the +// attempt), StaleConnection, then the disconnect. A firing that is not +// evidence — superseded by a frame the pump received first, or overlapping a +// blocked hand-off — is disregarded under the same rule every select applies, +// and evaluate re-arms it. +// +// It is the arbiter every arm that is about to do something ELSE on the +// socket runs first: follow a repair cadence or a poll-retry into a walk, +// accept a subscribe write, dispatch a ready frame, or issue a poll. Each of +// those makes an already-rendered verdict look in-flight — a walk's first +// poll defers it and grants a grace phase, a frame delivers from a socket +// whose death is already decided — where §23 says the state's own failure +// edge fires directly, staleness among its triggers. +func (l *loop) expiredStaleness(at *attempt, pending Timer) (cycleOutcome, bool) { + age, ok := l.probeStaleness(at) + if !ok { + return cycleOutcome{}, false + } + return l.staleTeardown(at, pending, age), true +} + +// staleTeardown is every staleness edge's teardown: a queued terminal verdict +// first, then the full disposal and the StaleConnection-then-Disconnected +// report. The scan comes first because staleness is a recoverable verdict and +// a queued protocol-fatal disconnect or correlated rejection is not — a +// teardown that discarded one would reconnect into a rejection §23 says is +// terminal with zero reconnects, and which arm of a select observed the +// expiry must not decide that. +func (l *loop) staleTeardown(at *attempt, pending Timer, age time.Duration) cycleOutcome { + if out, done := l.queuedServerVerdict(at, pending); done { + return out + } + l.disposeAttempt(at, pending) + l.observeStale(age) + return cycleOutcome{kind: outcomeFailed} +} + +// failSocket is every recoverable socket-failure teardown — a read error, a +// peer close, an invalid frame, a failed write: the server's last word first +// if the pump has already queued it, else the full disposal and the +// Disconnected report carrying the cause. The scan is what keeps a +// recoverable outcome that arrived first from ending the cycle over a +// verdict that arrived behind it — a garbled frame ahead of a queued +// invalid_event_stream_command reconnected into the rejection. +func (l *loop) failSocket(at *attempt, pending Timer, err error) cycleOutcome { + if out, done := l.queuedServerVerdict(at, pending); done { + return out + } + l.disposeAttempt(at, pending) + l.observeDisconnected("", err) + return cycleOutcome{kind: outcomeFailed} +} + +// lapse is the phase deadline's teardown (transitions 9 and 14), the same +// scan first: the deadline is spent, so the attempt's timer set is empty +// from here and disposal is handed nil. +func (l *loop) lapse(at *attempt) cycleOutcome { + if out, done := l.queuedServerVerdict(at, nil); done { + return out + } + lapsed := errDeadlineLapsed(l.state) + l.disposeAttempt(at, nil) + l.observeDisconnected("", lapsed) + return cycleOutcome{kind: outcomeFailed} +} + +// probeStaleness is expiredStaleness's verdict without its teardown: the +// non-blocking evaluate alone, for a caller that has something to do between +// learning the window has expired and disposing the attempt. +func (l *loop) probeStaleness(at *attempt) (time.Duration, bool) { + staleTimer, staleGen := at.lc.stale.current() + select { + case <-staleTimer.C(): + return at.lc.stale.evaluate(staleGen) + default: + return 0, false + } +} + // errSocketFailed is the generic cause an unrecognized socket failure is // reported as. var errSocketFailed = errors.New("event feed socket failed") diff --git a/go/pkg/basecamp/eventfeed/loop_test.go b/go/pkg/basecamp/eventfeed/loop_test.go index 7cfb3b2f6c..4012b4915d 100644 --- a/go/pkg/basecamp/eventfeed/loop_test.go +++ b/go/pkg/basecamp/eventfeed/loop_test.go @@ -2444,3 +2444,210 @@ func TestUnauthorizedMintCarriesNoRetryAfterFloor(t *testing.T) { t.Fatalf("backoff armed for %s: an unauthorized mint's seam-supplied RetryAfter floored the reconnect delay; unauthorized carries no retry_after and the local draw alone governs", d) } } + +// TestQueuedVerdictOutranksATeardownDuringABlockedSubscribeWrite: the +// subscribe write's wait has no frame arm — a write in flight cannot dispatch +// frames without re-entering the handshake it belongs to — and a conforming +// transport is only required to unblock a write on Close or cancellation. A +// verdict the pump queued while the write stayed blocked was therefore torn +// down with the socket when the wait ended for its own reason, and the +// connector reconnected into a rejection §23 says is terminal with zero +// reconnects: a raw invalid_event_stream_command, at the handshake deadline +// or the staleness window; or a correlated reject_subscription queued behind +// a duplicate welcome's blocked retransmit, at the confirmation deadline. +func TestQueuedVerdictOutranksATeardownDuringABlockedSubscribeWrite(t *testing.T) { + for _, tc := range []struct { + name string + // arrange drives the socket to the blocked write and queues the + // verdict; it returns the frame to queue. + verdict func(identifier string) []byte + // duplicate blocks the RETRANSMIT rather than the first subscribe, + // so the verdict is correlated to a subscribe the server received. + duplicate bool + // end takes the wait's own teardown. + end func(h *harness) + reason eventfeed.TerminalReason + }{ + { + name: "protocol-fatal at the handshake deadline", + verdict: func(string) []byte { return frameDisconnect("invalid_event_stream_command", false) }, + end: func(h *harness) { h.fireTimer(timerHandshakeDeadline) }, + reason: eventfeed.ReasonProtocolFatal, + }, + { + name: "protocol-fatal at the staleness window", + verdict: func(string) []byte { return frameDisconnect("invalid_event_stream_command", false) }, + end: func(h *harness) { h.clock.Advance(staleAfter) }, + reason: eventfeed.ReasonProtocolFatal, + }, + { + name: "rejection at the confirmation deadline", + verdict: frameReject, + duplicate: true, + end: func(h *harness) { h.fireTimer(timerConfirmationDeadline) }, + reason: eventfeed.ReasonSubscriptionRejected, + }, + } { + t.Run(tc.name, func(t *testing.T) { + h := newHarness(t) + h.minter.ScriptTicket(ticket(1)) + // A reconnecting connector ends at Terminal(mint_failed) with two + // mint calls; a correct one at the verdict's terminal with one. + h.minter.ScriptError(&eventfeed.MintError{Kind: eventfeed.MintUnrecoverable}) + handedOff := make(chan struct{}, 8) + h.conn.OnPumpHandedOff(func(bool) { + select { + case handedOff <- struct{}{}: + default: + } + }) + h.start() + var conn *feedtest.Conn + if tc.duplicate { + conn = h.driveToSubscribed() + conn.StallWrites() + conn.Serve(frameWelcome()) + } else { + conn = h.liveConn() + conn.StallWrites() + conn.Serve(frameWelcome()) + } + // The duplicate's own handling is the rendezvous — the first + // welcome's notification is already queued and must not be + // mistaken for it. + h.awaitFrameHandled("welcome") + if tc.duplicate { + h.awaitFrameHandled("welcome") + } + drain(handedOff) + conn.Serve(tc.verdict(noFilterIdentifier)) + select { + case <-handedOff: + case <-time.After(watchdog): + t.Fatal("the verdict frame never reached the hand-off queue") + } + tc.end(h) + h.awaitEndOrReconnect() + h.join() + + _, terminal, _ := h.snapshot() + if terminal == nil || terminal.Reason != tc.reason { + t.Fatalf("terminal = %v, want reason %q — the verdict was already queued when the wait ended", terminal, tc.reason) + } + if got := h.minter.Calls(); got != 1 { + t.Fatalf("mint calls = %d, want 1: a terminal verdict must not reconnect", got) + } + if !conn.Closed() { + t.Fatal("the socket was not explicitly closed") + } + assertTimers(t, h.clock, map[string]int{}) + }) + } +} + +// TestQueuedUnauthorizedBehindALapseCountsTowardTheThreshold: the scan every +// recoverable teardown runs recognises a disconnect of ANY reason, not the +// protocol-fatal alone, because the reason dispatch is what decides its +// meaning. A pre-welcome `unauthorized` queued behind a blocked subscribe +// write is the case that shows why: with two unauthorized failures already +// on the shared counter, a lapse that discarded it reconnected with the +// counter still at two, where dispatching it is the third and terminal. +func TestQueuedUnauthorizedBehindALapseCountsTowardTheThreshold(t *testing.T) { + h := newHarness(t) + h.minter.ScriptError(&eventfeed.MintError{Kind: eventfeed.MintUnauthorized}) + h.minter.ScriptError(&eventfeed.MintError{Kind: eventfeed.MintUnauthorized}) + h.minter.ScriptTicket(ticket(3)) + // A fourth mint would mean the queued unauthorized was discarded. + h.minter.ScriptError(&eventfeed.MintError{Kind: eventfeed.MintUnrecoverable}) + handedOff := make(chan struct{}, 8) + h.conn.OnPumpHandedOff(func(bool) { + select { + case handedOff <- struct{}{}: + default: + } + }) + h.start() + h.fireTimer(timerBackoff) + h.fireTimer(timerBackoff) + conn := h.liveConn() + conn.StallWrites() + conn.Serve(frameWelcome()) + h.awaitFrameHandled("welcome") + drain(handedOff) + conn.Serve(frameDisconnect("unauthorized", false)) + select { + case <-handedOff: + case <-time.After(watchdog): + t.Fatal("the unauthorized disconnect never reached the hand-off queue") + } + h.fireTimer(timerHandshakeDeadline) + h.awaitEndOrReconnect() + h.join() + + _, terminal, _ := h.snapshot() + if terminal == nil || terminal.Reason != eventfeed.ReasonAuthorizationFailed { + t.Fatalf("terminal = %v, want reason %q — the queued unauthorized was the third consecutive on the shared counter", terminal, eventfeed.ReasonAuthorizationFailed) + } + if got := h.minter.Calls(); got != 3 { + t.Fatalf("mint calls = %d, want 3", got) + } +} + +// TestQueuedFatalOutranksAWelcomePastItsDeadline: a welcome whose handshake +// deadline had already fired takes the lapse through Stop's false return — +// and a protocol-fatal disconnect queued behind it is the server's verdict, +// which the lapse must not discard. The deadline is fired while the state +// machine is parked on the welcome, so the write's completion and the fired +// deadline are both ready when it resumes: whichever arm wins, the lapse it +// takes scans first. (The confirmation branch's Stop-false lapse is the same +// helper.) +func TestQueuedFatalOutranksAWelcomePastItsDeadline(t *testing.T) { + h := newHarness(t) + h.minter.ScriptTicket(ticket(1)) + h.minter.ScriptError(&eventfeed.MintError{Kind: eventfeed.MintUnrecoverable}) + handedOff := make(chan struct{}, 8) + h.conn.OnPumpHandedOff(func(bool) { + select { + case handedOff <- struct{}{}: + default: + } + }) + parked := make(chan struct{}) + gate := make(chan struct{}) + var once sync.Once + h.conn.OnFrameHandled(func(kind string) { + if kind == "welcome" { + once.Do(func() { + close(parked) + <-gate + }) + } + }) + h.start() + conn := h.liveConn() + conn.Serve(frameWelcome()) + select { + case <-parked: + case <-time.After(watchdog): + t.Fatal("the state machine never parked on the welcome") + } + h.fireTimer(timerHandshakeDeadline) + drain(handedOff) + conn.Serve(frameDisconnect("invalid_event_stream_command", false)) + select { + case <-handedOff: + case <-time.After(watchdog): + t.Fatal("the fatal never reached the hand-off queue") + } + close(gate) + h.awaitEndOrReconnect() + h.join() + + _, terminal, _ := h.snapshot() + if terminal == nil || terminal.Reason != eventfeed.ReasonProtocolFatal { + t.Fatalf("terminal = %v, want reason %q — the verdict was queued when the lapse was taken", terminal, eventfeed.ReasonProtocolFatal) + } + if got := h.minter.Calls(); got != 1 { + t.Fatalf("mint calls = %d, want 1", got) + } +} From be6e18eecde2befa78abd4a1754e15cfd944fd55 Mon Sep 17 00:00:00 2001 From: Jeremy Daer Date: Thu, 10 Sep 2026 00:52:49 -0700 Subject: [PATCH 40/51] Event feed: a misbehaving seam fails closed instead of crashing or leaking MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two shapes of host-seam misbehavior reached the consumer's goroutine as a crash or a leak where the documented outcome exists. A seam returning a nil *MintError, *PollError, *DialError or *CloseError through a non-nil error interface — the ordinary Go footgun — satisfied errors.As with a nil target, and the classification that followed dereferenced it. The four sites now treat a nil target as the unclassified case: the mint and poll lanes take their unclassified terminal with a named cause rather than the nil value (rendering it is the same dereference), a dial is a transient failure into backoff rather than a policy verdict read off a nil Kind, and the observable-error reduction degrades to the generic socket failure. A product Clock that panics arming the staleness timer does so inside newLiveConn, after the dial succeeded and before the attempt owns the connection. The recovery disposed a live attempt through at.lc and stopped the phase timer otherwise, so the dialed socket stayed open with its pump never started — a permanent leak in a process an outer recovery keeps alive. The connection is declared ahead of the recovery and closed on unwind when no liveConn owns it yet. Tests: each typed-nil site, and a clock that panics on its first staleness timer, all red against the previous revision. --- go/pkg/basecamp/eventfeed/cable_test.go | 9 ++++ go/pkg/basecamp/eventfeed/catchup.go | 6 +++ go/pkg/basecamp/eventfeed/connector_test.go | 60 +++++++++++++++++++++ go/pkg/basecamp/eventfeed/loop.go | 37 ++++++++++--- go/pkg/basecamp/eventfeed/loop_test.go | 55 +++++++++++++++++++ 5 files changed, 160 insertions(+), 7 deletions(-) diff --git a/go/pkg/basecamp/eventfeed/cable_test.go b/go/pkg/basecamp/eventfeed/cable_test.go index 86931e21b0..cfd81fddb4 100644 --- a/go/pkg/basecamp/eventfeed/cable_test.go +++ b/go/pkg/basecamp/eventfeed/cable_test.go @@ -788,3 +788,12 @@ func TestParseFrame_UnknownTypeIgnoresUnrelatedFields(t *testing.T) { } } } + +// A typed-nil *invalidFrameError reaching the reduction is the in-package +// half of TestTypedNilSeamErrorsFailClosed: the match must not be read as +// the value. +func TestObservableSocketErrorTypedNilInvalidFrame(t *testing.T) { + if got := observableSocketError((*invalidFrameError)(nil)); got != errSocketFailed { + t.Fatalf("observable error = %v, want errSocketFailed by identity", got) + } +} diff --git a/go/pkg/basecamp/eventfeed/catchup.go b/go/pkg/basecamp/eventfeed/catchup.go index 5b2c898ccc..cdff7d67aa 100644 --- a/go/pkg/basecamp/eventfeed/catchup.go +++ b/go/pkg/basecamp/eventfeed/catchup.go @@ -1343,6 +1343,12 @@ func (l *loop) recoverPoll(at *attempt, cursor Cursor, err error) (walkStep, cyc Reason: ReasonPollFailed, Msg: "unclassified poll failure", Err: err, }}, true } + if pe == nil { + l.disposeAttempt(at, nil) + return walkStep{}, cycleOutcome{kind: outcomeTerminal, term: &TerminalError{ + Reason: ReasonPollFailed, Msg: "unclassified poll failure", Err: errNilSeamError, + }}, true + } switch pe.Kind { case PollTransient, PollThrottled: // The self-loop inside CatchingUp: a wait, not a state change. The diff --git a/go/pkg/basecamp/eventfeed/connector_test.go b/go/pkg/basecamp/eventfeed/connector_test.go index 6acc918ce1..06eb03ce40 100644 --- a/go/pkg/basecamp/eventfeed/connector_test.go +++ b/go/pkg/basecamp/eventfeed/connector_test.go @@ -1988,3 +1988,63 @@ func TestCloseFromTheDrainBodyStopsTheScan(t *testing.T) { t.Fatalf("clean close after one delivery: got %d elements, terminal %v", elements, terminal) } } + +// panickingStaleClock wraps the harness clock so the FIRST staleness timer +// it is asked for panics — the product Clock seam failing inside newLiveConn, +// after the dial succeeded and before the attempt owns the connection. +type panickingStaleClock struct { + *feedtest.Clock + once sync.Once +} + +func (c *panickingStaleClock) NewTimer(d time.Duration, kind string) eventfeed.Timer { + panicked := false + if kind == timerStaleness { + c.once.Do(func() { panicked = true }) + } + if panicked { + panic("host clock panicked arming staleness") + } + return c.Clock.NewTimer(d, kind) +} + +// TestPanicArmingStalenessClosesTheDialedSocket: the panic recovery disposes +// a live attempt through at.lc, but a Clock that panics from NewTimer inside +// newLiveConn does so before at.lc is assigned. The recovery then saw no live +// connection, stopped the handshake timer, and left the successfully dialed +// socket open with its pump never started — a permanent leak in a process an +// outer recovery keeps alive. +func TestPanicArmingStalenessClosesTheDialedSocket(t *testing.T) { + clock := &panickingStaleClock{Clock: feedtest.NewClock()} + tr := feedtest.NewTransport() + minter := feedtest.NewMinter() + minter.ScriptTicket(ticket(1)) + c, err := eventfeed.New(testOrigin, "5951425", minter, feedtest.NewPolls(), + eventfeed.WithTransport(tr), eventfeed.WithClock(clock)) + if err != nil { + t.Fatalf("New: %v", err) + } + t.Cleanup(func() { c.Close() }) + recovered := make(chan any, 1) + go func() { + defer func() { recovered <- recover() }() + for range c.Events(context.Background()) { + } + }() + select { + case r := <-recovered: + if r == nil { + t.Fatal("the panic did not propagate out of the iteration") + } + case <-time.After(watchdog): + t.Fatal("the iteration neither panicked nor returned") + } + sock := tr.LastConn() + if sock == nil { + t.Fatal("no connection was dialed") + } + if !sock.Closed() { + t.Fatal("the dialed socket was leaked by the panic unwinding") + } + assertTimers(t, clock.Clock, map[string]int{}) +} diff --git a/go/pkg/basecamp/eventfeed/loop.go b/go/pkg/basecamp/eventfeed/loop.go index d85a7f78e4..aa5d62aa2d 100644 --- a/go/pkg/basecamp/eventfeed/loop.go +++ b/go/pkg/basecamp/eventfeed/loop.go @@ -110,6 +110,14 @@ const sincePresent = "now" // errStaleConnection reports a staleness teardown. var errStaleConnection = errors.New("event feed connection stale: no inbound frames within the staleness window") +// errNilSeamError is what a seam's typed-nil error value is classified as: a +// non-nil error interface holding a nil *MintError, *PollError, *DialError or +// *CloseError. errors.As reports a match and leaves the target nil, so the +// classification that follows would dereference it and panic on the +// consumer's goroutine. The value is not retained as a cause — rendering it +// is the same dereference. +var errNilSeamError = errors.New("event feed seam returned a typed-nil error value") + // attempt is one connect cycle's mutable ownership: its cancellation scope // (cancelled on teardown, so an in-flight seam call or dial belonging to the // attempt returns promptly) and, once dialed, the live connection. @@ -855,12 +863,23 @@ func (l *loop) runCycle(delay time.Duration) cycleOutcome { // that already disposed re-disposes harmlessly; the panic itself // propagates untouched. Phase timers held as locals are deliberately not // chased: unfired, they fire into nothing. + // conn is declared ahead of the recovery so a socket that was dialed but + // not yet owned by a liveConn — the product Clock panicking inside + // newLiveConn, between the dial and the assignment of at.lc — is closed + // on unwind rather than leaked with its pump never started. + var conn CableConn defer func() { if r := recover(); r != nil { - if at.lc != nil { + switch { + case at.lc != nil: l.disposeAttempt(at, at.phase) - } else if at.phase != nil { - at.phase.Stop() + default: + if conn != nil { + _ = conn.Close(closeCodeNormal, "") + } + if at.phase != nil { + at.phase.Stop() + } } panic(r) } @@ -920,7 +939,6 @@ func (l *loop) runCycle(delay time.Duration) cycleOutcome { conn, dialErr := l.cfg.transport.Dial(at.ctx, ticket.URL, maxFrameBytes) dialCh <- dialResult{conn: conn, err: dialErr} }() - var conn CableConn select { case r := <-dialCh: if l.runCtx.Err() != nil { @@ -941,7 +959,7 @@ func (l *loop) runCycle(delay time.Duration) cycleOutcome { _ = r.conn.Close(closeCodeNormal, "") } var derr *DialError - if errors.As(r.err, &derr) && derr.Kind == DialPolicy { + if errors.As(r.err, &derr) && derr != nil && derr.Kind == DialPolicy { // The KIND is read; the text is not. This DialError came out // of CableTransport.Dial, a documented extension point, so // its Reason and its cause are host-authored — and the dialed @@ -1058,6 +1076,11 @@ func (l *loop) classifyMintFailure(err error) cycleOutcome { Reason: ReasonMintFailed, Msg: "unclassified mint failure", Err: err, }} } + if me == nil { + return cycleOutcome{kind: outcomeTerminal, term: &TerminalError{ + Reason: ReasonMintFailed, Msg: "unclassified mint failure", Err: errNilSeamError, + }} + } switch me.Kind { case MintTransient: return cycleOutcome{kind: outcomeFailed} // transition 4 @@ -1820,7 +1843,7 @@ func observableSocketError(err error) error { return context.DeadlineExceeded } var ce *CloseError - if errors.As(err, &ce) { + if errors.As(err, &ce) && ce != nil { // REBUILT code-only, not passed along. CloseError.Error renders the // integer alone, which is what made passing the value through look // safe — but Reason is an exported field holding the peer's string, @@ -1831,7 +1854,7 @@ func observableSocketError(err error) error { return &CloseError{Code: ce.Code} } var ife *invalidFrameError - if errors.As(err, &ife) { + if errors.As(err, &ife) && ife != nil { // Flat by construction and renders only its shape, over a closed // two-value vocabulary. See invalidFrameError. return ife diff --git a/go/pkg/basecamp/eventfeed/loop_test.go b/go/pkg/basecamp/eventfeed/loop_test.go index 4012b4915d..d0711d899d 100644 --- a/go/pkg/basecamp/eventfeed/loop_test.go +++ b/go/pkg/basecamp/eventfeed/loop_test.go @@ -2651,3 +2651,58 @@ func TestQueuedFatalOutranksAWelcomePastItsDeadline(t *testing.T) { t.Fatalf("mint calls = %d, want 1", got) } } + +// TestTypedNilSeamErrorsFailClosed: a host-supplied seam that returns a nil +// *MintError, *PollError, *DialError or *CloseError through a non-nil error +// interface is a Go footgun, not a contract the connector can rely on the +// host to avoid. errors.As reports a match and leaves the target nil, and the +// classification that followed dereferenced it — a panic on the consumer's +// goroutine, where the documented outcome is the seam's unclassified terminal +// (or, for a dial, a transient failure and a reconnect). +func TestTypedNilSeamErrorsFailClosed(t *testing.T) { + t.Run("mint", func(t *testing.T) { + h := newHarness(t) + h.minter.ScriptError((*eventfeed.MintError)(nil)) + h.start() + h.join() + _, terminal, _ := h.snapshot() + if terminal == nil || terminal.Reason != eventfeed.ReasonMintFailed { + t.Fatalf("terminal = %v, want reason %q", terminal, eventfeed.ReasonMintFailed) + } + if msg := terminal.Error(); !strings.Contains(msg, "typed-nil") { + t.Fatalf("terminal renders %q, want the typed-nil cause named rather than the nil value retained", msg) + } + }) + t.Run("poll", func(t *testing.T) { + h := newHarness(t) + h.minter.ScriptTicket(ticket(1)) + h.polls.ScriptError((*eventfeed.PollError)(nil)) + h.start() + conn := h.driveToSubscribed() + conn.Serve(frameConfirm(noFilterIdentifier)) + h.join() + _, terminal, _ := h.snapshot() + if terminal == nil || terminal.Reason != eventfeed.ReasonPollFailed { + t.Fatalf("terminal = %v, want reason %q", terminal, eventfeed.ReasonPollFailed) + } + if msg := terminal.Error(); !strings.Contains(msg, "typed-nil") { + t.Fatalf("terminal renders %q, want the typed-nil cause named", msg) + } + }) + t.Run("dial", func(t *testing.T) { + h := newHarness(t) + h.minter.ScriptTicket(ticket(1)) + h.tr.FailNextDial((*eventfeed.DialError)(nil)) + h.start() + // A typed-nil dial error is a transient failure: transition 7, one + // backoff round, never a policy terminal read off a nil Kind. + h.awaitTimer(timerBackoff) + assertTimers(t, h.clock, map[string]int{timerBackoff: 1}) + }) + t.Run("close error", func(t *testing.T) { + got := eventfeed.ExportObservableSocketError((*eventfeed.CloseError)(nil)) + if got != eventfeed.ExportSocketFailedErr() { + t.Fatalf("observable error = %v, want the generic socket failure by identity", got) + } + }) +} From 7f5c1ded77a0665eb791756bd15839c1eff3bb35 Mon Sep 17 00:00:00 2001 From: Jeremy Daer Date: Thu, 10 Sep 2026 00:56:01 -0700 Subject: [PATCH 41/51] Event feed: three renderings that said more, or other, than the contract MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The third unauthorized mint's authorization_failed terminal retained the generated error as its cause, where the poll lane's threshold terminal rebuilds a bare classified cause: §23's terminal table attaches the generated error to mint_failed and poll_failed only, and this reason's contract is its counter message. The mint lane now rebuilds the same way. The redirect-refused terminal rendered "a cross-origin redirect", but the kind covers every Location that fails the per-hop rule — downgraded, non-HTTP, unparseable — so the phrase misdiagnosed the other three. It names the rule the Location failed now, which is the violation class the continuation terminals already render. The CableTransport.Dial contract described a policy error's Reason and cause as reaching the consumer through the iteration's terminal, and hung the never-render-the-URL obligation on that. The loop reads the Kind alone and forwards nothing a transport wrote, for every kind; the doc says so, and states what the obligation still protects: the seam's return is opaque text to everything outside this package, and a host that wraps or logs it directly has no reduction in front of it. --- go/pkg/basecamp/eventfeed/catchup.go | 8 ++++++-- go/pkg/basecamp/eventfeed/catchup_test.go | 2 +- go/pkg/basecamp/eventfeed/loop.go | 9 +++++++- go/pkg/basecamp/eventfeed/seams.go | 25 ++++++++++++++--------- 4 files changed, 30 insertions(+), 14 deletions(-) diff --git a/go/pkg/basecamp/eventfeed/catchup.go b/go/pkg/basecamp/eventfeed/catchup.go index cdff7d67aa..9d5c7d4282 100644 --- a/go/pkg/basecamp/eventfeed/catchup.go +++ b/go/pkg/basecamp/eventfeed/catchup.go @@ -1396,7 +1396,11 @@ func (l *loop) recoverPoll(at *attempt, cursor Cursor, err error) (walkStep, cyc // A 3xx whose Location failed the seam's per-hop validation: the // continuation edge, NEVER poll_failed — and the terminal renders a // FIXED violation-class phrase, the continuation rejections' own - // contract. The refused origin is DATA, not a rendering: a hostile + // contract. The class is "failed per-hop validation", which is + // wider than cross-origin: a downgraded, non-HTTP, or unparseable + // Location takes this kind too, so the phrase names the rule the + // Location failed rather than one way of failing it. The refused + // origin is DATA, not a rendering: a hostile // redirect can reflect the caller's bearer into a host label (the // CloseError.Reason precedent), so no rendering may carry // LocationOrigin. It survives as a field on the sanitized cause for a @@ -1406,7 +1410,7 @@ func (l *loop) recoverPoll(at *attempt, cursor Cursor, err error) (walkStep, cyc l.disposeAttempt(at, nil) return walkStep{}, cycleOutcome{kind: outcomeTerminal, term: &TerminalError{ Reason: ReasonInvalidContinuation, - Msg: "the poll refused a cross-origin redirect", + Msg: "the poll refused a redirect whose Location failed continuation validation", Err: &PollError{Kind: PollRedirectRefused, LocationOrigin: pe.LocationOrigin}, }}, true case PollUnrecoverable: diff --git a/go/pkg/basecamp/eventfeed/catchup_test.go b/go/pkg/basecamp/eventfeed/catchup_test.go index 8d055c4b74..47931394b4 100644 --- a/go/pkg/basecamp/eventfeed/catchup_test.go +++ b/go/pkg/basecamp/eventfeed/catchup_test.go @@ -620,7 +620,7 @@ func TestPollFailureClassification(t *testing.T) { "redirect refused is invalid_continuation", &eventfeed.PollError{Kind: eventfeed.PollRedirectRefused, LocationOrigin: "https://attacker.example.com"}, eventfeed.ReasonInvalidContinuation, - "the poll refused a cross-origin redirect", + "the poll refused a redirect whose Location failed continuation validation", }, } for _, tc := range cases { diff --git a/go/pkg/basecamp/eventfeed/loop.go b/go/pkg/basecamp/eventfeed/loop.go index aa5d62aa2d..84ef463c07 100644 --- a/go/pkg/basecamp/eventfeed/loop.go +++ b/go/pkg/basecamp/eventfeed/loop.go @@ -1092,10 +1092,17 @@ func (l *loop) classifyMintFailure(err error) cycleOutcome { // connection-level counter's terminal — at the 3rd consecutive. l.authFailures++ if l.authFailures >= authFailureThreshold { + // The cause is rebuilt WITHOUT the generated error, as the poll + // lane's threshold terminal rebuilds its own: §23's terminal table + // attaches the generated error to mint_failed and poll_failed + // only, and authorization_failed's contract is its counter + // message alone. A generated error renders the request and the + // response's own text, neither of which this terminal is + // entitled to hand the consumer. return cycleOutcome{kind: outcomeTerminal, term: &TerminalError{ Reason: ReasonAuthorizationFailed, Msg: fmt.Sprintf("%d consecutive connection-level authorization failures", l.authFailures), - Err: err, + Err: &MintError{Kind: MintUnauthorized}, }} } // No retryAfter: SPEC pins `unauthorized` as "a kind carrying no diff --git a/go/pkg/basecamp/eventfeed/seams.go b/go/pkg/basecamp/eventfeed/seams.go index 87759ff249..f04deee971 100644 --- a/go/pkg/basecamp/eventfeed/seams.go +++ b/go/pkg/basecamp/eventfeed/seams.go @@ -284,16 +284,21 @@ type CableTransport interface { // the classification and a cause of your own choosing, never the peer's // or a library's rendering of the URL. // - // Where it is exposed, stated exactly, because the obvious guess is wrong: - // a dial failure does NOT reach Observer.Disconnected. There is no socket - // yet, so there is no teardown to report. A DialPolicy failure becomes - // Terminal(invalid_cable_url) and is yielded as the iteration's terminal - // error — the consumer's own error value, which is a stronger exposure - // than a callback, not a weaker one. Every other kind takes transition 7 - // to backoff, where the connector reports the classification and drops the - // cause entirely. So the obligation above is not softened by the callback - // never firing; it is what keeps a credential out of the error a caller - // receives from Events. + // What the connector does with the error, stated exactly, because the + // obvious guess is wrong: NOTHING it carries is forwarded. A dial failure + // does not reach Observer.Disconnected — there is no socket yet, so there + // is no teardown to report. A DialPolicy failure becomes + // Terminal(invalid_cable_url), and that terminal is the connector's own + // fixed message with no cause attached: the Kind is read, the Reason and + // the wrapped error are not. Every other kind takes transition 7 to + // backoff, where only the classification survives. The connector reduces + // rather than trusts, precisely because a documented extension point is + // host code and the dialed URL carries the ticket; a transport's + // diagnostics therefore never reach a consumer through this seam, and a + // transport that needs them surfaced logs them itself. The obligation + // above still binds — an error is opaque text to everything OUTSIDE this + // package, and a host that wraps or logs the seam's return directly has + // no reduction in front of it. Dial(ctx context.Context, wsURL string, maxFrameBytes int64) (CableConn, error) } From c93c620fe35d56e948d3854e0b93e1fb5bc829e3 Mon Sep 17 00:00:00 2001 From: Jeremy Daer Date: Thu, 10 Sep 2026 00:56:01 -0700 Subject: [PATCH 42/51] Event feed: two docs narrowed to what the code holds MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Wait's exclusion named consumer callbacks alone, and §23 said the same; the run also awaits every seam it calls synchronously — minter, poll source, transport and connection, store, clock — so a wait() from inside one of those blocks the very call the run is blocked on. Both now say "anything the run is waiting on" and list the seams. doc.go called the package unusable against the live API while the next sentence, and the README, describe hosts supplying TicketMinter and PollSource over the generated operations. Nothing ships those two seams yet; a host-supplied adapter is the supported path, and the paragraph says so instead of telling that host its path is impossible. Fixture 30's kill claim and the kill matrix's row 15 are deliberately not touched here: #778, stacked on this branch, already rewrites both (6135ca71) and scopes the zero-egress obligation to the Layer-1 adapter (a3189b59), and repeating that work here would only conflict with it. --- SPEC.md | 4 +++- go/pkg/basecamp/eventfeed/connector.go | 12 ++++++++---- go/pkg/basecamp/eventfeed/doc.go | 17 ++++++++++------- 3 files changed, 21 insertions(+), 12 deletions(-) diff --git a/SPEC.md b/SPEC.md index 08d28d77c3..e30e5c8778 100644 --- a/SPEC.md +++ b/SPEC.md @@ -3241,7 +3241,9 @@ is shared: until the run has exited, after which no save can be in flight. A consumer that owns the iteration needs nothing extra, since the iteration terminating is the same guarantee. Await termination — or `wait()` — before opening a second connector over the same store. - `wait()` is not callable from a consumer callback, for the reason `close()` does not wait. + `wait()` is not callable from anything the run is waiting on — a consumer callback, or a + seam, store, or clock implementation the run is synchronously awaiting — for the reason + `close()` does not wait. - A consumer break takes the identical teardown path; the in-flight page's checkpoint is **not** saved. - All Observer callbacks fire on the consumer's execution context, never concurrently with a diff --git a/go/pkg/basecamp/eventfeed/connector.go b/go/pkg/basecamp/eventfeed/connector.go index a6493dd9c3..dbc204d91c 100644 --- a/go/pkg/basecamp/eventfeed/connector.go +++ b/go/pkg/basecamp/eventfeed/connector.go @@ -592,10 +592,14 @@ func (c *Connector) Close() error { // loop needs nothing extra: the iterator returning is the same guarantee — the // run goroutine has exited by then, so no save can be in flight. // -// It is not callable from a consumer callback. Every callback — an observer, a -// signal handler, the loop body — runs ON the run goroutine, so waiting for -// that goroutine from inside one waits for itself. Close is the call that is -// safe from anywhere; this is the one that is safe from anywhere ELSE. +// It is not callable from anything the run is waiting on. Every consumer +// callback — an observer, a signal handler, the loop body — runs ON the run +// goroutine, so waiting for that goroutine from inside one waits for itself; +// and every seam the run calls synchronously — TicketMinter, PollSource, +// CableTransport and its CableConn, CheckpointStore, Clock — is awaited by +// that goroutine, so a Wait from inside one of those blocks the call the run +// is blocked on. Close is the call that is safe from anywhere; this is the one +// that is safe only from a goroutine taking no part in the run. func (c *Connector) Wait() { c.mu.Lock() done := c.runDone diff --git a/go/pkg/basecamp/eventfeed/doc.go b/go/pkg/basecamp/eventfeed/doc.go index 92d3fd69d8..e3142556a7 100644 --- a/go/pkg/basecamp/eventfeed/doc.go +++ b/go/pkg/basecamp/eventfeed/doc.go @@ -15,13 +15,16 @@ // policy and its WebSocket transport, and the deterministic fakes the // conformance harness drives. // -// ONE piece is still to land, and it is what keeps the package unusable -// against the live API: the Layer-1 adapters over the generated -// CreateStreamTicket and PollEvents operations that back the TicketMinter and -// PollSource seams. Until they exist, the seams have no production -// implementation and a host must supply its own. Two obligations ride on -// those adapters rather than on anything here — zero egress to a foreign -// redirect target (conformance/event-feed/README.md's row-15 note), and the +// ONE piece is still to land, and it is what keeps the package from running +// against the live API out of the box: the Layer-1 adapters over the +// generated CreateStreamTicket and PollEvents operations that back the +// TicketMinter and PollSource seams. Until they exist the package ships no +// implementation of those two seams, and a host that wants the live feed +// supplies its own over the generated operations — a supported path, and +// the one the seam contracts are written for, not a workaround. Two +// obligations ride on whichever adapters back the seams rather than on +// anything here — zero egress to a foreign redirect target +// (conformance/event-feed/README.md's row-15 note), and the // no-automatic-redirect-following rule §23 places on PollEvents. // // # Seams-first architecture From b51c788dfd1c7dc8cacfc62b299f7303bc190354 Mon Sep 17 00:00:00 2001 From: Jeremy Daer Date: Thu, 10 Sep 2026 00:58:17 -0700 Subject: [PATCH 43/51] Event feed: the tier-2 driver models a conforming adapter, and its loader sees twice MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The driver's mint mapping populated RetryAfter before classifying, so an unauthorized or unrecoverable outcome carried a value §23 defines for the throttled kind alone — and its comment said row 4 floors the reconnect on it, which is the opposite of what row 4 and classifyMint say. The value rides on the throttled branch now, and the comment names the tier-3 pin that proves the connector ignores a nonconforming one. The driver's HTTP-date branch could report a negative Retry-After: Sub saturates at time.Duration's ~292-year maximum, and rounding that up to whole seconds and multiplying back wrapped. It clamps to the digit branch's MaxInt32-seconds ceiling, and a year-9999 date is pinned. The loader promised that nothing in a fixture is silently ignored, and encoding/json broke that promise for one shape it cannot see: a member named twice keeps the last value and DisallowUnknownFields never meets the first, so a duplicated expectation was discarded while the fixture stayed green — and the schema gate's own parser collapses duplicates the same way. Fixtures are now walked as a token stream before decoding and any object naming a member twice, at any depth and under escape-equivalent spellings, is refused; the four levels are probed. --- .../eventfeed/scenario_conformance_test.go | 23 +++++-- .../eventfeed/scenario_fixture_test.go | 68 +++++++++++++++++++ .../eventfeed/scenario_selftest_test.go | 47 +++++++++++++ 3 files changed, 133 insertions(+), 5 deletions(-) diff --git a/go/pkg/basecamp/eventfeed/scenario_conformance_test.go b/go/pkg/basecamp/eventfeed/scenario_conformance_test.go index 4e7ab18e0c..29fbcbf661 100644 --- a/go/pkg/basecamp/eventfeed/scenario_conformance_test.go +++ b/go/pkg/basecamp/eventfeed/scenario_conformance_test.go @@ -1015,8 +1015,13 @@ func (h *scenarioHarness) violation() error { // Throttled-vs-transient is keyed on the PRESENCE of a §6-parsed Retry-After // (SPEC's fixed adapter mapping — "whatever its status"), never on the // status: a status key would honour the header at one retryable status and -// drop it at another. Unauthorized keeps a parsed value too — row 4 floors -// the below-threshold reconnect delay on it. +// drop it at another. The parsed value rides on the throttled kind ALONE: +// §23 defines retry_after for throttled and pins unauthorized as carrying +// none (row 4 — the backoff draw alone governs), so a driver populating it +// on a 401 would be modelling a nonconforming adapter, and a connector that +// wrongly honoured it would be indistinguishable here from one that did not. +// That the connector ignores such a value is a tier-3 pin +// (TestUnauthorizedMintCarriesNoRetryAfterFloor), not this driver's. func (d *driver) mintOutcomeFrom(respond mintRespond) mintOutcome { if respond.Body != nil { return mintOutcome{ticket: eventfeed.StreamTicket{ @@ -1025,16 +1030,16 @@ func (d *driver) mintOutcomeFrom(respond mintRespond) mintOutcome { URL: respond.Body.URL, }} } - retryAfter, present := d.retryAfterFrom(respond.Headers) - mintErr := &eventfeed.MintError{RetryAfter: retryAfter, Err: fmt.Errorf("mint responded %d", *respond.Status)} + mintErr := &eventfeed.MintError{Err: fmt.Errorf("mint responded %d", *respond.Status)} switch *respond.Status { case 401, 403: mintErr.Kind = eventfeed.MintUnauthorized case 404, 422: mintErr.Kind = eventfeed.MintUnrecoverable default: - if present { + if retryAfter, present := d.retryAfterFrom(respond.Headers); present { mintErr.Kind = eventfeed.MintThrottled + mintErr.RetryAfter = retryAfter } else { mintErr.Kind = eventfeed.MintTransient } @@ -1211,6 +1216,14 @@ func (d *driver) retryAfterFrom(headers map[string]string) (value time.Duration, if remainder%time.Second != 0 { seconds++ } + // The same ceiling as the digit branch, for the same reason: Sub + // saturates at time.Duration's ~292-year maximum, and rounding that + // remainder up to whole seconds and multiplying back overflows into a + // negative delay — a "present" Retry-After the connector would treat as + // no wait at all. A year-9999 date is a valid HTTP-date. + if seconds > math.MaxInt32 { + seconds = math.MaxInt32 + } return time.Duration(seconds) * time.Second, true } diff --git a/go/pkg/basecamp/eventfeed/scenario_fixture_test.go b/go/pkg/basecamp/eventfeed/scenario_fixture_test.go index 9dca83dbe4..a53163623f 100644 --- a/go/pkg/basecamp/eventfeed/scenario_fixture_test.go +++ b/go/pkg/basecamp/eventfeed/scenario_fixture_test.go @@ -13,7 +13,9 @@ package eventfeed_test import ( "bytes" "encoding/json" + "errors" "fmt" + "io" "regexp" "strconv" "strings" @@ -233,6 +235,9 @@ type expectPositionRejectedStep struct { // parseScenario decodes one substituted fixture, failing on anything the // driver does not model. func parseScenario(raw []byte, file string) (*scenario, error) { + if err := rejectDuplicateKeys(raw); err != nil { + return nil, err + } top, err := objectKeys(raw) if err != nil { return nil, err @@ -721,6 +726,69 @@ func validateInvocations(records []invocation) error { // --- decoding helpers ---------------------------------------------------- +// rejectDuplicateKeys walks raw as a token stream and fails on any object +// that names a member twice, at any depth. encoding/json keeps the LAST of +// two same-named members and DisallowUnknownFields never sees the first, so a +// fixture that spelled an expectation twice would silently discard one of +// them and stay green — against the loader's promise that nothing is +// silently ignored. Names are compared DECODED, so an escape-equivalent +// spelling ("ms" and "m\u0073") is the same name; the schema gate cannot +// see this either, since its validator's JSON parser collapses duplicates +// the same way. +func rejectDuplicateKeys(raw []byte) error { + dec := json.NewDecoder(bytes.NewReader(raw)) + // Numbers pass through as their literal text: this walk decides only + // whether a member name repeats, and a literal outside float64's range + // is a verdict for the loader's own number handling, not for this one. + dec.UseNumber() + type container struct { + object bool + seen map[string]bool + // key reports that the next token in an object is a member name. + key bool + } + var stack []*container + // valueDone marks that the enclosing object, if any, expects a name next. + valueDone := func() { + if n := len(stack); n > 0 && stack[n-1].object { + stack[n-1].key = true + } + } + for { + tok, err := dec.Token() + if errors.Is(err, io.EOF) { + return nil + } + if err != nil { + return err + } + switch v := tok.(type) { + case json.Delim: + switch v { + case '{': + stack = append(stack, &container{object: true, seen: map[string]bool{}, key: true}) + case '[': + stack = append(stack, &container{}) + default: + stack = stack[:len(stack)-1] + valueDone() + } + case string: + if n := len(stack); n > 0 && stack[n-1].object && stack[n-1].key { + if stack[n-1].seen[v] { + return fmt.Errorf("duplicate key %q", v) + } + stack[n-1].seen[v] = true + stack[n-1].key = false + continue + } + valueDone() + default: + valueDone() + } + } +} + // decodeStrict decodes exactly one JSON value into dst, rejecting unknown // object keys and trailing content. func decodeStrict(raw json.RawMessage, dst any) error { diff --git a/go/pkg/basecamp/eventfeed/scenario_selftest_test.go b/go/pkg/basecamp/eventfeed/scenario_selftest_test.go index 330c018a13..214057c81f 100644 --- a/go/pkg/basecamp/eventfeed/scenario_selftest_test.go +++ b/go/pkg/basecamp/eventfeed/scenario_selftest_test.go @@ -391,6 +391,37 @@ func TestScenarioDriverRejectsUnmodelledScripts(t *testing.T) { script: `{"name":"x","description":"d","only":true,"steps":[{"advance":{"ms":1}}],"finally":{"state":"closed"}}`, wants: `unknown key "only"`, }, + // Duplicate members at each level: encoding/json keeps the last one + // and DisallowUnknownFields never sees the first, so each of these + // would otherwise pass as a fixture with one expectation quietly + // dropped. + { + name: "duplicate top-level key", + script: `{"name":"x","description":"d","steps":[{"advance":{"ms":1}}],"steps":[],"finally":{"state":"closed"}}`, + wants: `duplicate key "steps"`, + }, + { + name: "duplicate directive in one step", + script: `{"name":"x","description":"d","steps":[{"advance":{"ms":1},"advance":{"ms":2}}],"finally":{"state":"closed"}}`, + wants: `duplicate key "advance"`, + }, + { + name: "duplicate key inside a payload", + script: `{"name":"x","description":"d","steps":[{"advance":{"ms":1,"ms":2}}],"finally":{"state":"closed"}}`, + wants: `duplicate key "ms"`, + }, + { + name: "escape-equivalent duplicate key", + script: `{"name":"x","description":"d","steps":[{"advance":{"ms":1,"m\u0073":2}}],"finally":{"state":"closed"}}`, + wants: `duplicate key "ms"`, + }, + { + // The duplicate walk must not be the thing that judges a number: + // a literal beyond float64 is the ms handling's verdict. + name: "duplicate key beside a number beyond float64", + script: `{"name":"x","description":"d","steps":[{"advance":{"ms":1e999,"ms":2}}],"finally":{"state":"closed"}}`, + wants: `duplicate key "ms"`, + }, { name: "unknown placeholder", script: `{"name":"x","description":"d","steps":[{"expectConnect":{"url":"{{CABLE_HOST:1}}"}}],"finally":{"state":"closed"}}`, @@ -698,6 +729,15 @@ func TestDriverSeamClassificationIsPresenceKeyed(t *testing.T) { d := &driver{h: newScenarioHarness()} defer d.h.close() + // The parsed value rides on the throttled kind alone: an unauthorized + // mint carries no retry_after by contract, whatever header the scripted + // response happened to carry. + unauthorized := d.mintOutcomeFrom(mintRespond{Status: intp(401), Headers: map[string]string{"Retry-After": "3"}}) + var ue *eventfeed.MintError + if !errors.As(unauthorized.err, &ue) || ue.Kind != eventfeed.MintUnauthorized || ue.RetryAfter != 0 { + t.Fatalf("401 with Retry-After: 3 = %+v, want MintUnauthorized carrying no RetryAfter", unauthorized.err) + } + // A retryable non-429/503 status WITH a parsed Retry-After is throttled, // carrying the value. out := d.mintOutcomeFrom(mintRespond{Status: intp(502), Headers: map[string]string{"Retry-After": "3"}}) @@ -759,6 +799,13 @@ func TestDriverRetryAfterParseIsTheSpecAlgorithm(t *testing.T) { if !errors.As(out.err, &me) || me.Kind != eventfeed.MintThrottled || me.RetryAfter != 91*time.Second { t.Fatalf("HTTP-date 90.3s out = %+v, want MintThrottled carrying 91s (sub-second remainder rounds UP)", out.err) } + // A far-future date saturates at the digit branch's ceiling rather than + // wrapping: Sub saturates at ~292 years, and rounding that up to seconds + // and multiplying back overflowed into a negative delay. + out = d.mintOutcomeFrom(mintRespond{Status: intp(429), Headers: map[string]string{"Retry-After": "Fri, 31 Dec 9999 23:59:59 GMT"}}) + if !errors.As(out.err, &me) || me.Kind != eventfeed.MintThrottled || me.RetryAfter != time.Duration(math.MaxInt32)*time.Second { + t.Fatalf("year-9999 HTTP-date = %+v, want MintThrottled saturated at %s", out.err, time.Duration(math.MaxInt32)*time.Second) + } // A date already passed is undefined — max(0, ·) is not > 0. past := now.Add(-time.Minute).UTC().Format(http.TimeFormat) for name, header := range map[string]string{ From 03f5d7047c72716f9fa012fbd36a06f16f9cd3a5 Mon Sep 17 00:00:00 2001 From: Jeremy Daer Date: Thu, 10 Sep 2026 02:33:55 -0700 Subject: [PATCH 44/51] Event feed: a state is announced with its timer already armed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Streaming arms its repair cadence before the state is announced, and says why: the announcement is made with the exact set §23 publishes for the state already outstanding, so an observer reacting to it sees that set and not the empty one a statement earlier. Connecting and Backoff were the two states that announced first and armed after — Connecting's handshake deadline sat behind the cable-URL check, Backoff's timer behind the announcement — and a tier-2 scenario that awaits a state and then asserts its timers could read {} where the table says {handshake-deadline} or {backoff}. Both arm first now; a cable-URL refusal stops the deadline on its way to the terminal, whose set is {}. --- go/pkg/basecamp/eventfeed/loop.go | 29 ++++++++++------- go/pkg/basecamp/eventfeed/loop_test.go | 45 ++++++++++++++++++++++++++ 2 files changed, 63 insertions(+), 11 deletions(-) diff --git a/go/pkg/basecamp/eventfeed/loop.go b/go/pkg/basecamp/eventfeed/loop.go index 84ef463c07..e3540e814e 100644 --- a/go/pkg/basecamp/eventfeed/loop.go +++ b/go/pkg/basecamp/eventfeed/loop.go @@ -772,11 +772,15 @@ func (l *loop) run(yield func(Event, error) bool) { return } // The failed attempt is fully disposed, so the exact - // outstanding-timer set on entry to Backoff is {backoff}. + // outstanding-timer set on entry to Backoff is {backoff} — and + // the timer is armed BEFORE the state is announced, as every + // state's timer is, so an observer reacting to the announcement + // sees the set §23 publishes for it rather than the empty set a + // statement earlier. l.failedCycles++ - l.setState(stateBackoff) d := reconnectDelay(out.retryAfter, l.failedCycles, l.cfg.rand) t := l.cfg.clock.NewTimer(d, timerBackoff) + l.setState(stateBackoff) select { case <-t.C(): // Transition 2: Backoff → Minting; a fresh ticket is ALWAYS @@ -915,21 +919,24 @@ func (l *loop) runCycle(delay time.Duration) cycleOutcome { return l.classifyMintFailure(err) } - // Transition 3 → Connecting: dial the mint's url verbatim. The cable-URL - // policy pre-check runs before anything is armed — a policy violation - // recurs on every re-mint, so it is Terminal(invalid_cable_url), never - // Backoff. The error never carries the URL: the ticket rides in its - // query string. + // Transition 3 → Connecting: dial the mint's url verbatim. The handshake + // deadline arms on entry to Connecting, BEFORE dial — it spans + // dial-to-welcome, so a stalled dial expires it (transition 7) — and + // before the state is announced, so the announcement is made with the + // set §23 publishes for Connecting already outstanding. The cable-URL + // policy pre-check follows: a policy violation recurs on every re-mint, + // so it is Terminal(invalid_cable_url), never Backoff, and the deadline + // is stopped on the way out (a terminal's set is {}). The error never + // carries the URL: the ticket rides in its query string. + hs := l.cfg.clock.NewTimer(handshakeDeadline, timerHandshakeDeadline) + at.phase = hs l.setState(stateConnecting) if derr := checkCableURL(ticket.URL); derr != nil { + hs.Stop() return cycleOutcome{kind: outcomeTerminal, term: &TerminalError{ Reason: ReasonInvalidCableURL, Msg: derr.Reason, Err: derr, }} } - // The handshake deadline arms on entry to Connecting, BEFORE dial — it - // spans dial-to-welcome, so a stalled dial expires it (transition 7). - hs := l.cfg.clock.NewTimer(handshakeDeadline, timerHandshakeDeadline) - at.phase = hs type dialResult struct { conn CableConn err error diff --git a/go/pkg/basecamp/eventfeed/loop_test.go b/go/pkg/basecamp/eventfeed/loop_test.go index d0711d899d..926ace1076 100644 --- a/go/pkg/basecamp/eventfeed/loop_test.go +++ b/go/pkg/basecamp/eventfeed/loop_test.go @@ -2706,3 +2706,48 @@ func TestTypedNilSeamErrorsFailClosed(t *testing.T) { } }) } + +// TestStateAnnouncementsCarryTheirTimerSet: §23's per-state exact timer sets +// are what an observer of a state transition sees at the announcement, so +// each state's timer is armed before the state is published — Connecting +// with {handshake-deadline} and Backoff with {backoff} were the two that +// announced first and armed a statement later, which an await-state-then- +// assert-timers scenario could observe as the empty set. +func TestStateAnnouncementsCarryTheirTimerSet(t *testing.T) { + h := newHarness(t) + var mu sync.Mutex + seen := map[string][]string{} + h.conn.OnStateChanged(func(state string) { + mu.Lock() + seen[state] = append([]string(nil), h.clock.Outstanding()...) + mu.Unlock() + }) + h.minter.ScriptTicket(ticket(1)) + h.tr.FailNextDial(&eventfeed.DialError{Kind: eventfeed.DialTransient}) + h.start() + h.awaitTimer(timerBackoff) + + mu.Lock() + defer mu.Unlock() + if got := seen["connecting"]; len(got) != 1 || got[0] != timerHandshakeDeadline { + t.Fatalf("outstanding at the Connecting announcement = %v, want [%s]", got, timerHandshakeDeadline) + } + if got := seen["backoff"]; len(got) != 1 || got[0] != timerBackoff { + t.Fatalf("outstanding at the Backoff announcement = %v, want [%s]", got, timerBackoff) + } +} + +// TestInvalidCableURLLeavesNoTimer: the handshake deadline is armed before +// the cable-URL check now, so a policy refusal must stop it on the way to +// its terminal — a terminal's exact set is {}. +func TestInvalidCableURLLeavesNoTimer(t *testing.T) { + h := newHarness(t) + h.minter.ScriptTicket(eventfeed.StreamTicket{Ticket: "t", ExpiresIn: 120, URL: "ws://cable.example.com/cable?ticket=t"}) + h.start() + h.join() + _, terminal, _ := h.snapshot() + if terminal == nil || terminal.Reason != eventfeed.ReasonInvalidCableURL { + t.Fatalf("terminal = %v, want reason %q", terminal, eventfeed.ReasonInvalidCableURL) + } + assertTimers(t, h.clock, map[string]int{}) +} From 7d803f954159ab042ea0d8ce51bb8f9e7634e3f6 Mon Sep 17 00:00:00 2001 From: Jeremy Daer Date: Thu, 10 Sep 2026 02:58:13 -0700 Subject: [PATCH 45/51] Event feed: a seam panic on a worker goroutine reaches the consumer, not the process MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Dial, WriteFrame and Poll run on worker goroutines so the state machine can keep selecting while they are in flight, and Go's recovery is goroutine-local: a host seam panicking on one of them ended the process, bypassing the recovery that — for the same panic on the consumer's goroutine — disposes the attempt and lets the consumer's own recover see it. Each worker now captures the panic and carries it back over its result channel, and the receiving arm re-raises it on the state machine's goroutine, where runCycle's deferred recovery runs as for any host-code panic: the socket is disposed where one exists, the phase timer is stopped where none does, and the consumer recovers the seam's own value. A poll abandoned at the grace deadline has no receiver left for its panic — its attempt is already torn down, and there is no consumer context to propagate into — which the result type says. One test per seam, each ending the test binary against the previous revision. --- go/pkg/basecamp/eventfeed/catchup.go | 18 +++- go/pkg/basecamp/eventfeed/connector_test.go | 108 ++++++++++++++++++++ go/pkg/basecamp/eventfeed/loop.go | 61 +++++++++-- 3 files changed, 178 insertions(+), 9 deletions(-) diff --git a/go/pkg/basecamp/eventfeed/catchup.go b/go/pkg/basecamp/eventfeed/catchup.go index 9d5c7d4282..1d4565333d 100644 --- a/go/pkg/basecamp/eventfeed/catchup.go +++ b/go/pkg/basecamp/eventfeed/catchup.go @@ -434,6 +434,11 @@ type deferredFrame struct { type pollResult struct { page PollPage err error + // panicked carries a panic the seam raised on the call's goroutine, to + // be re-raised on the state machine's (seamPanic). A call abandoned at + // the grace deadline has no receiver left for it: its attempt is already + // torn down, and there is no consumer context to propagate into. + panicked *seamPanic } // pollAttempt is one poll-page attempt as the walk sees it: the page the seam @@ -505,13 +510,19 @@ func (l *loop) pollPage(at *attempt, cursor Cursor) pollAttempt { // way in: the seam is host code, and handing it the connector's own // backing arrays would let an adapter that sorts or dedupes in place // repoint the subscription's lineage from under a live feed. - p, perr := l.cfg.polls.Poll(at.ctx, cursor, l.cfg.filters.clone()) - done <- pollResult{page: p, err: perr} + var r pollResult + r.panicked = capturePanic(func() { + r.page, r.err = l.cfg.polls.Poll(at.ctx, cursor, l.cfg.filters.clone()) + }) + done <- r }() for { staleTimer, staleGen := at.lc.stale.current() select { case r := <-done: + if r.panicked != nil { + panic(r.panicked.value) + } // The socket's verdict outranks the poll's: both can be ready, // and returning a FAILED result unchecked lets recoverPoll's // disposal discard the fired expiry — an unauthorized verdict @@ -666,6 +677,9 @@ func (l *loop) awaitSupersededPoll(at *attempt, done <-chan pollResult, deadline staleTimer, _ := at.lc.stale.current() select { case r := <-done: + if r.panicked != nil { + panic(r.panicked.value) + } return pollAttempt{page: r.page, err: r.err} case <-l.runCtx.Done(): return pollAttempt{superseded: true} diff --git a/go/pkg/basecamp/eventfeed/connector_test.go b/go/pkg/basecamp/eventfeed/connector_test.go index 06eb03ce40..bd2dfa4dde 100644 --- a/go/pkg/basecamp/eventfeed/connector_test.go +++ b/go/pkg/basecamp/eventfeed/connector_test.go @@ -2048,3 +2048,111 @@ func TestPanicArmingStalenessClosesTheDialedSocket(t *testing.T) { } assertTimers(t, clock.Clock, map[string]int{}) } + +// panickingDialTransport and panickingWriteConn make a seam panic on the +// worker goroutine the connector runs it on. +type panickingDialTransport struct{ inner *feedtest.Transport } + +func (p *panickingDialTransport) Dial(context.Context, string, int64) (eventfeed.CableConn, error) { + panic("host transport panicked in Dial") +} + +type panickingWriteTransport struct{ inner *feedtest.Transport } + +func (p *panickingWriteTransport) Dial(ctx context.Context, wsURL string, maxFrameBytes int64) (eventfeed.CableConn, error) { + conn, err := p.inner.Dial(ctx, wsURL, maxFrameBytes) + if err != nil { + return nil, err + } + return &panickingWriteConn{CableConn: conn}, nil +} + +type panickingWriteConn struct{ eventfeed.CableConn } + +func (c *panickingWriteConn) WriteFrame(context.Context, []byte) error { + panic("host connection panicked in WriteFrame") +} + +// TestSeamPanicsPropagateOnTheConsumerGoroutine: Dial, WriteFrame and Poll +// run on worker goroutines so the state machine can keep selecting, and Go's +// recovery is goroutine-local — a host seam panicking on one of them ended +// the process, bypassing the recovery that disposes the attempt for a panic +// on the consumer's goroutine. Each worker now captures the panic and the +// receiving arm re-raises it where runCycle's recovery runs: the consumer's +// recover sees the seam's own value, the socket (where one exists) is +// disposed, and no timer survives. +func TestSeamPanicsPropagateOnTheConsumerGoroutine(t *testing.T) { + for _, tc := range []struct { + name string + setup func(h *harness) (drive func(), want string) + opts func(h *harness) []eventfeed.Option + }{ + { + name: "poll", + setup: func(h *harness) (func(), string) { + h.polls.OnCall(func(feedtest.PollCall) { panic("host poll source panicked") }) + return func() { + conn := h.driveToSubscribed() + conn.Serve(frameConfirm(noFilterIdentifier)) + }, "host poll source panicked" + }, + }, + { + name: "write", + setup: func(h *harness) (func(), string) { + return func() { + conn := h.liveConn() + conn.Serve(frameWelcome()) + }, "host connection panicked in WriteFrame" + }, + opts: func(h *harness) []eventfeed.Option { + return []eventfeed.Option{eventfeed.WithTransport(&panickingWriteTransport{inner: h.tr})} + }, + }, + { + name: "dial", + setup: func(h *harness) (func(), string) { return func() {}, "host transport panicked in Dial" }, + opts: func(h *harness) []eventfeed.Option { + return []eventfeed.Option{eventfeed.WithTransport(&panickingDialTransport{inner: h.tr})} + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + base := newHarness(t) + var opts []eventfeed.Option + if tc.opts != nil { + opts = tc.opts(base) + } + h := newHarness(t, opts...) + if tc.opts != nil { + // The wrapper closes over the FIRST harness's transport; point + // this one at it so dials land where the test can see them. + h.tr = base.tr + } + h.minter.ScriptTicket(ticket(1)) + drive, want := tc.setup(h) + recovered := make(chan any, 1) + go func() { + defer func() { recovered <- recover() }() + for range h.conn.Events(context.Background()) { + } + }() + drive() + select { + case r := <-recovered: + if r == nil { + t.Fatal("the panic did not propagate out of the iteration") + } + if s, _ := r.(string); s != want { + t.Fatalf("recovered %v, want the seam's own panic value %q", r, want) + } + case <-time.After(watchdog): + t.Fatal("the iteration neither panicked nor returned") + } + if sock := h.tr.LastConn(); sock != nil && !sock.Closed() { + t.Fatal("the live socket was not disposed during panic unwinding") + } + assertTimers(t, h.clock, map[string]int{}) + }) + } +} diff --git a/go/pkg/basecamp/eventfeed/loop.go b/go/pkg/basecamp/eventfeed/loop.go index e3540e814e..da705f1727 100644 --- a/go/pkg/basecamp/eventfeed/loop.go +++ b/go/pkg/basecamp/eventfeed/loop.go @@ -148,6 +148,29 @@ type pumpItem struct { err error } +// seamPanic is a panic captured on a worker goroutine, carried back to the +// state machine's goroutine and re-raised there. Three seam calls run on +// workers so the state machine can keep selecting while they are in flight +// — Dial, WriteFrame, and Poll — and Go's recovery is goroutine-local: a +// host seam panicking on one of them would end the process, bypassing +// runCycle's recovery, where the same panic on the consumer's goroutine +// disposes the attempt and propagates to the consumer. The worker recovers, +// the receiving arm re-panics, and runCycle's deferred recovery runs as for +// any host-code panic. +type seamPanic struct{ value any } + +// capturePanic runs f, converting a panic into a seamPanic for the caller +// to re-raise on its own goroutine. +func capturePanic(f func()) (p *seamPanic) { + defer func() { + if r := recover(); r != nil { + p = &seamPanic{value: r} + } + }() + f() + return nil +} + // liveConn is one live socket with its reader pump and staleness holder. type liveConn struct { conn CableConn @@ -938,16 +961,26 @@ func (l *loop) runCycle(delay time.Duration) cycleOutcome { }} } type dialResult struct { - conn CableConn - err error + conn CableConn + err error + panicked *seamPanic } dialCh := make(chan dialResult, 1) go func() { - conn, dialErr := l.cfg.transport.Dial(at.ctx, ticket.URL, maxFrameBytes) - dialCh <- dialResult{conn: conn, err: dialErr} + var r dialResult + r.panicked = capturePanic(func() { + r.conn, r.err = l.cfg.transport.Dial(at.ctx, ticket.URL, maxFrameBytes) + }) + dialCh <- r }() select { case r := <-dialCh: + if r.panicked != nil { + // Re-raised here, on the state machine's goroutine: runCycle's + // recovery stops the handshake deadline (no liveConn exists yet) + // and the panic reaches the consumer as a host-code panic does. + panic(r.panicked.value) + } if l.runCtx.Err() != nil { hs.Stop() if r.conn != nil { @@ -1230,12 +1263,26 @@ func (l *loop) awaitConfirmation(at *attempt, deadline Timer) cycleOutcome { // wait carries the case like every other socket-open wait, generation dance // and all. func (l *loop) writeSubscribe(at *attempt, deadline *Timer) (cycleOutcome, bool) { - written := make(chan error, 1) - go func() { written <- at.lc.conn.WriteFrame(at.ctx, l.subscribeFrame) }() + type writeResult struct { + err error + panicked *seamPanic + } + written := make(chan writeResult, 1) + go func() { + var r writeResult + r.panicked = capturePanic(func() { r.err = at.lc.conn.WriteFrame(at.ctx, l.subscribeFrame) }) + written <- r + }() for { staleTimer, staleGen := at.lc.stale.current() select { - case werr := <-written: + case r := <-written: + if r.panicked != nil { + // Re-raised on the state machine's goroutine; runCycle's + // recovery disposes the live attempt. + panic(r.panicked.value) + } + werr := r.err // Close outranks the write's result: cancellation surfaces // through a conforming WriteFrame as its own error, so `written` // and runCtx.Done() can be ready together — and dispatching the From 03770007e89304dbf15e536f13920dc27d0b6a64 Mon Sep 17 00:00:00 2001 From: Jeremy Daer Date: Thu, 10 Sep 2026 03:21:28 -0700 Subject: [PATCH 46/51] Event feed: a cancelled dial's panic is re-raised, not dropped with its join MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The two arms that cancel a pending dial — the handshake deadline and Close — join the worker and inspect only the connection it may have returned. A dial that answers cancellation by panicking had that panic captured by the worker (the previous commit) and then dropped at the join, so the connector reconnected, or closed, over a host panic the consumer never saw. Both arms re-raise it once any returned connection is closed, with the handshake deadline stopped first on the Close arm as on its ordinary path. One test per arm, each red against the previous revision. --- go/pkg/basecamp/eventfeed/connector_test.go | 47 +++++++++++++++++++++ go/pkg/basecamp/eventfeed/loop.go | 23 +++++++--- 2 files changed, 65 insertions(+), 5 deletions(-) diff --git a/go/pkg/basecamp/eventfeed/connector_test.go b/go/pkg/basecamp/eventfeed/connector_test.go index bd2dfa4dde..88352f6fa7 100644 --- a/go/pkg/basecamp/eventfeed/connector_test.go +++ b/go/pkg/basecamp/eventfeed/connector_test.go @@ -2156,3 +2156,50 @@ func TestSeamPanicsPropagateOnTheConsumerGoroutine(t *testing.T) { }) } } + +// panickingOnCancelTransport blocks the dial until its context is cancelled +// and then panics — the seam answering cancellation with a panic. +type panickingOnCancelTransport struct{} + +func (panickingOnCancelTransport) Dial(ctx context.Context, _ string, _ int64) (eventfeed.CableConn, error) { + <-ctx.Done() + panic("host transport panicked on cancellation") +} + +// TestDialPanicOnCancellationPropagates: the two arms that cancel a pending +// dial — the handshake deadline and Close — join the worker and inspect only +// the connection it may have returned; a panic the cancelled dial raised was +// captured by the worker and then dropped, so the connector reconnected (or +// closed) over a host panic the consumer never saw. +func TestDialPanicOnCancellationPropagates(t *testing.T) { + for _, tc := range []struct { + name string + cancel func(h *harness) + }{ + {"handshake deadline", func(h *harness) { h.fireTimer(timerHandshakeDeadline) }}, + {"close", func(h *harness) { h.conn.Close() }}, //nolint:errcheck // asserted via the recover + } { + t.Run(tc.name, func(t *testing.T) { + h := newHarness(t, eventfeed.WithTransport(panickingOnCancelTransport{})) + h.minter.ScriptTicket(ticket(1)) + h.minter.ScriptError(&eventfeed.MintError{Kind: eventfeed.MintUnrecoverable}) + recovered := make(chan any, 1) + go func() { + defer func() { recovered <- recover() }() + for range h.conn.Events(context.Background()) { + } + }() + h.awaitTimer(timerHandshakeDeadline) + tc.cancel(h) + select { + case r := <-recovered: + if s, _ := r.(string); s != "host transport panicked on cancellation" { + t.Fatalf("recovered %v, want the seam's own panic value", r) + } + case <-time.After(watchdog): + t.Fatal("the iteration neither panicked nor returned") + } + assertTimers(t, h.clock, map[string]int{}) + }) + } +} diff --git a/go/pkg/basecamp/eventfeed/loop.go b/go/pkg/basecamp/eventfeed/loop.go index da705f1727..35f8d84041 100644 --- a/go/pkg/basecamp/eventfeed/loop.go +++ b/go/pkg/basecamp/eventfeed/loop.go @@ -1039,16 +1039,29 @@ func (l *loop) runCycle(delay time.Duration) cycleOutcome { case <-hs.C(): // Transition 7: the deadline expired mid-dial — cancel the pending // dial (the seam contract requires a prompt return) and dispose any - // connection it raced to open. + // connection it raced to open. A dial that answers cancellation by + // panicking is a host panic like any other: re-raised once the + // connection it may have returned is closed. at.cancel() - if r := <-dialCh; r.conn != nil { - _ = r.conn.Close(closeCodeNormal, "") + if r := <-dialCh; r.conn != nil || r.panicked != nil { + if r.conn != nil { + _ = r.conn.Close(closeCodeNormal, "") + } + if r.panicked != nil { + panic(r.panicked.value) + } } return cycleOutcome{kind: outcomeFailed} case <-l.runCtx.Done(): at.cancel() - if r := <-dialCh; r.conn != nil { - _ = r.conn.Close(closeCodeNormal, "") + if r := <-dialCh; r.conn != nil || r.panicked != nil { + if r.conn != nil { + _ = r.conn.Close(closeCodeNormal, "") + } + if r.panicked != nil { + hs.Stop() + panic(r.panicked.value) + } } hs.Stop() return cycleOutcome{kind: outcomeClosed} From 0df591a276804b43bebadabcc8c054d6a06a2ead Mon Sep 17 00:00:00 2001 From: Jeremy Daer Date: Thu, 10 Sep 2026 03:46:43 -0700 Subject: [PATCH 47/51] Event feed: the staleness probe trusts the latch before the channel MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The holder latches an expiry on arm's Stop-false verdict — a frame that arrived after the window closed — and the shipped SystemClock fires through time.AfterFunc, whose callback deregisters and then sends, so Stop reports false a moment before the timer's channel is ready. The non-blocking probe every arm runs before acting on the socket consulted the channel alone, so inside that gap a frame was delivered, or a poll issued, from a socket whose verdict the holder had already decided. The probe reads the latch first. The test wraps the clock so a staleness firing is held back from its channel after Stop has reported it, serves a frame, and asserts nothing is delivered; red against the previous revision. --- go/pkg/basecamp/eventfeed/catchup_test.go | 82 +++++++++++++++++++++++ go/pkg/basecamp/eventfeed/loop.go | 22 ++++++ 2 files changed, 104 insertions(+) diff --git a/go/pkg/basecamp/eventfeed/catchup_test.go b/go/pkg/basecamp/eventfeed/catchup_test.go index 47931394b4..3bd75d1888 100644 --- a/go/pkg/basecamp/eventfeed/catchup_test.go +++ b/go/pkg/basecamp/eventfeed/catchup_test.go @@ -3756,3 +3756,85 @@ func TestQueuedFatalOutranksARecoverableFrameAheadOfIt(t *testing.T) { t.Fatalf("mint calls = %d, want 1", got) } } + +// heldFiringClock wraps the harness clock so a timer's Stop reports the +// firing before its channel delivers it — the gap the shipped SystemClock has +// between time.AfterFunc's callback starting (Stop reports false) and its +// send. The firing is forwarded only once the test releases it. +type heldFiringClock struct { + *feedtest.Clock + hold chan struct{} + holding string +} + +func (c *heldFiringClock) NewTimer(d time.Duration, kind string) eventfeed.Timer { + t := c.Clock.NewTimer(d, kind) + if kind != c.holding { + return t + } + proxy := make(chan time.Time, 1) + go func() { + v, ok := <-t.C() + if !ok { + return + } + <-c.hold + proxy <- v + }() + return &heldFiringTimer{Timer: t, c: proxy} +} + +type heldFiringTimer struct { + eventfeed.Timer + c chan time.Time +} + +func (t *heldFiringTimer) C() <-chan time.Time { return t.c } + +// TestLatchedStalenessOutranksAFrameBeforeTheFiringDelivers: a frame that +// arrives after the window expired latches the expiry on arm's Stop verdict, +// and the staleness probe used to consult the timer channel alone — so in the +// gap before the firing delivers, the frame was delivered from a socket whose +// verdict was already in. The probe consults the latch first. +func TestLatchedStalenessOutranksAFrameBeforeTheFiringDelivers(t *testing.T) { + clock := &heldFiringClock{Clock: feedtest.NewClock(), hold: make(chan struct{}), holding: timerStaleness} + var mu sync.Mutex + var ages []time.Duration + h := newHarness(t, + eventfeed.WithClock(clock), + eventfeed.WithObserver(eventfeed.Observer{ + StaleConnection: func(age time.Duration) { + mu.Lock() + ages = append(ages, age) + mu.Unlock() + }, + })) + h.clock = clock.Clock + t.Cleanup(func() { close(clock.hold) }) + h.minter.ScriptTicket(ticket(1)) + h.minter.ScriptError(&eventfeed.MintError{Kind: eventfeed.MintUnrecoverable}) + h.polls.ScriptPage(eventfeed.PollPage{Position: "pos-1"}) + h.start() + conn := h.driveToSubscribed() + conn.Serve(frameConfirm(noFilterIdentifier)) + h.awaitStreaming() + // The window expires (the firing is held back from the channel), and a + // frame then arrives: the pump's reset sees Stop report false and latches. + h.fireTimer(timerStaleness) + conn.Serve(frameMessage(noFilterIdentifier, 101)) + h.waitUntil("the verdict or a delivery", func() bool { + mu.Lock() + defer mu.Unlock() + return len(ages) > 0 || len(h.deliveredIDs()) > 0 + }) + if ids := h.deliveredIDs(); len(ids) != 0 { + t.Fatalf("delivered %v from a socket whose staleness verdict was latched", ids) + } + h.awaitEndOrReconnect() + h.join() + mu.Lock() + defer mu.Unlock() + if len(ages) != 1 { + t.Fatalf("StaleConnection ages = %v, want exactly one", ages) + } +} diff --git a/go/pkg/basecamp/eventfeed/loop.go b/go/pkg/basecamp/eventfeed/loop.go index 35f8d84041..e5f6e7e152 100644 --- a/go/pkg/basecamp/eventfeed/loop.go +++ b/go/pkg/basecamp/eventfeed/loop.go @@ -506,6 +506,22 @@ func (h *staleHolder) graceWake(d time.Duration) { h.timer = h.clock.NewTimer(max(d, 0), timerStaleness) } +// latched reports an expiry the holder has already decided authoritative — +// arm's Stop-false verdict — with the silence it measured, whether or not the +// fired timer's channel has delivered yet. The two are not the same instant: +// the shipped SystemClock fires through time.AfterFunc, whose callback +// deregisters and THEN sends, so Stop reports false a moment before the +// channel is ready, and a probe that consults the channel alone would act +// on a socket whose verdict is already in. +func (h *staleHolder) latched() (time.Duration, bool) { + h.mu.Lock() + defer h.mu.Unlock() + if h.stopped || !h.expired { + return 0, false + } + return h.clock.Now().Sub(h.last), true +} + // stop cancels the timer permanently. func (h *staleHolder) stop() { h.mu.Lock() @@ -1822,6 +1838,12 @@ func (l *loop) lapse(at *attempt) cycleOutcome { // non-blocking evaluate alone, for a caller that has something to do between // learning the window has expired and disposing the attempt. func (l *loop) probeStaleness(at *attempt) (time.Duration, bool) { + // The latch outranks the channel: an expiry arm has already decided + // authoritative is one whether or not the firing has been delivered yet + // (staleHolder.latched). + if age, ok := at.lc.stale.latched(); ok { + return age, true + } staleTimer, staleGen := at.lc.stale.current() select { case <-staleTimer.C(): From 996a59fe938cdc562f4ecad8975c1c7651e7a35c Mon Sep 17 00:00:00 2001 From: Jeremy Daer Date: Tue, 15 Sep 2026 22:00:25 -0700 Subject: [PATCH 48/51] Event feed: reconcile the run loop with the shipped feed contract MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The shipped feed's 410 `resume` URL re-enters at the epoch with the canonical filter set preserved — positioned in served history, so the servable history above the fence is not skipped — not at the present as the pre-merge branch documented. The recovery matrix accordingly treats an accepted gap's resume as a position-resume entry: pages save on acceptance and the buffered live events drain after them, since everything between the fence and the present is poll-served and nothing behind such an entry depends on the live buffer. The inbox lane's resume (since=0, the earliest retained item) is the same class for the same reason. A 409's two digests reach the host: Observer.FilterConflict fires with the body's position_digest and filters_digest before PositionRejected(filter_changed). The re-entry is unchanged — the digests are diagnostics, and the one that matters is a filters_digest differing from the SDK's own Filters.Digest() for the same set. The tier-2 driver follows the foundations: config and identifier params carry performers, exclude_performers and actor_types; poll rows forward performed_by_id and details; push events are pinned at the shipped eleven-key shape. Fixtures spell the resume URL as the epoch re-entry (since=), tests spell continuations as the position- carrying URLs the server actually returns, and SPEC §23 names the class change, the observer callback, the header echoes and since's signed 64-bit range. --- SPEC.md | 32 +++++++---- conformance/event-feed/README.md | 2 +- .../fixtures/16-gap-410-accepted-resume.json | 6 +- .../fixtures/23-gap-410-default-terminal.json | 2 +- .../fixtures/25-gap-handler-terminate.json | 2 +- .../27-hostile-resume-cross-origin.json | 2 +- go/README.md | 9 ++- go/pkg/basecamp/eventfeed/catchup_test.go | 27 ++++----- go/pkg/basecamp/eventfeed/connector_test.go | 6 +- go/pkg/basecamp/eventfeed/loop_test.go | 2 + go/pkg/basecamp/eventfeed/reconnect_test.go | 2 +- go/pkg/basecamp/eventfeed/recovery.go | 49 ++++++++++------ go/pkg/basecamp/eventfeed/recovery_test.go | 56 ++++++++++++------- .../eventfeed/scenario_conformance_test.go | 27 ++++++--- .../eventfeed/scenario_fixture_test.go | 36 +++++++----- .../eventfeed/scenario_harness_test.go | 9 ++- .../eventfeed/scenario_selftest_test.go | 2 +- go/pkg/basecamp/eventfeed/seams.go | 12 +++- 18 files changed, 181 insertions(+), 102 deletions(-) diff --git a/SPEC.md b/SPEC.md index e30e5c8778..82c424b448 100644 --- a/SPEC.md +++ b/SPEC.md @@ -3635,13 +3635,16 @@ section states honestly rather than papers over. **Present-class entries, defined.** The amendment below applies to every entry whose cursor resolves at the server's present head — the class, used by this name throughout -this section and the state table: the zero Cursor (bare present entry); `since="now"`; a -410 reset's resume URL (the server documents it as `since=now` with the canonical filter -set preserved); and a 400-position/409 re-entry that falls back to the present because no -poll-served id exists (the reset cursor is poll-lane-only — a live-delivered id never -positions a re-entry). Entries positioned in served history — `position=`, -`since=`, `since=0` — are **position-resume class** and keep the unamended per-page -save discipline. +this section and the state table: the zero Cursor (bare present entry); `since="now"`; +and a 400-position/409 re-entry that falls back to the present because no poll-served id +exists (the reset cursor is poll-lane-only — a live-delivered id never positions a +re-entry). Entries positioned in served history — `position=`, `since=`, `since=0`, +and a 410 reset's `resume` URL (the server documents it as re-entering **at the epoch** +with the canonical filter set preserved, so the servable history above the fence is not +skipped; the inbox lane's re-enters at `since=0`, the earliest retained item) — are +**position-resume class** and keep the unamended per-page save discipline: every event +between such an entry and the present is poll-served, so nothing behind it depends on +the live buffer. **Entry sequencing (present-class entries only):** hold the entry poll's returned position → take the **ownership cut** → fix the snapshot → drain-and-accept → only then @@ -3719,7 +3722,7 @@ SignalHandler : (Signal) → Accept | Terminate for a 410 gap. An unhandled semantic signal cannot disappear, and **a 410 never silently auto-continues**. - **Accept on `FeedGap`** resumes via the provided resume URL (it preserves the canonical - filter set). **Accept on `BufferOverflow`** means the consumer owns the acknowledged + filter set and re-enters at the epoch — a position-resume entry, Entry Boundary above). **Accept on `BufferOverflow`** means the consumer owns the acknowledged incompleteness — and acceptance is not license to skip retained deliveries (the conjunctive invariant above still gates the `save`). - **A registered handler is invoked exactly once per semantic signal, synchronously, on the @@ -3838,20 +3841,23 @@ END RECORD Cursor -- exactly one field set; the zero Cursor is the bare present entry position : String? -- resume/repair token (in-memory authoritative within a run; -- durable via write-through when saves succeed) - since : String? -- "now", "0", or a decimal event id + since : String? -- "now", "0", or a decimal event id (a signed 64-bit integer on + -- the wire; out of range draws the position 400, not an empty page) page_url : String? -- absolute URL: a `next` continuation OR a 410 resume URL. -- Same-origin + no-downgrade validated BEFORE any poll call -- (Continuation and Resume URL Validation) END RECORD PollPage -- the body envelope IS the contract; never bind to response headers + -- (X-Feed-Position and Link rel="next" merely echo position/next) events : List position : String -- the ONLY thing that ever advances the checkpoint next : String? -- continuation URL; absent = the walk reached its frozen head. -- Bound to that walk; NEVER persisted. END -- Poll errors carry a kind: transient | throttled(retry_after) | position_invalid | --- filter_invalid(server message) | filter_changed | gone(epoch_after_id, resume_url) | +-- filter_invalid(server message) | filter_changed(position_digest, filters_digest) | +-- gone(epoch_after_id, resume_url) | -- unauthorized | redirect_refused(location_origin) | unrecoverable(error). -- The adapter maps every §6/§7 outcome of the generated call onto exactly one kind: -- 429/503 and §7-retryable outcomes exhausted inside the seam → throttled(retry_after) @@ -4180,7 +4186,11 @@ extensible without breaking implementers: `connecting(attempt, delay)`, `connect `confirmed()`, `disconnected(reason, error)`, `catch_up_started(cursor)`, `page_delivered(count, position)`, `checkpoint(position)` (after that page's events were accepted), `checkpoint_save_failed(error)`, `caught_up()`, `gap(epoch_after_id, -resume_url)`, `position_rejected(kind)`, `stale_connection(since_last_frame)`, +resume_url)`, `position_rejected(kind)`, `filter_conflict(position_digest, filters_digest)` +(a 409's two bare srv2 digests — the one the refused position was minted for and the one +the request's filters hashed to; a `filters_digest` that differs from the SDK's own +`Filters.digest()` for the same set means the local canonicalization has drifted from the +server's), `stale_connection(since_last_frame)`, `buffer_overflow(dropped_count)`. All are observability-only; none carries a disposition. ### Security Invariants `[static]` diff --git a/conformance/event-feed/README.md b/conformance/event-feed/README.md index ec6ab4f481..1e673fb7d9 100644 --- a/conformance/event-feed/README.md +++ b/conformance/event-feed/README.md @@ -276,7 +276,7 @@ reason via a constant, not the literal. | `since=now` / bare entry mints the cursor at the newest visible id; an empty entry page positions above an in-flight lower id N | 2 | 19, 20 | | Safety-horizon bound: position-relative, best-effort, ~30s — never wall-clock | 2 | premise of 19/20 (not directly assertable client-side; the entry-boundary fixtures encode its consequence) | | Frozen-head `next` predicate: absent `next` = the walk reached its head | 2 | every fixture whose walk ends on a 200 page without `next`: 01, 02, 05, 07, 12, 16, 17, 19, 20, 22, 29 (mechanically derived; re-derive when the set changes) | -| 410 `resume` re-enters at `since=now` with the canonical filter set preserved | 2 | 16 (resume URL followed verbatim); 27 (hostile variant) | +| 410 `resume` re-enters at the epoch (`since=`, in served history — a position-resume entry) with the canonical filter set preserved | 2 | 16 (resume URL followed verbatim); 27 (hostile variant) | | 400-position / 409 re-entry semantics (`since=`, present-class fallback) | 2 | no PR-2 fixture — pinned at PR-4 | | Ticket statelessness + ~120s TTL (server-owned `expires_in`) | 2 | 05 (TTL-advance premise; `expires_in` never schedules anything) | | 3-second server heartbeat cadence (input to the 7500ms staleness policy) | 2 | no PR-2 fixture — PR 4 (staleness fixture 08) | diff --git a/conformance/event-feed/fixtures/16-gap-410-accepted-resume.json b/conformance/event-feed/fixtures/16-gap-410-accepted-resume.json index 02f92a89f7..ead30cda05 100644 --- a/conformance/event-feed/fixtures/16-gap-410-accepted-resume.json +++ b/conformance/event-feed/fixtures/16-gap-410-accepted-resume.json @@ -1,6 +1,6 @@ { "name": "16-gap-410-accepted-resume", - "description": "A 410 epoch gap with an accepting feedGap handler: Observer.gap fires, the handler is invoked exactly once with {feedGap, accept}, the provided resume URL is followed verbatim (a present-class re-entry at since=now preserving the canonical filter set), and the stream continues. The accept complement of fixture 23; with 24/25's records it corners bypass-configured-handler from the accept side. Wire pins: 410 body keys epoch_after_id/resume (class 1) and resume-follows-verbatim (class 2).", + "description": "A 410 epoch gap with an accepting feedGap handler: Observer.gap fires, the handler is invoked exactly once with {feedGap, accept}, the provided resume URL is followed verbatim (a position-resume re-entry at the epoch, preserving the canonical filter set), and the stream continues. The accept complement of fixture 23; with 24/25's records it corners bypass-configured-handler from the accept side. Wire pins: 410 body keys epoch_after_id/resume (class 1) and resume-follows-verbatim (class 2).", "config": { "position": "{{POS:0}}", "signalDisposition": { @@ -50,7 +50,7 @@ "body": { "error": "position expired", "epoch_after_id": 100, - "resume": "{{API_ORIGIN}}/events.json?since=now" + "resume": "{{API_ORIGIN}}/events.json?since=100" } } } @@ -72,7 +72,7 @@ }, { "expectPoll": { - "url": "{{API_ORIGIN}}/events.json?since=now", + "url": "{{API_ORIGIN}}/events.json?since=100", "respond": { "status": 200, "body": { diff --git a/conformance/event-feed/fixtures/23-gap-410-default-terminal.json b/conformance/event-feed/fixtures/23-gap-410-default-terminal.json index f2daab1c60..132dc45bbd 100644 --- a/conformance/event-feed/fixtures/23-gap-410-default-terminal.json +++ b/conformance/event-feed/fixtures/23-gap-410-default-terminal.json @@ -47,7 +47,7 @@ "body": { "error": "position expired", "epoch_after_id": 100, - "resume": "{{API_ORIGIN}}/events.json?since=now" + "resume": "{{API_ORIGIN}}/events.json?since=100" } } } diff --git a/conformance/event-feed/fixtures/25-gap-handler-terminate.json b/conformance/event-feed/fixtures/25-gap-handler-terminate.json index d25a301c7d..6788b95ba3 100644 --- a/conformance/event-feed/fixtures/25-gap-handler-terminate.json +++ b/conformance/event-feed/fixtures/25-gap-handler-terminate.json @@ -30,7 +30,7 @@ "body": { "error": "position expired", "epoch_after_id": 100, - "resume": "{{API_ORIGIN}}/events.json?since=now" + "resume": "{{API_ORIGIN}}/events.json?since=100" } } } diff --git a/conformance/event-feed/fixtures/27-hostile-resume-cross-origin.json b/conformance/event-feed/fixtures/27-hostile-resume-cross-origin.json index 5c797fa73a..a251f0f70a 100644 --- a/conformance/event-feed/fixtures/27-hostile-resume-cross-origin.json +++ b/conformance/event-feed/fixtures/27-hostile-resume-cross-origin.json @@ -30,7 +30,7 @@ "body": { "error": "position expired", "epoch_after_id": 100, - "resume": "https://attacker.example.com/events.json?since=now" + "resume": "https://attacker.example.com/events.json?since=100" } } } diff --git a/go/README.md b/go/README.md index 537b1c906a..8843afe2a0 100644 --- a/go/README.md +++ b/go/README.md @@ -813,11 +813,18 @@ import ( // Poll(ctx, cursor, filters) (eventfeed.PollPage, error) // PollEvents var minter eventfeed.TicketMinter var polls eventfeed.PollSource +var agentPersonID int64 // the acting principal's own person id ctx := context.Background() feed, err := eventfeed.New("https://3.basecampapi.com", "5951425", minter, polls, - eventfeed.WithFilters(eventfeed.Filters{Types: []string{"message.created"}}), + eventfeed.WithFilters(eventfeed.Filters{ + Types: []string{"message.created"}, + // The loop guard for an agent that acts on what it hears: its own + // performances excluded by id (the server's `self` literal, resolved + // by the caller — the connector takes ids only). + ExcludePerformers: []int64{agentPersonID}, + }), eventfeed.WithCheckpointStore(eventfeed.NewFileCheckpointStore("/var/lib/myapp/feed.json")), eventfeed.WithConsumerNamespace("myapp"), eventfeed.WithSignalHandler(func(sig eventfeed.Signal) eventfeed.Disposition { diff --git a/go/pkg/basecamp/eventfeed/catchup_test.go b/go/pkg/basecamp/eventfeed/catchup_test.go index 3bd75d1888..789b40fee0 100644 --- a/go/pkg/basecamp/eventfeed/catchup_test.go +++ b/go/pkg/basecamp/eventfeed/catchup_test.go @@ -24,8 +24,9 @@ import ( "github.com/basecamp/basecamp-sdk/go/pkg/basecamp/eventfeed/feedtest" ) -// pollEvent builds one poll-lane row: the eight always-present keys, and no -// visible_to_clients (poll rows omit it — the presence asymmetry). +// pollEvent builds one poll-lane row: the nine always-present keys +// (performed_by_id null — a direct action), and neither actor_type nor +// visible_to_clients (poll rows omit both — the presence asymmetry). func pollEvent(id int64) eventfeed.Event { return eventfeed.Event{ ID: id, @@ -99,7 +100,7 @@ func TestCatchUpWalkCheckpointsAfterEachPage(t *testing.T) { store.Stored("pos-0") h := storedHarness(t, store) h.minter.ScriptTicket(ticket(1)) - next := testOrigin + "/999/events.json?after=102" + next := testOrigin + "/999/events.json?position=102" h.polls.ScriptPage(eventfeed.PollPage{ Events: []eventfeed.Event{pollEvent(101), pollEvent(102)}, Position: "pos-1", @@ -410,7 +411,7 @@ func TestCheckpointSaveFailureContinues(t *testing.T) { } h := storedHarness(t, store, eventfeed.WithObserver(obs)) h.minter.ScriptTicket(ticket(1)) - next := testOrigin + "/999/events.json?after=101" + next := testOrigin + "/999/events.json?position=101" h.polls.ScriptPage(eventfeed.PollPage{Events: []eventfeed.Event{pollEvent(101)}, Position: "pos-1", Next: next}) h.polls.ScriptPage(eventfeed.PollPage{Events: []eventfeed.Event{pollEvent(102)}, Position: "pos-2"}) h.start() @@ -440,7 +441,7 @@ func TestHostileContinuationIsTerminal(t *testing.T) { h.polls.ScriptPage(eventfeed.PollPage{ Events: []eventfeed.Event{pollEvent(101)}, Position: "pos-1", - Next: "https://attacker.example.com/999/events.json?after=101&token=secret", + Next: "https://attacker.example.com/999/events.json?position=101&token=secret", }) h.start() @@ -483,7 +484,7 @@ func TestContinuationValidation(t *testing.T) { {"protocol downgrade", "http://3.basecampapi.com/999/events.json"}, {"foreign port", "https://3.basecampapi.com:8443/999/events.json"}, {"non-http scheme", "file:///etc/passwd"}, - {"relative", "/999/events.json?after=101"}, + {"relative", "/999/events.json?position=101"}, // Authority is not hostname: ":443" is a nonempty url.Host with an // empty hostname. CanonicalOrigin already reads Hostname(), so this // takes the terminal path; the case is here so it stays that way. @@ -513,7 +514,7 @@ func TestContinuationValidation(t *testing.T) { t.Run("same-origin continuation is followed", func(t *testing.T) { h := newHarness(t) h.minter.ScriptTicket(ticket(1)) - next := testOrigin + "/999/events.json?after=101&filter=x" + next := testOrigin + "/999/events.json?position=101&filter=x" h.polls.ScriptPage(eventfeed.PollPage{Events: []eventfeed.Event{pollEvent(101)}, Position: "pos-1", Next: next}) h.polls.ScriptPage(eventfeed.PollPage{Events: []eventfeed.Event{pollEvent(102)}, Position: "pos-2"}) h.start() @@ -750,7 +751,7 @@ func TestSocketFailureMidWalk(t *testing.T) { h := storedHarness(t, store) h.minter.ScriptTicket(ticket(1)) h.minter.ScriptTicket(ticket(2)) - next := testOrigin + "/999/events.json?after=101" + next := testOrigin + "/999/events.json?position=101" h.polls.ScriptPage(eventfeed.PollPage{Events: []eventfeed.Event{pollEvent(101)}, Position: "pos-1", Next: next}) h.polls.ScriptError(&eventfeed.PollError{Kind: eventfeed.PollTransient, Err: errors.New("502")}) h.start() @@ -1696,7 +1697,7 @@ func TestProtocolFatalBehindADeferredRecoverableOutranksIt(t *testing.T) { name: "page boundary", opts: []eventfeed.Option{eventfeed.WithLiveBufferCapacity(1)}, arm: func(h *harness) { - h.polls.ScriptPage(eventfeed.PollPage{Position: "pos-1", Next: testOrigin + "/999/events.json?after=1"}) + h.polls.ScriptPage(eventfeed.PollPage{Position: "pos-1", Next: testOrigin + "/999/events.json?position=1"}) h.polls.ScriptPage(eventfeed.PollPage{Position: "pos-2"}) }, wedge: func(conn *feedtest.Conn) { conn.Serve(framePing()) }, @@ -2013,7 +2014,7 @@ func TestPollPageWithNoPositionIsMalformed(t *testing.T) { store.Stored("pos-0") h := storedHarness(t, store) h.minter.ScriptTicket(ticket(1)) - next := testOrigin + "/999/events.json?after=101" + next := testOrigin + "/999/events.json?position=101" h.polls.ScriptPage(eventfeed.PollPage{ Events: []eventfeed.Event{pollEvent(101)}, Position: "pos-1", @@ -2270,7 +2271,7 @@ func TestStalenessDuringAnInFlightPollFinishesThePage(t *testing.T) { var mu sync.Mutex var ages []time.Duration h, deferral := staleDeferralHarness(t, store, &ages, &mu) - next := testOrigin + "/999/events.json?after=101" + next := testOrigin + "/999/events.json?position=101" h.polls.ScriptPage(eventfeed.PollPage{ Events: []eventfeed.Event{pollEvent(101)}, Position: "pos-1", @@ -3552,7 +3553,7 @@ func TestParkedFatalSurvivesALateGraceWakeAtThePageBoundary(t *testing.T) { // A reconnecting connector ends at Terminal(mint_failed) with two mint // calls; a correct one at Terminal(protocol_fatal) with one. h.minter.ScriptError(&eventfeed.MintError{Kind: eventfeed.MintUnrecoverable}) - h.polls.ScriptPage(eventfeed.PollPage{Position: "pos-1", Next: testOrigin + "/999/events.json?after=1"}) + h.polls.ScriptPage(eventfeed.PollPage{Position: "pos-1", Next: testOrigin + "/999/events.json?position=1"}) h.polls.ScriptPage(eventfeed.PollPage{Position: "pos-2"}) h.drainHandled() deferral := &deferralWatch{ch: make(chan struct{}, 4)} @@ -3682,7 +3683,7 @@ func TestExpiredStalenessAtThePageBoundaryOutranksAQueuedOverflow(t *testing.T) }) h.minter.ScriptTicket(ticket(1)) h.minter.ScriptError(&eventfeed.MintError{Kind: eventfeed.MintUnrecoverable}) - h.polls.ScriptPage(eventfeed.PollPage{Position: "pos-1", Next: testOrigin + "/999/events.json?after=1"}) + h.polls.ScriptPage(eventfeed.PollPage{Position: "pos-1", Next: testOrigin + "/999/events.json?position=1"}) h.drainHandled() var once sync.Once h.polls.OnCall(func(feedtest.PollCall) { diff --git a/go/pkg/basecamp/eventfeed/connector_test.go b/go/pkg/basecamp/eventfeed/connector_test.go index 88352f6fa7..8eec9c6bbe 100644 --- a/go/pkg/basecamp/eventfeed/connector_test.go +++ b/go/pkg/basecamp/eventfeed/connector_test.go @@ -485,7 +485,7 @@ func TestPollSeamCannotRepointTheLineage(t *testing.T) { c.Filters.Creators[0] = 98 }) h.minter.ScriptTicket(ticket(1)) - next := testOrigin + "/999/events.json?after=101" + next := testOrigin + "/999/events.json?position=101" h.polls.ScriptPage(eventfeed.PollPage{ Events: []eventfeed.Event{pollEvent(101)}, Position: "pos-1", @@ -1136,7 +1136,7 @@ func TestCloseDuringPageDeliveredFinishesThePageAndStops(t *testing.T) { h.polls.ScriptPage(eventfeed.PollPage{ Events: []eventfeed.Event{pollEvent(101)}, Position: "pos-1", - Next: testOrigin + "/999/events.json?after=101", + Next: testOrigin + "/999/events.json?position=101", }) h.polls.ScriptPage(eventfeed.PollPage{Events: []eventfeed.Event{pollEvent(102)}, Position: "pos-2"}) h.start() @@ -1231,7 +1231,7 @@ func TestCloseDuringPageDeliveredSilencesThePageBoundary(t *testing.T) { h.polls.ScriptPage(eventfeed.PollPage{ Events: []eventfeed.Event{pollEvent(101)}, Position: "pos-1", - Next: testOrigin + "/999/events.json?after=101", + Next: testOrigin + "/999/events.json?position=101", }) h.polls.ScriptPage(eventfeed.PollPage{Position: "pos-2"}) h.start() diff --git a/go/pkg/basecamp/eventfeed/loop_test.go b/go/pkg/basecamp/eventfeed/loop_test.go index 926ace1076..a9f1b352b1 100644 --- a/go/pkg/basecamp/eventfeed/loop_test.go +++ b/go/pkg/basecamp/eventfeed/loop_test.go @@ -429,6 +429,8 @@ func frameMessageCreatedAt(identifier string, id int64, createdAt string) []byte "created_at": createdAt, "bucket_id": 2, "creator_id": 3, + "performed_by_id": nil, + "actor_type": "person", "recording_id": 900, "visible_to_clients": false, } diff --git a/go/pkg/basecamp/eventfeed/reconnect_test.go b/go/pkg/basecamp/eventfeed/reconnect_test.go index 7b2750193e..c72aceea36 100644 --- a/go/pkg/basecamp/eventfeed/reconnect_test.go +++ b/go/pkg/basecamp/eventfeed/reconnect_test.go @@ -382,7 +382,7 @@ func TestWalkFailureBetweenPages(t *testing.T) { h := storedHarness(t, store) h.minter.ScriptTicket(ticket(1)) h.minter.ScriptTicket(ticket(2)) - next := testOrigin + "/999/events.json?after=101" + next := testOrigin + "/999/events.json?position=101" h.polls.ScriptPage(eventfeed.PollPage{ Events: []eventfeed.Event{pollEvent(101)}, Position: "pos-1", diff --git a/go/pkg/basecamp/eventfeed/recovery.go b/go/pkg/basecamp/eventfeed/recovery.go index ebf7a709a9..6bed0abe20 100644 --- a/go/pkg/basecamp/eventfeed/recovery.go +++ b/go/pkg/basecamp/eventfeed/recovery.go @@ -17,9 +17,9 @@ import ( // cursor comes from: a 410's is the resume URL the SERVER provided, gated // behind the FeedGap semantic signal; a 400-position's and a 409's is the // poll-lane-only reset cursor below. All three restart the entry boundary — -// the walk drops any held position and re-takes the ownership cut on the new -// entry — because a re-entry IS an entry; only the positions already saved -// survive it. +// the walk drops any held position and, where the new entry is +// present-class, re-takes the ownership cut on it — because a re-entry IS an +// entry; only the positions already saved survive it. // // A re-entry re-polls IMMEDIATELY: §23 models rows 17/18/19 as CatchingUp // self-loops with no wait of their own (only transients and throttles take @@ -77,6 +77,17 @@ func (l *loop) reenterWalk(at *attempt, pe *PollError) (walkStep, cycleOutcome, case PollGone: return l.recoverGone(at, pe) case PollFilterChanged: + // The 409 body names both sides of the conflict — the digest the + // refused position was minted for and the digest of the filters this + // request presented — and the observer is the only place they reach + // a host: the re-entry below is the same whatever they say, so they + // are diagnostics, not disposition. A filters_digest that differs + // from Filters.Digest() for the same set is the one that matters — + // it means the local srv2 canonicalization has drifted from the + // server's. + if l.cfg.observer.FilterConflict != nil { + l.cfg.observer.FilterConflict(pe.PositionDigest, pe.FiltersDigest) + } // Transition 19: the held position is DISCARDED before the re-entry. // Its lineage belongs to a filter set that is not this connector's, so // it can never be resumed from again — an attempt torn down before the @@ -139,23 +150,25 @@ func (l *loop) recoverGone(at *attempt, pe *PollError) (walkStep, cycleOutcome, Msg: "the 410 carried no resume URL to continue from", }}, true } - // The resume URL is a present-class entry (the server documents it as - // since=now with the canonical filter set preserved), so the entry - // boundary's hold-then-save discipline governs the pages it serves — - // and the old cursor is unusable, which is precisely the durability - // boundary the invariant's exclusion names. Unusable means DISCARDED, - // not merely superseded: leaving it in memory would let a socket torn - // down before the resume poll's first page re-select it on the next - // connection, drawing the same 410 and re-invoking a gap handler that - // has already been asked and has already answered Accept. The resume - // URL takes its place as latched reconnect state so the accepted - // present-class reset is what continues. + // The resume URL is a POSITION-RESUME entry: the server documents it + // as re-entering at the epoch (the inbox's at since=0, the earliest + // retained item) with the canonical filter set preserved, so the + // servable history above the fence is not skipped — an entry + // positioned in served history, where every event between the fence + // and the present is poll-served and the unamended per-page save + // discipline applies. The old cursor is unusable, which is precisely + // the durability boundary the invariant's exclusion names. Unusable + // means DISCARDED, not merely superseded: leaving it in memory would + // let a socket torn down before the resume poll's first page + // re-select it on the next connection, drawing the same 410 and + // re-invoking a gap handler that has already been asked and has + // already answered Accept. The resume URL takes its place as latched + // reconnect state so the accepted reset is what continues. l.position = "" - l.reentry = &reentryCursor{cursor: Cursor{PageURL: pe.ResumeURL}, presentClass: true} + l.reentry = &reentryCursor{cursor: Cursor{PageURL: pe.ResumeURL}} return walkStep{ - cursor: l.reentry.cursor, - presentClass: true, - reentry: true, + cursor: l.reentry.cursor, + reentry: true, }, cycleOutcome{}, false } // Terminate, or no handler: the typed terminal, and no save — the error diff --git a/go/pkg/basecamp/eventfeed/recovery_test.go b/go/pkg/basecamp/eventfeed/recovery_test.go index 56ec09735a..1d1d699139 100644 --- a/go/pkg/basecamp/eventfeed/recovery_test.go +++ b/go/pkg/basecamp/eventfeed/recovery_test.go @@ -99,9 +99,10 @@ func gonePoll(resume string) error { // TestFeedGapAcceptedResumesViaProvidedURL is fixture 16: a 410 with an // accepting handler fires Observer.Gap, invokes the handler exactly once, and -// re-enters via the URL the SERVER provided — verbatim, and as a present-class -// entry, so the resume page's position is held until the admitted straggler -// has been delivered. +// re-enters via the URL the SERVER provided — verbatim, and as a +// position-resume entry (the URL re-enters at the epoch, in served history), +// so the resume page's position saves as the page is accepted and the +// admitted live event drains after it. func TestFeedGapAcceptedResumesViaProvidedURL(t *testing.T) { store := feedtest.NewStore() store.Stored("pos-0") @@ -114,7 +115,7 @@ func TestFeedGapAcceptedResumesViaProvidedURL(t *testing.T) { eventfeed.WithSignalHandler(ledger.handler), eventfeed.WithObserver(obs)) h.minter.ScriptTicket(ticket(1)) - resume := testOrigin + "/999/events.json?since=now" + resume := testOrigin + "/999/events.json?since=100" h.polls.ScriptError(gonePoll(resume)) h.polls.ScriptPage(eventfeed.PollPage{Position: "pos-1"}) h.start() @@ -132,7 +133,7 @@ func TestFeedGapAcceptedResumesViaProvidedURL(t *testing.T) { if len(gaps) != 1 || gaps[0] != fmt.Sprintf("gap 100 %s", testOrigin) { t.Fatalf("Observer.Gap = %v, want exactly one firing carrying the epoch and the resume URL's ORIGIN", gaps) } - if strings.Contains(gaps[0], "?") || strings.Contains(gaps[0], "since=now") { + if strings.Contains(gaps[0], "?") || strings.Contains(gaps[0], "since=100") { t.Errorf("Observer.Gap = %q, which retains the resume URL's query", gaps[0]) } assertInvocations(t, ledger.invocations(), "feedGap/accept") @@ -154,10 +155,12 @@ func TestFeedGapAcceptedResumesViaProvidedURL(t *testing.T) { if calls[1].Cursor != (eventfeed.Cursor{PageURL: resume}) { t.Fatalf("re-entry cursor = %+v, want the provided resume URL verbatim", calls[1].Cursor) } - // The resume is a present-class entry: the held position saves only after - // the buffered straggler is delivered. A position-resume treatment would - // save first and fail this ledger. - assertLedger(t, h.ledger(), []string{"event 41", "save pos-1"}) + // The resume is a position-resume entry: the page saves on acceptance and + // the buffered live event drains after it. A present-class treatment + // would hold the save behind the drain and fail this ledger — and the + // resume URL is the server's re-entry at the epoch, so a straggler behind + // it is poll-repairable, unlike one behind a present entry. + assertLedger(t, h.ledger(), []string{"save pos-1", "event 41"}) assertPositions(t, store.Saves(), "pos-1") if conn.Closed() { t.Fatal("an accepted gap continues the feed on the same socket") @@ -180,7 +183,7 @@ func TestFeedGapDefaultTerminal(t *testing.T) { Gap: func(int64, string) { gaps++ }, })) h.minter.ScriptTicket(ticket(1)) - resume := testOrigin + "/999/events.json?since=now" + resume := testOrigin + "/999/events.json?since=100" h.polls.ScriptError(gonePoll(resume)) h.polls.ScriptPage(eventfeed.PollPage{Position: "pos-1"}) // never reached base := runtime.NumGoroutine() @@ -223,7 +226,7 @@ func TestFeedGapHandlerTerminate(t *testing.T) { ledger := &signalLedger{give: eventfeed.Terminate} h := storedHarness(t, store, eventfeed.WithSignalHandler(ledger.handler)) h.minter.ScriptTicket(ticket(1)) - resume := testOrigin + "/999/events.json?since=now" + resume := testOrigin + "/999/events.json?since=100" h.polls.ScriptError(gonePoll(resume)) h.start() @@ -264,7 +267,7 @@ func TestFeedGapAcceptedHostileResumeIsInvalidContinuation(t *testing.T) { ledger := &signalLedger{give: eventfeed.Accept} h := storedHarness(t, store, eventfeed.WithSignalHandler(ledger.handler)) h.minter.ScriptTicket(ticket(1)) - h.polls.ScriptError(gonePoll("https://attacker.example.com/999/events.json?since=now&token=secret")) + h.polls.ScriptError(gonePoll("https://attacker.example.com/999/events.json?since=100&token=secret")) h.start() conn := h.driveToSubscribed() @@ -417,7 +420,7 @@ func TestRepairWalkBuffersLiveFramesAndDrainsThem(t *testing.T) { store.Stored("pos-0") h := storedHarness(t, store) h.minter.ScriptTicket(ticket(1)) - next := testOrigin + "/999/events.json?after=91" + next := testOrigin + "/999/events.json?position=91" h.polls.ScriptPage(eventfeed.PollPage{Position: "pos-1"}) h.polls.ScriptPage(eventfeed.PollPage{ Events: []eventfeed.Event{pollEvent(91)}, @@ -468,7 +471,7 @@ func TestPositionInvalidReEntersAtLastPollServedID(t *testing.T) { PositionRejected: func(kind eventfeed.PollErrorKind) { rejected = append(rejected, kind) }, })) h.minter.ScriptTicket(ticket(1)) - next := testOrigin + "/999/events.json?after=102" + next := testOrigin + "/999/events.json?position=102" h.polls.ScriptPage(eventfeed.PollPage{ Events: []eventfeed.Event{pollEvent(101), pollEvent(102)}, Position: "pos-1", @@ -575,12 +578,20 @@ func TestFilterChangedDiscardsTheHeldPosition(t *testing.T) { store := feedtest.NewStore() store.Stored("pos-0") var rejected []eventfeed.PollErrorKind + var conflicts []string h := storedHarness(t, store, eventfeed.WithObserver(eventfeed.Observer{ PositionRejected: func(kind eventfeed.PollErrorKind) { rejected = append(rejected, kind) }, + FilterConflict: func(positionDigest, filtersDigest string) { + conflicts = append(conflicts, positionDigest+"/"+filtersDigest) + }, })) h.minter.ScriptTicket(ticket(1)) h.minter.ScriptTicket(ticket(2)) - h.polls.ScriptError(&eventfeed.PollError{Kind: eventfeed.PollFilterChanged}) + h.polls.ScriptError(&eventfeed.PollError{ + Kind: eventfeed.PollFilterChanged, + PositionDigest: "38b223c13c89dc89", + FiltersDigest: "44136fa355b3678a", + }) h.polls.ScriptError(&eventfeed.PollError{Kind: eventfeed.PollTransient, Err: errors.New("502")}) h.polls.ScriptPage(eventfeed.PollPage{Position: "pos-1"}) h.start() @@ -611,6 +622,11 @@ func TestFilterChangedDiscardsTheHeldPosition(t *testing.T) { if len(rejected) != 1 || rejected[0] != eventfeed.PollFilterChanged { t.Fatalf("Observer.PositionRejected = %v, want one filter_changed", rejected) } + // The 409 body's two digests reach the host through the observer, once, + // as data: the re-entry above is the same whatever they say. + if len(conflicts) != 1 || conflicts[0] != "38b223c13c89dc89/44136fa355b3678a" { + t.Fatalf("Observer.FilterConflict = %v, want exactly one firing with the body's two digests", conflicts) + } assertPositions(t, store.Saves(), "pos-1") } @@ -657,7 +673,7 @@ func TestPollRetryIndexGrowsAndResets(t *testing.T) { transient := func() error { return &eventfeed.PollError{Kind: eventfeed.PollTransient, Err: errors.New("502")} } - next := testOrigin + "/999/events.json?after=101" + next := testOrigin + "/999/events.json?position=101" h.polls.ScriptError(transient()) // k=1 h.polls.ScriptError(transient()) // k=2 h.polls.ScriptPage(eventfeed.PollPage{Events: []eventfeed.Event{pollEvent(101)}, Position: "pos-1", Next: next}) @@ -696,7 +712,7 @@ func TestResetCursorSurvivesAReconnect(t *testing.T) { h := storedHarness(t, store) h.minter.ScriptTicket(ticket(1)) h.minter.ScriptTicket(ticket(2)) - next := testOrigin + "/999/events.json?after=101" + next := testOrigin + "/999/events.json?position=101" h.polls.ScriptPage(eventfeed.PollPage{ Events: []eventfeed.Event{pollEvent(101)}, Position: "pos-1", @@ -739,7 +755,7 @@ func TestResetCursorSurvivesAReconnect(t *testing.T) { // connection's entry selects, drawing the same 410 and re-invoking a gap // handler that has already been asked and has already answered Accept. The // accepted resume URL takes its place until a page replaces it, so the -// present-class reset the consumer accepted is what actually continues. +// reset the consumer accepted is what actually continues. func TestAcceptedGapResumeSurvivesAReconnect(t *testing.T) { store := feedtest.NewStore() store.Stored("pos-0") @@ -747,7 +763,7 @@ func TestAcceptedGapResumeSurvivesAReconnect(t *testing.T) { h := storedHarness(t, store, eventfeed.WithSignalHandler(ledger.handler)) h.minter.ScriptTicket(ticket(1)) h.minter.ScriptTicket(ticket(2)) - resume := testOrigin + "/999/events.json?since=now" + resume := testOrigin + "/999/events.json?since=100" h.polls.ScriptError(gonePoll(resume)) h.polls.ScriptError(&eventfeed.PollError{Kind: eventfeed.PollTransient, Err: errors.New("502")}) h.polls.ScriptPage(eventfeed.PollPage{Position: "pos-1"}) @@ -810,7 +826,7 @@ func TestCatchUpStartedRedactsTheLatchedResumeURL(t *testing.T) { h.minter.ScriptTicket(ticket(1)) h.minter.ScriptTicket(ticket(2)) const secret = "resume-token-do-not-log" - resume := testOrigin + "/999/events.json?since=now&t=" + secret + resume := testOrigin + "/999/events.json?since=100&t=" + secret h.polls.ScriptError(gonePoll(resume)) h.polls.ScriptError(&eventfeed.PollError{Kind: eventfeed.PollUnauthorized}) h.polls.ScriptPage(eventfeed.PollPage{Position: "pos-1"}) diff --git a/go/pkg/basecamp/eventfeed/scenario_conformance_test.go b/go/pkg/basecamp/eventfeed/scenario_conformance_test.go index 29fbcbf661..0bfe149e5d 100644 --- a/go/pkg/basecamp/eventfeed/scenario_conformance_test.go +++ b/go/pkg/basecamp/eventfeed/scenario_conformance_test.go @@ -536,6 +536,15 @@ func derivedQuery(call *pollCall) map[string]string { if len(call.filters.Creators) > 0 { query["creators"] = joinIDs(call.filters.Creators) } + if len(call.filters.Performers) > 0 { + query["performers"] = joinIDs(call.filters.Performers) + } + if len(call.filters.ExcludePerformers) > 0 { + query["exclude_performers"] = joinIDs(call.filters.ExcludePerformers) + } + if len(call.filters.ActorTypes) > 0 { + query["actor_types"] = strings.Join(call.filters.ActorTypes, ",") + } return query } @@ -1106,14 +1115,16 @@ func pollPageFrom(body json.RawMessage) (pollOutcome, error) { return pollOutcome{}, fmt.Errorf("poll event created_at: %w", err) } page.Events = append(page.Events, eventfeed.Event{ - ID: row.ID, - Kind: row.Kind, - EventType: row.EventType, - Action: row.Action, - CreatedAt: createdAt, - BucketID: row.BucketID, - CreatorID: row.CreatorID, - RecordingID: row.RecordingID, + ID: row.ID, + Kind: row.Kind, + EventType: row.EventType, + Action: row.Action, + CreatedAt: createdAt, + BucketID: row.BucketID, + CreatorID: row.CreatorID, + PerformedByID: row.PerformedByID, + RecordingID: row.RecordingID, + Details: row.Details, }) } return pollOutcome{page: page}, nil diff --git a/go/pkg/basecamp/eventfeed/scenario_fixture_test.go b/go/pkg/basecamp/eventfeed/scenario_fixture_test.go index a53163623f..4d8600160c 100644 --- a/go/pkg/basecamp/eventfeed/scenario_fixture_test.go +++ b/go/pkg/basecamp/eventfeed/scenario_fixture_test.go @@ -42,6 +42,9 @@ type scenarioConfig struct { Types []string `json:"types"` Buckets []int64 `json:"buckets"` Creators []int64 `json:"creators"` + Performers []int64 `json:"performers"` + ExcludePerformers []int64 `json:"exclude_performers"` + ActorTypes []string `json:"actorTypes"` Position string `json:"position"` ConfirmationDeadlineMs int `json:"confirmationDeadlineMs"` RepairPollBaseMs int `json:"repairPollBaseMs"` @@ -173,14 +176,16 @@ type pollEnvelope struct { } type pollEventRow struct { - ID int64 `json:"id"` - Kind string `json:"kind"` - EventType string `json:"event_type"` - Action string `json:"action"` - CreatedAt string `json:"created_at"` - BucketID int64 `json:"bucket_id"` - CreatorID int64 `json:"creator_id"` - RecordingID int64 `json:"recording_id"` + ID int64 `json:"id"` + Kind string `json:"kind"` + EventType string `json:"event_type"` + Action string `json:"action"` + CreatedAt string `json:"created_at"` + BucketID int64 `json:"bucket_id"` + CreatorID int64 `json:"creator_id"` + PerformedByID *int64 `json:"performed_by_id"` + RecordingID int64 `json:"recording_id"` + Details json.RawMessage `json:"details"` } type goneBody struct { @@ -582,10 +587,11 @@ func validateServe(s *serveStep) error { } } -// validatePushEvent pins the 9-key push payload: a fixture's event body is -// forwarded to the connector VERBATIM, so the driver only checks that the -// contract's keys are all present (the schema requires them; a driver that -// never looked would let a rewritten fixture through). +// validatePushEvent pins the 11-key push payload (plus the optional details +// object): a fixture's event body is forwarded to the connector VERBATIM, so +// the driver only checks that the contract's keys are all present (the +// schema requires them; a driver that never looked would let a rewritten +// fixture through). func validatePushEvent(raw json.RawMessage) error { keys, err := objectKeys(raw) if err != nil { @@ -593,7 +599,8 @@ func validatePushEvent(raw json.RawMessage) error { } for _, key := range []string{ "id", "kind", "event_type", "action", "created_at", - "bucket_id", "creator_id", "recording_id", "visible_to_clients", + "bucket_id", "creator_id", "performed_by_id", "actor_type", "recording_id", + "visible_to_clients", } { if _, ok := keys[key]; !ok { return fmt.Errorf("push event is missing required key %q", key) @@ -601,7 +608,8 @@ func validatePushEvent(raw json.RawMessage) error { } return allowedKeys("push event", keys, "id", "kind", "event_type", "action", "created_at", - "bucket_id", "creator_id", "recording_id", "visible_to_clients") + "bucket_id", "creator_id", "performed_by_id", "actor_type", "recording_id", + "visible_to_clients", "details") } // decodePollQuery decodes expectPoll.query into its subset/exact form. diff --git a/go/pkg/basecamp/eventfeed/scenario_harness_test.go b/go/pkg/basecamp/eventfeed/scenario_harness_test.go index 5ae6fc9fd6..43051b5ef7 100644 --- a/go/pkg/basecamp/eventfeed/scenario_harness_test.go +++ b/go/pkg/basecamp/eventfeed/scenario_harness_test.go @@ -665,9 +665,12 @@ func (h *scenarioHarness) newConnector(cfg scenarioConfig) (*eventfeed.Connector eventfeed.WithCheckpointStore(h), eventfeed.WithConsumerNamespace(scenarioNamespace), eventfeed.WithFilters(eventfeed.Filters{ - Types: cfg.Types, - Buckets: cfg.Buckets, - Creators: cfg.Creators, + Types: cfg.Types, + Buckets: cfg.Buckets, + Creators: cfg.Creators, + Performers: cfg.Performers, + ExcludePerformers: cfg.ExcludePerformers, + ActorTypes: cfg.ActorTypes, }), eventfeed.WithObserver(eventfeed.Observer{ Gap: func(epochAfterID int64, _ string) { h.recordGap(epochAfterID) }, diff --git a/go/pkg/basecamp/eventfeed/scenario_selftest_test.go b/go/pkg/basecamp/eventfeed/scenario_selftest_test.go index 214057c81f..21575083b3 100644 --- a/go/pkg/basecamp/eventfeed/scenario_selftest_test.go +++ b/go/pkg/basecamp/eventfeed/scenario_selftest_test.go @@ -449,7 +449,7 @@ func TestScenarioDriverRejectsUnmodelledScripts(t *testing.T) { }, { name: "a push event missing a payload key", - script: `{"name":"x","description":"d","steps":[{"serve":{"frame":"message","event":{"id":1,"kind":"m","event_type":"t","action":"a","created_at":"2026-08-01T12:00:00Z","bucket_id":2,"creator_id":3,"recording_id":4}}}],"finally":{"state":"closed"}}`, + script: `{"name":"x","description":"d","steps":[{"serve":{"frame":"message","event":{"id":1,"kind":"m","event_type":"t","action":"a","created_at":"2026-08-01T12:00:00Z","bucket_id":2,"creator_id":3,"performed_by_id":null,"actor_type":"person","recording_id":4}}}],"finally":{"state":"closed"}}`, wants: "visible_to_clients", }, { diff --git a/go/pkg/basecamp/eventfeed/seams.go b/go/pkg/basecamp/eventfeed/seams.go index f04deee971..ce21259c25 100644 --- a/go/pkg/basecamp/eventfeed/seams.go +++ b/go/pkg/basecamp/eventfeed/seams.go @@ -111,7 +111,9 @@ type Cursor struct { // Position is the durable resume/repair token (in-memory authoritative // within a run; durable via write-through when saves succeed). Position string - // Since is "now", "0", or a decimal event id. + // Since is "now", "0", or a decimal event id — a signed 64-bit integer + // on the wire; a value outside that range draws the position 400, not + // an empty page. On the inbox lane it is an item (addressing) id. Since string // PageURL is an absolute URL: a `next` continuation or a 410 resume URL. // Same-origin + no-downgrade validated before any poll call; never @@ -120,7 +122,8 @@ type Cursor struct { } // PollPage is one poll-lane page. The body envelope is the contract — never -// bind to response headers. +// bind to response headers: the X-Feed-Position and Link rel="next" response +// headers merely echo Position and Next. type PollPage struct { // Events are the page's rows, in strict event-id order. Events []Event @@ -562,6 +565,11 @@ type Observer struct { Gap func(epochAfterID int64, resumeURL string) // PositionRejected fires on a 400-position or 409 re-entry. PositionRejected func(kind PollErrorKind) + // FilterConflict fires on a 409, before PositionRejected(filter_changed), + // with the body's two digests: the srv2 digest the refused position was + // minted for, and the digest of the filters this request presented. + // Observability only — the re-entry is the same either way. + FilterConflict func(positionDigest, filtersDigest string) // StaleConnection fires when staleness tears a socket down. StaleConnection func(sinceLastFrame time.Duration) // BufferOverflow fires when the live buffer drops events — observability From baa295bdc587e9d8c3e9594dd6951e272eb60dd9 Mon Sep 17 00:00:00 2001 From: Jeremy Daer Date: Tue, 15 Sep 2026 22:59:08 -0700 Subject: [PATCH 49/51] =?UTF-8?q?Event=20feed:=20pin=20the=20FeedGap=20res?= =?UTF-8?q?ume=20URL=20and=20the=20close=20code=20at=20tier=202,=20and=20s?= =?UTF-8?q?ettle=20two=20=C2=A723=20sentences?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The tier-2 driver now proves more of what it claims. A feedGap signal expectation can pin the resume URL the handler received — whole, as the record contract says — and fixtures 16, 23, 25 and 27 do; a scripted 409's digests reach the connector on the poll error; every expected client close must carry status 1000; a terminal element must be exactly (Event{}, error); an observation record no expect step consumed — a second Gap, a stray PositionRejected, an unmatched signal — fails the scenario at `finally` as an unmatched outbound action does; and the query allowlist admits the three feed dimensions the schema already permits. Two §23 sentences are settled against the code rather than left in disagreement with it. Subscribe is sent on the welcome that opens each connection, which is the only welcome Action Cable sends; a later frame of that type is liveness only. And the two URL-bearing observer callbacks carry poll and resume URLs reduced to their origin — they are server-chosen text on a logging surface — while the FeedGap signal hands the handler the resume URL whole, since its disposition is a decision about which URL to follow. --- SPEC.md | 12 +++-- .../fixtures/16-gap-410-accepted-resume.json | 7 +++ .../fixtures/23-gap-410-default-terminal.json | 7 +++ .../fixtures/25-gap-handler-terminate.json | 2 +- .../27-hostile-resume-cross-origin.json | 1 + conformance/event-feed/schema.json | 5 ++ .../eventfeed/scenario_conformance_test.go | 48 ++++++++++++++++++- .../eventfeed/scenario_fixture_test.go | 9 +++- 8 files changed, 84 insertions(+), 7 deletions(-) diff --git a/SPEC.md b/SPEC.md index 82c424b448..081981bb6b 100644 --- a/SPEC.md +++ b/SPEC.md @@ -3566,7 +3566,10 @@ Two dispatch clarifications, pinned: map-marshaled so any retransmit is byte-identical. Subscription parameters mirror the poll filters with the same caps; `self` is resolved by the caller (Consumer Surface). The server absorbs identical retransmits and rejects different ones. -- Subscribe is sent on each `welcome` received. Confirm/reject correlation is exact string +- Subscribe is sent on the `welcome` that opens each connection — Action Cable sends exactly + one, before any other frame; a later frame of that type is liveness only, never a second + handshake (a mid-connection resubscribe would race the one already confirmed). Confirm/reject + correlation is exact string equality against the connector's identifier; frames carrying other identifiers are ignored. - Ping parsing accepts both `{"type":"ping"}` and `{"type":"ping","message":}`. @@ -4200,8 +4203,11 @@ server's), `stale_connection(since_last_frame)`, Values Are Never Rendered" names. A dial failure names the policy class it violated from a closed vocabulary, never any component of the URL, and never chains the transport's own error where a caller or runtime would render it. Poll and resume URLs - are not credentials — polls authenticate with the bearer header — so `gap(resume_url)` - and `catch_up_started(cursor)` carry them whole. + are not credentials — polls authenticate with the bearer header — but they are + server-chosen text arriving on a logging surface, so `gap(resume_url)` and + `catch_up_started(cursor)` carry them reduced to their origin (a fixed placeholder for a + cross-origin or unparsable one); the `FeedGap` signal hands the handler the resume URL + whole, since its disposition is a decision about which URL to follow. - **Bound the inbound frame size** (`EVENT_FEED_MAX_FRAME_BYTES`, 1 MiB default) and bound/truncate any error rendering of frame contents (§9's `MAX_ERROR_MESSAGE_LENGTH` applies). diff --git a/conformance/event-feed/fixtures/16-gap-410-accepted-resume.json b/conformance/event-feed/fixtures/16-gap-410-accepted-resume.json index ead30cda05..2f78befe72 100644 --- a/conformance/event-feed/fixtures/16-gap-410-accepted-resume.json +++ b/conformance/event-feed/fixtures/16-gap-410-accepted-resume.json @@ -60,6 +60,13 @@ "epochAfterId": 100 } }, + { + "expectSignal": { + "kind": "feedGap", + "epochAfterId": 100, + "resumeUrl": "{{API_ORIGIN}}/events.json?since=100" + } + }, { "expectHandlerInvocations": { "exact": [ diff --git a/conformance/event-feed/fixtures/23-gap-410-default-terminal.json b/conformance/event-feed/fixtures/23-gap-410-default-terminal.json index 132dc45bbd..985f5ebbf4 100644 --- a/conformance/event-feed/fixtures/23-gap-410-default-terminal.json +++ b/conformance/event-feed/fixtures/23-gap-410-default-terminal.json @@ -57,6 +57,13 @@ "epochAfterId": 100 } }, + { + "expectSignal": { + "kind": "feedGap", + "epochAfterId": 100, + "resumeUrl": "{{API_ORIGIN}}/events.json?since=100" + } + }, { "expectClientClose": {} }, diff --git a/conformance/event-feed/fixtures/25-gap-handler-terminate.json b/conformance/event-feed/fixtures/25-gap-handler-terminate.json index 6788b95ba3..36cb35ea97 100644 --- a/conformance/event-feed/fixtures/25-gap-handler-terminate.json +++ b/conformance/event-feed/fixtures/25-gap-handler-terminate.json @@ -36,7 +36,7 @@ } }, { "expectGap": { "epochAfterId": 100 } }, - { "expectSignal": { "kind": "feedGap", "epochAfterId": 100 } }, + { "expectSignal": { "kind": "feedGap", "epochAfterId": 100, "resumeUrl": "{{API_ORIGIN}}/events.json?since=100" } }, { "expectHandlerInvocations": { "exact": [{ "kind": "feedGap", "disposition": "terminate" }] diff --git a/conformance/event-feed/fixtures/27-hostile-resume-cross-origin.json b/conformance/event-feed/fixtures/27-hostile-resume-cross-origin.json index a251f0f70a..3803151159 100644 --- a/conformance/event-feed/fixtures/27-hostile-resume-cross-origin.json +++ b/conformance/event-feed/fixtures/27-hostile-resume-cross-origin.json @@ -36,6 +36,7 @@ } }, { "expectGap": { "epochAfterId": 100 } }, + {"expectSignal": {"kind": "feedGap", "epochAfterId": 100, "resumeUrl": "https://attacker.example.com/events.json?since=100"}}, { "expectHandlerInvocations": { "exact": [{ "kind": "feedGap", "disposition": "accept" }] diff --git a/conformance/event-feed/schema.json b/conformance/event-feed/schema.json index 12a4b02d20..5768630c1d 100644 --- a/conformance/event-feed/schema.json +++ b/conformance/event-feed/schema.json @@ -1527,6 +1527,11 @@ "epochAfterId": { "type": "integer", "minimum": 0 + }, + "resumeUrl": { + "type": "string", + "minLength": 1, + "description": "The resume URL the FeedGap record carried — whole, as the handler receives it (Observer.gap carries only its origin). Use {{API_ORIGIN}}/... for a same-origin resume; hostile fixtures use their literal foreign URL." } } } diff --git a/go/pkg/basecamp/eventfeed/scenario_conformance_test.go b/go/pkg/basecamp/eventfeed/scenario_conformance_test.go index 0bfe149e5d..ad6fa22d8b 100644 --- a/go/pkg/basecamp/eventfeed/scenario_conformance_test.go +++ b/go/pkg/basecamp/eventfeed/scenario_conformance_test.go @@ -26,6 +26,7 @@ import ( "net/http" "os" "path/filepath" + "reflect" "slices" "strconv" "strings" @@ -100,6 +101,12 @@ func runScenario(h *scenarioHarness, sc *scenario) (err error) { defer close(iteration) for event, iterErr := range conn.Events(ctx) { if iterErr != nil { + // The terminal element is exactly (Event{}, error): an event + // riding beside the error would be stale data a consumer must + // guess the meaning of. + if !reflect.DeepEqual(event, eventfeed.Event{}) { + h.violate("the terminal element carried event %d beside its error; §23's final element is (Event{}, error)", event.ID) + } h.recordTerminal(iterErr) continue } @@ -564,6 +571,12 @@ func (d *driver) expectClientClose() error { if frame.kind != "close" { return fmt.Errorf("the connector sent a frame where the script expects its close: %s", frame.data) } + // Every client-initiated close is a normal closure: the code is what the + // peer observes, and an abnormal or protocol-error status would read as + // a fault the connector never found. + if frame.code != 1000 { + return fmt.Errorf("the connector closed with status %d, want 1000 (normal closure)", frame.code) + } return nil } @@ -787,6 +800,9 @@ func matchSignal(step *expectSignalStep, signal eventfeed.Signal) string { if got.EpochAfterID != *step.EpochAfterID { return fmt.Sprintf("epoch_after_id %d, want %d", got.EpochAfterID, *step.EpochAfterID) } + if step.ResumeURL != nil && got.ResumeURL != *step.ResumeURL { + return fmt.Sprintf("resume url %q, want the server's %q", got.ResumeURL, *step.ResumeURL) + } default: return fmt.Sprintf("unknown signal type %T", got) } @@ -990,6 +1006,24 @@ func (d *driver) assertNoResidue() error { return fmt.Errorf("the scripted store calls %v were consumed %d deep — an unconsumed outcome fails the scenario", d.h.saveScript, d.h.saveUsed) } + // Observation ledgers are matched one record per expect step; a record + // nobody consumed is a callback the script never asked for — a duplicate + // Gap, a second signal, a stray PositionRejected — and it fails the + // scenario as an unmatched outbound action does. + for _, ledger := range []struct { + what string + taken, seen int + }{ + {"Observer.gap", d.h.gapsTaken, len(d.h.gaps)}, + {"semantic signal", d.h.signalsTaken, len(d.h.signals)}, + {"Observer.checkpointSaveFailed", d.h.saveFailuresTaken, d.h.saveFailures}, + {"Observer.positionRejected", d.h.posRejectedTaken, len(d.h.positionRejected)}, + {"invalid-frame disconnect", d.h.invalidFramesTaken, d.h.invalidFrames}, + } { + if ledger.taken != ledger.seen { + return fmt.Errorf("%d %s record(s) happened, %d matched by an expect step", ledger.seen, ledger.what, ledger.taken) + } + } return nil } @@ -1071,9 +1105,19 @@ func (d *driver) pollOutcomeFrom(respond pollRespond) (pollOutcome, error) { case 400: return pollOutcome{}, errors.New("400 poll responses are not modeled: the position-vs-filter discriminating bodies are class-1 literals still PROVISIONAL in the family README's dependency table, so a driver cannot discriminate them without guessing") case 409: + conflict := struct { + Error string `json:"error"` + PositionDigest string `json:"position_digest"` + FiltersDigest string `json:"filters_digest"` + }{} + if err := decodeStrict(respond.Body, &conflict); err != nil { + return pollOutcome{}, fmt.Errorf("409 body: %w", err) + } return pollOutcome{err: &eventfeed.PollError{ - Kind: eventfeed.PollFilterChanged, - Err: errors.New("poll responded 409"), + Kind: eventfeed.PollFilterChanged, + PositionDigest: conflict.PositionDigest, + FiltersDigest: conflict.FiltersDigest, + Err: errors.New("poll responded 409"), }}, nil case 410: return goneOutcome(respond.Body) diff --git a/go/pkg/basecamp/eventfeed/scenario_fixture_test.go b/go/pkg/basecamp/eventfeed/scenario_fixture_test.go index 4d8600160c..296917f312 100644 --- a/go/pkg/basecamp/eventfeed/scenario_fixture_test.go +++ b/go/pkg/basecamp/eventfeed/scenario_fixture_test.go @@ -229,6 +229,9 @@ type expectSignalStep struct { DroppedIDs []int64 `json:"droppedIds"` DroppedCount *int `json:"droppedCount"` EpochAfterID *int64 `json:"epochAfterId"` + // ResumeURL pins the FeedGap record's resume URL — the handler receives + // it whole, so a fixture can assert the exact URL the signal carried. + ResumeURL *string `json:"resumeUrl"` } type expectPositionRejectedStep struct { @@ -638,7 +641,8 @@ func decodePollQuery(step *expectPollStep) error { } for name := range params { switch name { - case "position", "since", "types", "buckets", "creators": + case "position", "since", "types", "buckets", "creators", + "performers", "exclude_performers", "actor_types": default: return fmt.Errorf("unknown query param %q", name) } @@ -707,6 +711,9 @@ func validateSignalExpect(s *expectSignalStep) error { if s.EpochAfterID != nil { return fmt.Errorf("a bufferOverflow signal carries no epochAfterId") } + if s.ResumeURL != nil { + return fmt.Errorf("a bufferOverflow signal carries no resumeUrl") + } case "feedGap": if s.EpochAfterID == nil { return fmt.Errorf("a feedGap signal needs epochAfterId") From e32dcfabb8385bcc59781b31c1ee35373fd51ff7 Mon Sep 17 00:00:00 2001 From: Jeremy Daer Date: Tue, 15 Sep 2026 23:36:07 -0700 Subject: [PATCH 50/51] Event feed: Close outranks the re-entry after FilterConflict, Wait publishes quiescence last, and tier 2 pins the observer's reduced gap URL MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A consumer that closes the connector from Observer.FilterConflict no longer receives PositionRejected after Close returned: the re-entry checks the run context first, as every callback that precedes a further act does. Events' deferred cleanup unwinds in the order the guarantee needs — cancel the run context, clear the registered cancel, then close the done channel — so Wait never returns while the run context is live. The tier-2 driver records Observer.gap's URL argument, and an expectGap step can pin it: the resume URL reduced to its origin, or the fixed cross-origin placeholder — never the whole URL, which only the FeedGap signal carries. The four 410 fixtures pin both. --- .../fixtures/16-gap-410-accepted-resume.json | 3 ++- .../fixtures/23-gap-410-default-terminal.json | 3 ++- .../fixtures/25-gap-handler-terminate.json | 2 +- .../fixtures/27-hostile-resume-cross-origin.json | 2 +- conformance/event-feed/schema.json | 5 +++++ go/pkg/basecamp/eventfeed/connector.go | 16 ++++++++++------ go/pkg/basecamp/eventfeed/recovery.go | 7 +++++++ .../eventfeed/scenario_conformance_test.go | 14 +++++++++----- .../basecamp/eventfeed/scenario_fixture_test.go | 5 +++++ .../basecamp/eventfeed/scenario_harness_test.go | 15 +++++++++++---- 10 files changed, 53 insertions(+), 19 deletions(-) diff --git a/conformance/event-feed/fixtures/16-gap-410-accepted-resume.json b/conformance/event-feed/fixtures/16-gap-410-accepted-resume.json index 2f78befe72..ec3981c575 100644 --- a/conformance/event-feed/fixtures/16-gap-410-accepted-resume.json +++ b/conformance/event-feed/fixtures/16-gap-410-accepted-resume.json @@ -57,7 +57,8 @@ }, { "expectGap": { - "epochAfterId": 100 + "epochAfterId": 100, + "resumeUrl": "{{API_ORIGIN}}" } }, { diff --git a/conformance/event-feed/fixtures/23-gap-410-default-terminal.json b/conformance/event-feed/fixtures/23-gap-410-default-terminal.json index 985f5ebbf4..ec772a37a3 100644 --- a/conformance/event-feed/fixtures/23-gap-410-default-terminal.json +++ b/conformance/event-feed/fixtures/23-gap-410-default-terminal.json @@ -54,7 +54,8 @@ }, { "expectGap": { - "epochAfterId": 100 + "epochAfterId": 100, + "resumeUrl": "{{API_ORIGIN}}" } }, { diff --git a/conformance/event-feed/fixtures/25-gap-handler-terminate.json b/conformance/event-feed/fixtures/25-gap-handler-terminate.json index 36cb35ea97..1f040d4516 100644 --- a/conformance/event-feed/fixtures/25-gap-handler-terminate.json +++ b/conformance/event-feed/fixtures/25-gap-handler-terminate.json @@ -35,7 +35,7 @@ } } }, - { "expectGap": { "epochAfterId": 100 } }, + { "expectGap": { "epochAfterId": 100, "resumeUrl": "{{API_ORIGIN}}" } }, { "expectSignal": { "kind": "feedGap", "epochAfterId": 100, "resumeUrl": "{{API_ORIGIN}}/events.json?since=100" } }, { "expectHandlerInvocations": { diff --git a/conformance/event-feed/fixtures/27-hostile-resume-cross-origin.json b/conformance/event-feed/fixtures/27-hostile-resume-cross-origin.json index 3803151159..2ae8782002 100644 --- a/conformance/event-feed/fixtures/27-hostile-resume-cross-origin.json +++ b/conformance/event-feed/fixtures/27-hostile-resume-cross-origin.json @@ -35,7 +35,7 @@ } } }, - { "expectGap": { "epochAfterId": 100 } }, + { "expectGap": { "epochAfterId": 100, "resumeUrl": "[cross-origin URL]" } }, {"expectSignal": {"kind": "feedGap", "epochAfterId": 100, "resumeUrl": "https://attacker.example.com/events.json?since=100"}}, { "expectHandlerInvocations": { diff --git a/conformance/event-feed/schema.json b/conformance/event-feed/schema.json index 5768630c1d..c0f2ef92c1 100644 --- a/conformance/event-feed/schema.json +++ b/conformance/event-feed/schema.json @@ -1466,6 +1466,11 @@ "epochAfterId": { "type": "integer", "minimum": 0 + }, + "resumeUrl": { + "type": "string", + "minLength": 1, + "description": "What Observer.gap received as its URL argument: the resume URL reduced to its origin ({{API_ORIGIN}} for a same-origin resume) or the fixed placeholder `[cross-origin URL]` — never the whole URL, which only the FeedGap signal carries (expectSignal.resumeUrl)." } } }, diff --git a/go/pkg/basecamp/eventfeed/connector.go b/go/pkg/basecamp/eventfeed/connector.go index dbc204d91c..c9a826bac8 100644 --- a/go/pkg/basecamp/eventfeed/connector.go +++ b/go/pkg/basecamp/eventfeed/connector.go @@ -474,8 +474,17 @@ func (c *Connector) Events(ctx context.Context) iter.Seq2[Event, error] { c.cancelRun = cancel } c.mu.Unlock() - defer cancel() + // Deferred in the order they must UNWIND, last first: the run's + // context is cancelled, then the registered cancel is cleared, and + // only then is quiescence published — so Wait never returns while + // the run context is live or a Close can still find a cancel to call. defer close(done) + defer func() { + c.mu.Lock() + c.cancelRun = nil + c.mu.Unlock() + }() + defer cancel() // Fired only once the cancellation is REGISTERED: before that point // the run is protected by the isClosed latch checked just above, not // by the cancel func, so a context handed out earlier would invite an @@ -483,11 +492,6 @@ func (c *Connector) Events(ctx context.Context) iter.Seq2[Event, error] { if c.hooks.runContext != nil { c.hooks.runContext(runCtx) } - defer func() { - c.mu.Lock() - c.cancelRun = nil - c.mu.Unlock() - }() newLoop(runCtx, &c.cfg, c.hooks).run(yield) } } diff --git a/go/pkg/basecamp/eventfeed/recovery.go b/go/pkg/basecamp/eventfeed/recovery.go index 6bed0abe20..42efc02ee0 100644 --- a/go/pkg/basecamp/eventfeed/recovery.go +++ b/go/pkg/basecamp/eventfeed/recovery.go @@ -88,6 +88,13 @@ func (l *loop) reenterWalk(at *attempt, pe *PollError) (walkStep, cycleOutcome, if l.cfg.observer.FilterConflict != nil { l.cfg.observer.FilterConflict(pe.PositionDigest, pe.FiltersDigest) } + // Close outranks the re-entry, as at every callback that precedes a + // further act: FilterConflict is a supported Close site, and the + // re-entry below would announce PositionRejected after Close returned. + if l.runCtx.Err() != nil { + l.disposeAttempt(at, nil) + return walkStep{}, cycleOutcome{kind: outcomeClosed}, true + } // Transition 19: the held position is DISCARDED before the re-entry. // Its lineage belongs to a filter set that is not this connector's, so // it can never be resumed from again — an attempt torn down before the diff --git a/go/pkg/basecamp/eventfeed/scenario_conformance_test.go b/go/pkg/basecamp/eventfeed/scenario_conformance_test.go index ad6fa22d8b..11f97f43b9 100644 --- a/go/pkg/basecamp/eventfeed/scenario_conformance_test.go +++ b/go/pkg/basecamp/eventfeed/scenario_conformance_test.go @@ -329,7 +329,7 @@ func (d *driver) runStep(step scenarioStep) error { case *errorExpect: return d.awaitTerminal(payload.Reason) case *expectGapStep: - return d.expectGap(payload.EpochAfterID) + return d.expectGap(payload) case *expectBufferedStep: return d.expectBuffered(payload.Count) case *expectSignalStep: @@ -732,13 +732,17 @@ func (d *driver) awaitIterationEnded() error { }) } -func (d *driver) expectGap(epochAfterID int64) error { - return d.h.await(fmt.Sprintf("Observer.gap(%d)", epochAfterID), func() (bool, string) { +func (d *driver) expectGap(step *expectGapStep) error { + return d.h.await(fmt.Sprintf("Observer.gap(%d)", step.EpochAfterID), func() (bool, string) { if d.h.gapsTaken >= len(d.h.gaps) { return false, "" } - if got := d.h.gaps[d.h.gapsTaken]; got != epochAfterID { - return false, fmt.Sprintf("gap epoch_after_id %d, want %d", got, epochAfterID) + got := d.h.gaps[d.h.gapsTaken] + if got.epochAfterID != step.EpochAfterID { + return false, fmt.Sprintf("gap epoch_after_id %d, want %d", got.epochAfterID, step.EpochAfterID) + } + if step.ResumeURL != nil && got.resumeURL != *step.ResumeURL { + return false, fmt.Sprintf("gap url argument %q, want the reduced %q", got.resumeURL, *step.ResumeURL) } d.h.gapsTaken++ return true, "" diff --git a/go/pkg/basecamp/eventfeed/scenario_fixture_test.go b/go/pkg/basecamp/eventfeed/scenario_fixture_test.go index 296917f312..12bb94085d 100644 --- a/go/pkg/basecamp/eventfeed/scenario_fixture_test.go +++ b/go/pkg/basecamp/eventfeed/scenario_fixture_test.go @@ -218,6 +218,11 @@ type expectStateStep struct { type expectGapStep struct { EpochAfterID int64 `json:"epochAfterId"` + // ResumeURL pins what Observer.gap received as its URL argument: the + // resume URL REDUCED to its origin (the API origin for a same-origin + // resume) or the fixed cross-origin placeholder — never the whole URL, + // which only the FeedGap signal carries. + ResumeURL *string `json:"resumeUrl"` } type expectBufferedStep struct { diff --git a/go/pkg/basecamp/eventfeed/scenario_harness_test.go b/go/pkg/basecamp/eventfeed/scenario_harness_test.go index 43051b5ef7..8d4e00bab2 100644 --- a/go/pkg/basecamp/eventfeed/scenario_harness_test.go +++ b/go/pkg/basecamp/eventfeed/scenario_harness_test.go @@ -112,7 +112,7 @@ type scenarioHarness struct { signals []eventfeed.Signal signalsTaken int invocations []invocation - gaps []int64 + gaps []gapRecord gapsTaken int saveFailures int saveFailuresTaken int @@ -612,10 +612,17 @@ func (h *scenarioHarness) recordInvocation(rec invocation) { h.notifyLocked() } -func (h *scenarioHarness) recordGap(epochAfterID int64) { +// gapRecord is one Observer.gap notification: the epoch and the URL argument +// as the connector rendered it for a logging surface. +type gapRecord struct { + epochAfterID int64 + resumeURL string +} + +func (h *scenarioHarness) recordGap(epochAfterID int64, resumeURL string) { h.mu.Lock() defer h.mu.Unlock() - h.gaps = append(h.gaps, epochAfterID) + h.gaps = append(h.gaps, gapRecord{epochAfterID: epochAfterID, resumeURL: resumeURL}) h.notifyLocked() } @@ -673,7 +680,7 @@ func (h *scenarioHarness) newConnector(cfg scenarioConfig) (*eventfeed.Connector ActorTypes: cfg.ActorTypes, }), eventfeed.WithObserver(eventfeed.Observer{ - Gap: func(epochAfterID int64, _ string) { h.recordGap(epochAfterID) }, + Gap: func(epochAfterID int64, resumeURL string) { h.recordGap(epochAfterID, resumeURL) }, CheckpointSaveFailed: func(error) { h.recordSaveFailed() }, PositionRejected: func(kind eventfeed.PollErrorKind) { h.recordPositionRejected(kind.String()) }, Disconnected: func(_ string, err error) { From de9171e84a4c5e07c1cd84309bd9d1884bd237a7 Mon Sep 17 00:00:00 2001 From: Jeremy Daer Date: Wed, 16 Sep 2026 00:20:31 -0700 Subject: [PATCH 51/51] Event feed: tier 2 records Observer.filterConflict, and fixture 34 pins the 409 re-entry MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The rejection ledger is one ordered record for both Observer.positionRejected and Observer.filterConflict, so a script that places expectFilterConflict — the 409 body's two digests — before expectPositionRejected(filter_changed) pins the payload and the ordering. Fixture 34 serves a 409 with both digests, watches the conflict callback precede the rejection, and follows the walk into its present-class re-entry at since=now: the 409 pin the family README had deferred. --- conformance/event-feed/README.md | 5 +- ...r-changed-409-reenters-at-the-present.json | 117 ++++++++++++++++++ conformance/event-feed/schema.json | 19 +++ .../eventfeed/scenario_conformance_test.go | 31 ++++- .../eventfeed/scenario_fixture_test.go | 17 +++ .../eventfeed/scenario_harness_test.go | 22 +++- 6 files changed, 203 insertions(+), 8 deletions(-) create mode 100644 conformance/event-feed/fixtures/34-filter-changed-409-reenters-at-the-present.json diff --git a/conformance/event-feed/README.md b/conformance/event-feed/README.md index 1e673fb7d9..fb36716dcb 100644 --- a/conformance/event-feed/README.md +++ b/conformance/event-feed/README.md @@ -239,6 +239,7 @@ revoked-mint threshold). | 28 | `28-checkpoint-load-failure.json` | store load Failed → Terminal(`checkpoint_load`) with ZERO wire attempts; distinct from Missing (which proceeds to a present entry) | | 29 | `29-checkpoint-save-failure-continues.json` | save Failed → feed continues and a SUBSEQUENT save is attempted (exact store-call script: no save circuit breaker) | | 30 | `30-continuation-redirect-cross-origin.json` | validated same-origin `next` answering 302 + cross-origin Location → Terminal(`invalid_continuation`), zero foreign egress | +| 34 | `34-filter-changed-409-reenters-at-the-present.json` | 409 with both digests → `Observer.filterConflict` (digests pinned) before `Observer.positionRejected(filter_changed)`; the held position is discarded and the walk re-enters at `since=now` (present-class, no poll-served id) | **Hostile-URL coverage note (stated author's choice, per the PR-1 review):** the downgrade (HTTPS→HTTP) variant is deliberately not a separate fixture here. Tier-2 @@ -267,7 +268,7 @@ reason via a constant, not the literal. | Poll body envelope keys `events` / `position` / `next` | 1 | every fixture serving a 200 poll: 01, 02, 05, 07, 12, 16, 17, 19, 20, 22, 26, 29, 30 (mechanically derived from the fixture files; re-derive when the set changes) | | Mint response body `{ticket, expires_in, url}`, status 200 | 1 | every fixture with `expectMint` (all but 28) | | Subscribe identifier literals: channel `EventsChannel`, param spellings `types`/`buckets`/`creators`/`performers`/`exclude_performers`/`actor_types`, comma-joined values | 1 | channel: every `expectSubscribe`; `types` spelling: 01 (its `expectSubscribe` pins `params` explicitly, single-valued); `buckets`/`creators` spellings + comma-joining: no PR-2 fixture — pinned at PR-4 (fixture 15, whose retransmit case also pins byte-identity of the identifier) | -| 409 body: all three keys `error` / `position_digest` / `filters_digest` required; digest values bare 16-hex (no `srv2-` prefix), `error` content unconstrained | 1 | schema-pinned shape only (the 409 respond variant requires all three keys); **no PR-2 fixture serves a 409** — pinned live at PR-4 (the tier-1 dispatch case additionally owns the wire pin when tier 1 lands) | +| 409 body: all three keys `error` / `position_digest` / `filters_digest` required; digest values bare 16-hex (no `srv2-` prefix), `error` content unconstrained | 1 | 34 (served, both digests forwarded to the connector and pinned on Observer.filterConflict); the tier-1 dispatch case additionally owns the wire pin | | 410 body keys `epoch_after_id` / `resume` | 1 | 16, 23, 25, 27 | | 400 position-vs-filter discriminating bodies (verbatim transcript shapes) | 1 | no PR-2 fixture — pinned at PR-4 (tier 1 additionally owns it when `PollEvents` lands); the schema's 400 variant requires a verbatim body and this table is its source of truth | | srv2 digest vectors (eleven-vector table) + canonicalization algorithm | 1 (vectors) / 2 (algorithm) | sibling family `conformance/event-feed-digest/` | @@ -277,7 +278,7 @@ reason via a constant, not the literal. | Safety-horizon bound: position-relative, best-effort, ~30s — never wall-clock | 2 | premise of 19/20 (not directly assertable client-side; the entry-boundary fixtures encode its consequence) | | Frozen-head `next` predicate: absent `next` = the walk reached its head | 2 | every fixture whose walk ends on a 200 page without `next`: 01, 02, 05, 07, 12, 16, 17, 19, 20, 22, 29 (mechanically derived; re-derive when the set changes) | | 410 `resume` re-enters at the epoch (`since=`, in served history — a position-resume entry) with the canonical filter set preserved | 2 | 16 (resume URL followed verbatim); 27 (hostile variant) | -| 400-position / 409 re-entry semantics (`since=`, present-class fallback) | 2 | no PR-2 fixture — pinned at PR-4 | +| 400-position / 409 re-entry semantics (`since=`, present-class fallback) | 2 | 34 (409, present-class fallback); the 400-position and poll-served-id variants remain PR-4's | | Ticket statelessness + ~120s TTL (server-owned `expires_in`) | 2 | 05 (TTL-advance premise; `expires_in` never schedules anything) | | 3-second server heartbeat cadence (input to the 7500ms staleness policy) | 2 | no PR-2 fixture — PR 4 (staleness fixture 08) | | Subscribe retransmit contract (identical absorbed, different rejected) | 2 | no PR-2 fixture — PR 4 (fixture 15) | diff --git a/conformance/event-feed/fixtures/34-filter-changed-409-reenters-at-the-present.json b/conformance/event-feed/fixtures/34-filter-changed-409-reenters-at-the-present.json new file mode 100644 index 0000000000..a670ec698b --- /dev/null +++ b/conformance/event-feed/fixtures/34-filter-changed-409-reenters-at-the-present.json @@ -0,0 +1,117 @@ +{ + "name": "34-filter-changed-409-reenters-at-the-present", + "description": "A held position minted for a different filter set: the poll answers 409 with position_digest and filters_digest (class-1 keys, bare srv2 hex), Observer.filterConflict fires with both digests BEFORE Observer.positionRejected(filter_changed), the held position is discarded and the walk re-enters at since=now — a present-class entry, since no id has been poll-served on this lineage — whose page is held through the drain and then saved. Row 19 of the state table; the 409 pin the family README deferred to this fixture.", + "config": { + "types": [ + "message.created" + ], + "position": "{{POS:0}}" + }, + "steps": [ + { + "expectMint": { + "respond": { + "status": 200, + "body": { + "ticket": "{{TICKET:1}}", + "expires_in": 120, + "url": "{{CABLE_URL:1}}" + } + } + } + }, + { + "expectConnect": { + "url": "{{CABLE_URL:1}}" + } + }, + { + "serve": { + "frame": "welcome" + } + }, + { + "expectSubscribe": { + "channel": "EventsChannel", + "params": { + "types": "message.created" + } + } + }, + { + "serve": { + "frame": "confirm" + } + }, + { + "expectPoll": { + "query": { + "position": "{{POS:0}}", + "types": "message.created" + }, + "respond": { + "status": 409, + "body": { + "error": "Positions are bound to the filter set they were minted for. Acknowledge the filter change by re-entering with since= or since=now.", + "position_digest": "5d0b83594d8e5630", + "filters_digest": "38b223c13c89dc89" + } + } + } + }, + { + "expectFilterConflict": { + "positionDigest": "5d0b83594d8e5630", + "filtersDigest": "38b223c13c89dc89" + } + }, + { + "expectPositionRejected": { + "kind": "filter_changed" + } + }, + { + "expectPoll": { + "query": { + "exact": { + "since": "now", + "types": "message.created" + } + }, + "respond": { + "status": 200, + "body": { + "events": [], + "position": "{{POS:1}}" + } + } + } + }, + { + "expectCheckpoint": { + "position": "{{POS:1}}" + } + } + ], + "finally": { + "state": "streaming", + "mintCount": 1, + "connectCount": 1, + "pollCount": 2, + "delivered": { + "exact": [] + }, + "checkpoints": { + "exact": [ + "{{POS:1}}" + ] + }, + "timers": { + "exact": { + "staleness": 1, + "repair-poll": 1 + } + }, + "socket": "open" + } +} diff --git a/conformance/event-feed/schema.json b/conformance/event-feed/schema.json index c0f2ef92c1..ddd90eb873 100644 --- a/conformance/event-feed/schema.json +++ b/conformance/event-feed/schema.json @@ -484,6 +484,25 @@ "expectDisconnectedInvalidFrame": { "const": true, "description": "Rendezvous: Observer.disconnected fired carrying the invalid-frame indication. Only `true` is valid — a falsy value would assert nothing." + }, + "expectFilterConflict": { + "type": "object", + "additionalProperties": false, + "required": [ + "positionDigest", + "filtersDigest" + ], + "description": "Rendezvous: Observer.filter_conflict fired with exactly these two bare srv2 digests (the 409 body's position_digest and filters_digest). Shares one ordered ledger with expectPositionRejected, so placing it before expectPositionRejected(filter_changed) pins that the conflict callback precedes the rejection it explains.", + "properties": { + "positionDigest": { + "type": "string", + "pattern": "^[0-9a-f]{16}$" + }, + "filtersDigest": { + "type": "string", + "pattern": "^[0-9a-f]{16}$" + } + } } } }, diff --git a/go/pkg/basecamp/eventfeed/scenario_conformance_test.go b/go/pkg/basecamp/eventfeed/scenario_conformance_test.go index 11f97f43b9..8d43fb9d47 100644 --- a/go/pkg/basecamp/eventfeed/scenario_conformance_test.go +++ b/go/pkg/basecamp/eventfeed/scenario_conformance_test.go @@ -275,7 +275,7 @@ func (d *driver) stepSatisfiedLocked(step scenarioStep) bool { return len(h.invocations) >= len(payload.Exact) case *expectSaveFailedStep: return h.saveFailuresTaken < h.saveFailures - case *expectPositionRejectedStep: + case *expectPositionRejectedStep, *expectFilterConflictStep: return h.posRejectedTaken < len(h.positionRejected) case *expectDisconnectedInvalidFrameStep: return h.invalidFramesTaken < h.invalidFrames @@ -340,6 +340,8 @@ func (d *driver) runStep(step scenarioStep) error { return d.expectSaveFailed() case *expectPositionRejectedStep: return d.expectPositionRejected(payload.Kind) + case *expectFilterConflictStep: + return d.expectFilterConflict(payload) case *expectDisconnectedInvalidFrameStep: return d.expectInvalidFrameDisconnect() default: @@ -837,8 +839,29 @@ func (d *driver) expectPositionRejected(kind string) error { if d.h.posRejectedTaken >= len(d.h.positionRejected) { return false, "" } - if got := d.h.positionRejected[d.h.posRejectedTaken]; got != kind { - return false, fmt.Sprintf("position_rejected kind %q, want %q", got, kind) + if got := d.h.positionRejected[d.h.posRejectedTaken]; got.kind != kind { + return false, fmt.Sprintf("rejection ledger next entry %q, want position_rejected %q", got.kind, kind) + } + d.h.posRejectedTaken++ + return true, "" + }) +} + +// expectFilterConflict consumes the next rejection-ledger entry, which must be +// the Observer.filterConflict firing carrying exactly the scripted digests — +// so a script that places it before expectPositionRejected(filter_changed) +// pins both the payload and the ordering. +func (d *driver) expectFilterConflict(step *expectFilterConflictStep) error { + return d.h.await("Observer.filter_conflict", func() (bool, string) { + if d.h.posRejectedTaken >= len(d.h.positionRejected) { + return false, "" + } + got := d.h.positionRejected[d.h.posRejectedTaken] + if got.kind != "filter_conflict" { + return false, fmt.Sprintf("rejection ledger next entry %q, want filter_conflict", got.kind) + } + if got.positionDigest != step.PositionDigest || got.filtersDigest != step.FiltersDigest { + return false, fmt.Sprintf("filter_conflict digests %s/%s, want %s/%s", got.positionDigest, got.filtersDigest, step.PositionDigest, step.FiltersDigest) } d.h.posRejectedTaken++ return true, "" @@ -1021,7 +1044,7 @@ func (d *driver) assertNoResidue() error { {"Observer.gap", d.h.gapsTaken, len(d.h.gaps)}, {"semantic signal", d.h.signalsTaken, len(d.h.signals)}, {"Observer.checkpointSaveFailed", d.h.saveFailuresTaken, d.h.saveFailures}, - {"Observer.positionRejected", d.h.posRejectedTaken, len(d.h.positionRejected)}, + {"Observer.positionRejected / filterConflict", d.h.posRejectedTaken, len(d.h.positionRejected)}, {"invalid-frame disconnect", d.h.invalidFramesTaken, d.h.invalidFrames}, } { if ledger.taken != ledger.seen { diff --git a/go/pkg/basecamp/eventfeed/scenario_fixture_test.go b/go/pkg/basecamp/eventfeed/scenario_fixture_test.go index 12bb94085d..539cf50d8c 100644 --- a/go/pkg/basecamp/eventfeed/scenario_fixture_test.go +++ b/go/pkg/basecamp/eventfeed/scenario_fixture_test.go @@ -243,6 +243,14 @@ type expectPositionRejectedStep struct { Kind string `json:"kind"` } +// expectFilterConflictStep pins Observer.filterConflict: the 409 body's two +// digests, in the rejection ledger's order (before the filter_changed +// rejection it explains). +type expectFilterConflictStep struct { + PositionDigest string `json:"positionDigest"` + FiltersDigest string `json:"filtersDigest"` +} + // --- loading ------------------------------------------------------------- // parseScenario decodes one substituted fixture, failing on anything the @@ -446,6 +454,15 @@ func decodeDirective(kind string, body json.RawMessage) (any, error) { return nil, fmt.Errorf("unknown kind %q", step.Kind) } return step, nil + case "expectFilterConflict": + step := &expectFilterConflictStep{} + if err := decodeStrict(body, step); err != nil { + return nil, err + } + if step.PositionDigest == "" || step.FiltersDigest == "" { + return nil, fmt.Errorf("a filter-conflict expectation needs both digests") + } + return step, nil case "expectDisconnectedInvalidFrame": var flag bool if err := decodeStrict(body, &flag); err != nil { diff --git a/go/pkg/basecamp/eventfeed/scenario_harness_test.go b/go/pkg/basecamp/eventfeed/scenario_harness_test.go index 8d4e00bab2..c153700e84 100644 --- a/go/pkg/basecamp/eventfeed/scenario_harness_test.go +++ b/go/pkg/basecamp/eventfeed/scenario_harness_test.go @@ -116,7 +116,7 @@ type scenarioHarness struct { gapsTaken int saveFailures int saveFailuresTaken int - positionRejected []string + positionRejected []rejectionRecord posRejectedTaken int invalidFrames int invalidFramesTaken int @@ -633,10 +633,27 @@ func (h *scenarioHarness) recordSaveFailed() { h.notifyLocked() } +// rejectionRecord is one entry of the ORDERED rejection ledger: an +// Observer.positionRejected firing (kind position_invalid / filter_changed) +// or an Observer.filterConflict firing (kind filter_conflict, with the 409 +// body's two digests). One ledger for both, so a script pins that the +// conflict callback precedes the rejection it explains. +type rejectionRecord struct { + kind string + positionDigest, filtersDigest string +} + func (h *scenarioHarness) recordPositionRejected(kind string) { h.mu.Lock() defer h.mu.Unlock() - h.positionRejected = append(h.positionRejected, kind) + h.positionRejected = append(h.positionRejected, rejectionRecord{kind: kind}) + h.notifyLocked() +} + +func (h *scenarioHarness) recordFilterConflict(positionDigest, filtersDigest string) { + h.mu.Lock() + defer h.mu.Unlock() + h.positionRejected = append(h.positionRejected, rejectionRecord{kind: "filter_conflict", positionDigest: positionDigest, filtersDigest: filtersDigest}) h.notifyLocked() } @@ -683,6 +700,7 @@ func (h *scenarioHarness) newConnector(cfg scenarioConfig) (*eventfeed.Connector Gap: func(epochAfterID int64, resumeURL string) { h.recordGap(epochAfterID, resumeURL) }, CheckpointSaveFailed: func(error) { h.recordSaveFailed() }, PositionRejected: func(kind eventfeed.PollErrorKind) { h.recordPositionRejected(kind.String()) }, + FilterConflict: h.recordFilterConflict, Disconnected: func(_ string, err error) { if eventfeed.ExportIsInvalidFrameError(err) { h.recordInvalidFrame()