diff --git a/CLAUDE.md b/CLAUDE.md index 73b18b5..b16bb77 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -78,7 +78,8 @@ All commands follow a `:RESOURCE:ACTION:` naming convention. New commands must u | `:PLACED:CREATE:`, `:PLACED:EVENT:` | Placed object (mine/explosive) lifecycle | | `:ACE3:DEATH:`, `:ACE3:UNCONSCIOUS:` | ACE3 integration events | | `:SYS:INIT:`, `:STORAGE:INIT:` | Initialize extension and storage | -| `:MISSION:START:`, `:MISSION:SAVE:` | Begin/end recording | +| `:MISSION:START:`, `:MISSION:SAVE:`, `:MISSION:DISCARD:` | Begin/end recording, or end it unsaved | +| `:MISSION:KEEP:` | Keep a recording the game moves on from (it met the addon's save limits) | ### Data Flow diff --git a/README.md b/README.md index f39901f..b5b2500 100644 --- a/README.md +++ b/README.md @@ -153,15 +153,43 @@ paced. `0` disables pacing. per mission (0 = unlimited). Once the limit is reached, spooling stops for that mission, and a later full resend is partial. -The spool also survives the game server. If it crashes mid-mission, or a save -is not confirmed, the mission stays in `spoolDir`. The next extension process -checks `spoolDir` at startup and then every minute, resends such missions to -the web service, and deletes each spool once the web service has saved its -mission: +The spool also survives the game server. If it crashes mid-mission, or a +mission ends without a confirmed save, the mission stays in `spoolDir`. The +extension checks `spoolDir` at startup and then every minute, resends such +missions to the web service, and deletes each spool once the web service has +saved its mission. A mission resent in full is marked as recovered, so a web +service at protocol version 3 does not take it for the producer's next +mission and save the mission it records now as `partial`. An older web service +does, so with one the running process resends its own missions in full only +while it records none: - A mission cut short by a crash is ended on the crashed process' behalf and saved as `partial`, with everything recorded up to the last flush (at most about a second is lost). - A mission whose save was not confirmed is saved as recorded. +- A mission whose discard was not confirmed is not resent; the web service is + only told again to drop it, including any `partial` recording it saved of + the mission in the meantime. This needs the spool: without `spoolDir`, or + with a spool that hit its limit or failed, nothing is kept, and the web + service saves the mission as `partial` once its resume grace expires. + +When the game moves on to the next mission without saving or discarding the +current one, for example after an admin `#missions`, the proxy ends it as +interrupted as soon as the next mission loads (`:STORAGE:INIT:`), and the web +service saves it as `partial`. Only if that is not confirmed within 30 seconds +does its spool stay behind for recovery. One that recorded nothing after its +start is discarded instead, when the web service supports it, and so is one +the addon would not save yet: it starts such a recording with `"discard"` and +sends `:MISSION:KEEP:` once it meets `OCAP_settings_minMissionTime`, so a +mission an admin only loaded and left with `#missions` or `#restart` leaves +no `partial` behind. A web service +without resume support is disconnected instead, which saves the mission as +`partial` there. + +When the addon ends a recording that does not meet its save limits (for +example one shorter than `OCAP_settings_minMissionTime`), it sends +`:MISSION:DISCARD:`, and the proxy tells the web service to drop the recording +(protocol version 3) and deletes its spool. A web service that predates +discard keeps saving such recordings as `partial`. Game servers can share one `spoolDir`, for example several instances started from the same directory. Each spool has an operating system file lock @@ -265,9 +293,11 @@ All commands follow a `:RESOURCE:ACTION:` naming convention. New commands must u | Command | Purpose | |---------|---------| | `:SYS:INIT:` | Initialize extension, send `:SYS:READY:` callback | -| `:STORAGE:INIT:` | Initialize storage backend, ungate buffered handlers | -| `:MISSION:START:` | Start recording mission | +| `:STORAGE:INIT:` | Initialize storage backend once per process (later calls reuse it), ungate buffered handlers; answers with a `:STORAGE:OK:` callback, or `:STORAGE:ERROR:` with the error | +| `:MISSION:START:` | Start recording mission; answers with a `:MISSION:OK:` callback, or `:MISSION:FAILED:` with the error. An optional third argument, a registration token, is echoed as the callback's last value. An optional fourth, `discard`, drops the recording if the game moves on from it before `:MISSION:KEEP:` (`websocket`) | +| `:MISSION:KEEP:` | The recording met the addon's save limits: if the game moves on from it now, it is kept as `partial` (`websocket`) | | `:MISSION:SAVE:` | End recording, flush data, upload if configured | +| `:MISSION:DISCARD:` | End recording without saving it (`memory`, `websocket`); answers with a `:MISSION:DISCARDED:` callback | | `:MISSION:FOCUS_START:` | Set playback focus start frame | | `:MISSION:FOCUS_END:` | Set playback focus end frame | | `:SYS:VERSION:` | Get extension version | diff --git a/cmd/ocap_recorder/main.go b/cmd/ocap_recorder/main.go index bbaabcd..fa2eca6 100644 --- a/cmd/ocap_recorder/main.go +++ b/cmd/ocap_recorder/main.go @@ -14,6 +14,7 @@ import ( "path/filepath" "runtime" "sync" + "sync/atomic" "time" "github.com/OCAP2/extension/v5/internal/api" @@ -91,6 +92,10 @@ var ( // storageReady is closed when storage (DB or memory) is initialized and ready storageReady = make(chan struct{}) storageReadyOnce sync.Once + // missionLoadedAt is when the last :STORAGE:INIT:, sent on every + // mission load, was dispatched. Compared with Before, which uses the + // monotonic clock, so a wall clock stepping back does not matter. + missionLoadedAt atomic.Pointer[time.Time] // Services parserService parser.Service @@ -312,9 +317,43 @@ func initAPIClient() { // handleNewMission handles the :MISSION:START: command: parses mission data, // delegates DB persistence to the storage backend, and sets runtime state. -func handleNewMission(e dispatcher.Event) (any, error) { +// It answers with a :MISSION:OK: callback, or :MISSION:FAILED: with the error, +// so the addon never waits for a mission that did not start. +// +// An optional third argument is the addon's registration token, echoed in +// both callbacks, so a late answer to an earlier mission's start is not taken +// for this one's. An optional fourth, "discard", drops the recording if the +// game moves on from it before :MISSION:KEEP: (see storage.Abandonable). +func handleNewMission(e dispatcher.Event) (result any, err error) { + token := "" + if len(e.Args) > 2 { + token = util.TrimQuotes(e.Args[2]) + } + reportFailure := func(msg string) { + if cbErr := a3interface.WriteArmaCallback(ExtensionName, ":MISSION:FAILED:", msg, token); cbErr != nil { + Logger.Warn("Failed to send MISSION:FAILED callback", "error", cbErr) + } + } + defer func() { + if r := recover(); r != nil { + reportFailure(fmt.Sprintf("panic during mission start: %v", r)) + panic(r) // the dispatcher logs it + } + }() + result, err = startNewMission(e) + if err != nil { + reportFailure(err.Error()) + return result, err + } + if err := a3interface.WriteArmaCallback(ExtensionName, ":MISSION:OK:", "OK", token); err != nil { + Logger.Warn("Failed to send MISSION:OK callback", "error", err) + } + return result, nil +} + +func startNewMission(e dispatcher.Event) (any, error) { if parserService == nil { - return nil, nil + return nil, fmt.Errorf("extension not initialized") } // 1. Parse (returns core types) @@ -334,20 +373,27 @@ func handleNewMission(e dispatcher.Event) (any, error) { MarkerCache.Reset() EntityCache.Reset() - // 3. Start backend (handles DB ops for GORM/SQLite backends, stores missionID internally) + // 3. Start backend (handles DB ops for GORM/SQLite backends, stores missionID internally), + // once the previous mission's save or discard, dispatched before this + // start, has run. + saveState.waitIdle() + // A start that waited behind a save while the game loaded the next + // mission belongs to the mission that is gone: nothing would record it. + if loaded := missionLoadedAt.Load(); loaded != nil && !e.Timestamp.IsZero() && e.Timestamp.Before(*loaded) { + return nil, fmt.Errorf("the mission was unloaded before it started") + } if storageBackend != nil { if err := storageBackend.StartMission(&coreMission, &coreWorld); err != nil { Logger.Error("Failed to start mission in storage backend", "error", err) return nil, err } + // Before :MISSION:OK:, so before the addon records anything. + if a, ok := storageBackend.(storage.Abandonable); ok && len(e.Args) > 3 && util.TrimQuotes(e.Args[3]) == "discard" { + a.SetDiscardAbandoned(true) + } } Logger.Info("New mission logged", "missionName", coreMission.MissionName) - - // 4. ArmA callback - if err := a3interface.WriteArmaCallback(ExtensionName, ":MISSION:OK:", "OK"); err != nil { - Logger.Warn("Failed to send MISSION:OK callback", "error", err) - } return nil, nil } @@ -360,11 +406,11 @@ func registerLifecycleHandlers(d *dispatcher.Dispatcher) { }) d.Register(":STORAGE:INIT:", func(e dispatcher.Event) (any, error) { - go func() { - if err := initStorage(); err != nil { - Logger.Error("Storage initialization failed", "error", err) - } - }() + // A mission loaded: a start dispatched before it belongs to the + // previous one (see startNewMission). + loaded := e.Timestamp + missionLoadedAt.Store(&loaded) + go startStorage() return nil, nil }) @@ -403,7 +449,10 @@ func registerLifecycleHandlers(d *dispatcher.Dispatcher) { return nil, nil }) - d.Register(":MISSION:START:", handleNewMission, dispatcher.Buffered(1), dispatcher.Blocking(), dispatcher.Gated(storageReady)) + // Not blocking: a start waits for a save or discard still ending the + // previous mission, and a queue it filled must not hold up the game's + // thread. A refused start is reported to the addon by callExtension. + d.Register(":MISSION:START:", handleNewMission, dispatcher.Buffered(1), dispatcher.Gated(storageReady)) d.Register(":MISSION:SAVE:", func(e dispatcher.Event) (any, error) { // Optional save-time tag override (addon exportData _tag). Sent as a @@ -416,4 +465,22 @@ func registerLifecycleHandlers(d *dispatcher.Dispatcher) { Logger.Info("Received :MISSION:SAVE: command, queuing async save", "tag", tag) return triggerMissionSave(saveState, storageBackend, apiClient, tag) }) + + // The addon ended a recording without saving it, for example one shorter + // than its minimum mission time. + d.Register(":MISSION:DISCARD:", func(e dispatcher.Event) (any, error) { + Logger.Info("Received :MISSION:DISCARD: command, queuing async discard") + return triggerMissionDiscard(saveState, storageBackend) + }) + + // The recording met the addon's save limits: if the game moves on from + // it now, it is kept (as partial) instead of dropped. Synchronous, so it + // is in place before a mission load that follows it. + d.Register(":MISSION:KEEP:", func(e dispatcher.Event) (any, error) { + Logger.Info("Received :MISSION:KEEP: command; an abandoned recording is kept") + if a, ok := storageBackend.(storage.Abandonable); ok { + a.SetDiscardAbandoned(false) + } + return nil, nil + }) } diff --git a/cmd/ocap_recorder/main_test.go b/cmd/ocap_recorder/main_test.go new file mode 100644 index 0000000..ddb2bc0 --- /dev/null +++ b/cmd/ocap_recorder/main_test.go @@ -0,0 +1,146 @@ +package main + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/OCAP2/extension/v5/internal/dispatcher" + "github.com/OCAP2/extension/v5/internal/logging" + "github.com/OCAP2/extension/v5/internal/parser" + "github.com/OCAP2/extension/v5/pkg/a3interface" + "github.com/OCAP2/extension/v5/pkg/core" +) + +// isolateMissionStart records callbacks and gives a test its own parser and +// no storage. Logger is left alone: the package init still reads it. +func isolateMissionStart(t *testing.T, p parser.Service) *callbackRecorder { + t.Helper() + rec := &callbackRecorder{} + a3interface.SetCallbackSinkForTest(rec.sink) + previousParser, previousBackend := parserService, storageBackend + t.Cleanup(func() { + a3interface.SetCallbackSinkForTest(nil) + parserService, storageBackend = previousParser, previousBackend + }) + parserService = p + storageBackend = nil + return rec +} + +func TestHandleNewMission_FailureIsReported(t *testing.T) { + rec := isolateMissionStart(t, nil) + + _, err := handleNewMission(dispatcher.Event{Args: []string{"{}", "{}", `"tok-1"`}}) + require.Error(t, err) + cb := rec.wait(t, ":MISSION:FAILED:") + require.Len(t, cb.data, 2) + assert.Contains(t, cb.data[0], "not initialized") + assert.Equal(t, `"tok-1"`, cb.data[1], "the addon's registration token is echoed") + + rec.mu.Lock() + rec.calls = nil + rec.mu.Unlock() + parserService = parser.NewParser(Logger) + _, err = handleNewMission(dispatcher.Event{Args: []string{"not json"}}) + require.Error(t, err) + cb = rec.wait(t, ":MISSION:FAILED:") + assert.Equal(t, `""`, cb.data[1], "an addon without a token gets an empty one") +} + +// acceptingParser starts every mission. +type acceptingParser struct{ parser.Service } + +func (acceptingParser) ParseMission([]string) (core.Mission, core.World, error) { + return core.Mission{MissionName: "M"}, core.World{WorldName: "W"}, nil +} + +func TestHandleNewMission_WaitsForSaveOrDiscard(t *testing.T) { + rec := isolateMissionStart(t, acceptingParser{}) + + // The previous mission's discard was dispatched just before this start. + saveState.backendBusy.Store(true) + t.Cleanup(func() { saveState.backendBusy.Store(false) }) + done := make(chan error, 1) + go func() { + _, err := handleNewMission(dispatcher.Event{Args: []string{"{}", "{}", `"tok-3"`}}) + done <- err + }() + select { + case <-done: + t.Fatal("the start ran before the discard finished") + case <-time.After(100 * time.Millisecond): + } + saveState.backendBusy.Store(false) + require.NoError(t, <-done) + rec.wait(t, ":MISSION:OK:") +} + +func TestHandleNewMission_RejectsStartOfUnloadedMission(t *testing.T) { + rec := isolateMissionStart(t, acceptingParser{}) + previous := missionLoadedAt.Load() + t.Cleanup(func() { missionLoadedAt.Store(previous) }) + + // The start was dispatched, then the next mission loaded while it waited. + dispatched := time.Now() + loaded := dispatched.Add(time.Second) + missionLoadedAt.Store(&loaded) + _, err := handleNewMission(dispatcher.Event{Args: []string{"{}", "{}", `"tok-4"`}, Timestamp: dispatched}) + require.Error(t, err) + cb := rec.wait(t, ":MISSION:FAILED:") + assert.Equal(t, `"tok-4"`, cb.data[1]) + + // A start dispatched after the load goes through. + _, err = handleNewMission(dispatcher.Event{Args: []string{"{}", "{}", `"tok-5"`}, Timestamp: dispatched.Add(2 * time.Second)}) + require.NoError(t, err) + rec.wait(t, ":MISSION:OK:") +} + +func TestHandleNewMission_OKEchoesToken(t *testing.T) { + rec := isolateMissionStart(t, acceptingParser{}) + + _, err := handleNewMission(dispatcher.Event{Args: []string{"{}", "{}", `"tok-2"`}}) + require.NoError(t, err) + cb := rec.wait(t, ":MISSION:OK:") + assert.Equal(t, []string{`"OK"`, `"tok-2"`}, cb.data) +} + +func TestHandleNewMission_DiscardAbandonedUntilKept(t *testing.T) { + rec := isolateMissionStart(t, acceptingParser{}) + backend := &abandoningBackend{rec: rec} + storageBackend = backend + + _, err := handleNewMission(dispatcher.Event{Args: []string{"{}", "{}", `"tok-6"`}}) + require.NoError(t, err) + assert.Empty(t, backend.discard, "an addon that does not ask keeps an abandoned recording") + + _, err = handleNewMission(dispatcher.Event{Args: []string{"{}", "{}", `"tok-7"`, `"discard"`}}) + require.NoError(t, err) + assert.Equal(t, []bool{true}, backend.discard) + + // The recording met the addon's save limits. + d, err := dispatcher.New(logging.NewDispatcherLogger(Logger)) + require.NoError(t, err) + registerLifecycleHandlers(d) + _, err = d.Dispatch(dispatcher.Event{Command: ":MISSION:KEEP:"}) + require.NoError(t, err) + assert.Equal(t, []bool{true, false}, backend.discard) +} + +// panickingParser fails every mission start with a panic. +type panickingParser struct{ parser.Service } + +func (panickingParser) ParseMission([]string) (core.Mission, core.World, error) { + panic("boom") +} + +func TestHandleNewMission_PanicIsReported(t *testing.T) { + rec := isolateMissionStart(t, panickingParser{}) + + assert.Panics(t, func() { _, _ = handleNewMission(dispatcher.Event{}) }, "the dispatcher still sees the panic") + cb := rec.wait(t, ":MISSION:FAILED:") + require.Len(t, cb.data, 2) + assert.Contains(t, cb.data[0], "panic") +} diff --git a/cmd/ocap_recorder/mission_save.go b/cmd/ocap_recorder/mission_save.go index 62d7ae8..2e429d8 100644 --- a/cmd/ocap_recorder/mission_save.go +++ b/cmd/ocap_recorder/mission_save.go @@ -2,6 +2,7 @@ package main import ( "context" + "errors" "fmt" "runtime/debug" "sync/atomic" @@ -10,17 +11,31 @@ import ( "github.com/OCAP2/extension/v5/internal/api" "github.com/OCAP2/extension/v5/internal/storage" "github.com/OCAP2/extension/v5/pkg/a3interface" + "github.com/OCAP2/extension/v5/pkg/core" ) // missionSaveState holds the cross-call state for the async save worker. // There is exactly one instance, owned by the package-level saveState variable. type missionSaveState struct { inFlight atomic.Bool + // backendBusy is set while a dispatched save or discard has not yet + // ended the mission in the backend; an upload after it does not count. + backendBusy atomic.Bool } // saveState is the package-level state used by the :MISSION:SAVE: handler. var saveState = &missionSaveState{} +// waitIdle waits until a dispatched save or discard has ended the mission in +// the backend. A mission start runs on its own handler, so without it, a +// discard dispatched just before the next start could reach the backend +// after it, and drop the new mission. +func (s *missionSaveState) waitIdle() { + for s.backendBusy.Load() { + time.Sleep(10 * time.Millisecond) + } +} + // triggerMissionSave is the logic behind the :MISSION:SAVE: dispatcher handler, // extracted so it can be exercised in tests with a fake backend and a fake // api client. @@ -41,17 +56,93 @@ func triggerMissionSave(state *missionSaveState, backend storage.Backend, client if !state.inFlight.CompareAndSwap(false, true) { return nil, fmt.Errorf("save already in progress") } + state.backendBusy.Store(true) go runMissionSaveWorker(state, backend, client, tag) return "queued", nil } +// triggerMissionDiscard is the logic behind the :MISSION:DISCARD: dispatcher +// handler: the addon ended a recording that does not meet its save limits. +// It shares the in-flight guard with saves, as both end the mission. +// +// Contract: +// - Returns "queued" + nil error when a goroutine was spawned; it reports +// completion via a ":MISSION:DISCARDED:" callback: ["ok"], ["kept"] for +// a backend that keeps what it recorded, or ["error", message]. +// - Returns a non-nil error if a save or discard is already in progress. +func triggerMissionDiscard(state *missionSaveState, backend storage.Backend) (any, error) { + if backend == nil { + return nil, fmt.Errorf("no storage backend configured") + } + if !state.inFlight.CompareAndSwap(false, true) { + return nil, fmt.Errorf("mission save or discard already in progress") + } + state.backendBusy.Store(true) + + go runMissionDiscardWorker(state, backend) + return "queued", nil +} + +// runMissionDiscardWorker performs DiscardMission, recovers from panics, +// flushes OTel, and dispatches the :MISSION:DISCARDED: callback. +func runMissionDiscardWorker(state *missionSaveState, backend storage.Backend) { + err := storage.ErrMissionKept + if d, ok := backend.(storage.Discardable); ok { + err = func() (err error) { + defer func() { + if r := recover(); r != nil { + err = fmt.Errorf("panic during mission discard: %v", r) + if Logger != nil { + Logger.Error("panic during mission discard", "panic", r, "stack", string(debug.Stack())) + } + } + }() + defer state.backendBusy.Store(false) + return d.DiscardMission() + }() + } + state.backendBusy.Store(false) // a backend that keeps the mission did nothing + + flushOTel() + // Cleared before the callback: the addon may start the next mission's + // save or discard as soon as it hears this one ended. + state.inFlight.Store(false) + + switch { + case errors.Is(err, storage.ErrMissionKept): + if Logger != nil { + Logger.Info("Storage keeps the discarded mission", "reason", err) + } + _ = a3interface.WriteArmaCallback(ExtensionName, ":MISSION:DISCARDED:", "kept") + case err != nil: + if Logger != nil { + Logger.Error("Failed to discard mission in storage backend", "error", err) + } + _ = a3interface.WriteArmaCallback(ExtensionName, ":MISSION:DISCARDED:", "error", err.Error()) + default: + if Logger != nil { + Logger.Info("Mission recording discarded") + } + _ = a3interface.WriteArmaCallback(ExtensionName, ":MISSION:DISCARDED:", "ok") + } +} + +// flushOTel flushes OTel data at mission end if the provider is available. +func flushOTel() { + if OTelProvider == nil { + return + } + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if err := OTelProvider.Flush(ctx); err != nil && Logger != nil { + Logger.Warn("Failed to flush OTel data", "error", err) + } +} + // runMissionSaveWorker performs EndMission + optional upload, recovers from // panics, flushes OTel, and dispatches the :MISSION:SAVED: callback. func runMissionSaveWorker(state *missionSaveState, backend storage.Backend, client *api.Client, tag string) { - // Always clear the in-flight flag before returning. - defer state.inFlight.Store(false) - start := time.Now() var ( @@ -74,7 +165,22 @@ func runMissionSaveWorker(state *missionSaveState, backend storage.Backend, clie } }() - if err := backend.EndMission(tag); err != nil { + // The next mission may start once the backend ended this one; the + // upload below does not hold it up. The export is read first: the + // next start resets it. + u, uploadable := backend.(storage.Uploadable) + var exportPath string + var exportMeta core.UploadMetadata + if err := func() error { + defer state.backendBusy.Store(false) + if err := backend.EndMission(tag); err != nil { + return err + } + if uploadable { + exportPath, exportMeta = u.GetExportedFilePath(), u.GetExportMetadata() + } + return nil + }(); err != nil { endErr = err if Logger != nil { Logger.Error("Failed to end mission in storage backend", "error", err) @@ -87,11 +193,10 @@ func runMissionSaveWorker(state *missionSaveState, backend storage.Backend, clie ) } - if u, ok := backend.(storage.Uploadable); ok && client != nil { - if path := u.GetExportedFilePath(); path != "" { + if uploadable && client != nil { + if path := exportPath; path != "" { savedPath = path - meta := u.GetExportMetadata() - if err := client.Upload(path, meta); err != nil { + if err := client.Upload(path, exportMeta); err != nil { uploadErr = err if Logger != nil { Logger.Error("Failed to upload to OCAP web", @@ -106,20 +211,16 @@ func runMissionSaveWorker(state *missionSaveState, backend storage.Backend, clie ) } } - } else if u, ok := backend.(storage.Uploadable); ok { + } else if uploadable { // Client not configured — still report the path so the addon can inform players. - savedPath = u.GetExportedFilePath() + savedPath = exportPath } }() - // Flush OTel data if provider is available. - if OTelProvider != nil { - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - if err := OTelProvider.Flush(ctx); err != nil && Logger != nil { - Logger.Warn("Failed to flush OTel data", "error", err) - } - } + flushOTel() + // Cleared before the callback: the addon may start the next mission's + // save or discard as soon as it hears this one ended. + state.inFlight.Store(false) // Build callback payload. switch { diff --git a/cmd/ocap_recorder/mission_save_test.go b/cmd/ocap_recorder/mission_save_test.go index 742d64e..7bd2cd1 100644 --- a/cmd/ocap_recorder/mission_save_test.go +++ b/cmd/ocap_recorder/mission_save_test.go @@ -240,3 +240,124 @@ type panickingBackend struct{ fakeBackend } func (p *panickingBackend) EndMission(tag string) error { panic(fmt.Errorf("boom")) } + +// discardingBackend is a fakeBackend that also implements storage.Discardable. +type discardingBackend struct { + fakeBackend + discardErr error + discardPanic bool + discards int +} + +func (d *discardingBackend) DiscardMission() error { + d.discards++ + if d.discardPanic { + panic("boom") + } + return d.discardErr +} + +var _ storage.Discardable = (*discardingBackend)(nil) + +func TestRunMissionDiscard_Success(t *testing.T) { + rec := &callbackRecorder{} + a3interface.SetCallbackSinkForTest(rec.sink) + t.Cleanup(func() { a3interface.SetCallbackSinkForTest(nil) }) + + state := newTestSaveState() + backend := &discardingBackend{} + + result, err := triggerMissionDiscard(state, backend) + require.NoError(t, err) + assert.Equal(t, "queued", result) + + cb := rec.wait(t, ":MISSION:DISCARDED:") + assert.Equal(t, []string{`"ok"`}, cb.data) + assert.Equal(t, 1, backend.discards) + assert.Empty(t, backend.endTag, "a discarded mission is not saved") + assert.False(t, state.inFlight.Load()) +} + +func TestRunMissionDiscard_BackendKeepsData(t *testing.T) { + rec := &callbackRecorder{} + a3interface.SetCallbackSinkForTest(rec.sink) + t.Cleanup(func() { a3interface.SetCallbackSinkForTest(nil) }) + + state := newTestSaveState() + _, err := triggerMissionDiscard(state, &fakeBackend{}) + require.NoError(t, err) + + cb := rec.wait(t, ":MISSION:DISCARDED:") + assert.Equal(t, []string{`"kept"`}, cb.data) + assert.False(t, state.inFlight.Load()) +} + +func TestRunMissionDiscard_StorageKeepsMission(t *testing.T) { + rec := &callbackRecorder{} + a3interface.SetCallbackSinkForTest(rec.sink) + t.Cleanup(func() { a3interface.SetCallbackSinkForTest(nil) }) + + // For example a websocket server too old to drop it. + state := newTestSaveState() + backend := &discardingBackend{discardErr: fmt.Errorf("old server: %w", storage.ErrMissionKept)} + _, err := triggerMissionDiscard(state, backend) + require.NoError(t, err) + + cb := rec.wait(t, ":MISSION:DISCARDED:") + assert.Equal(t, []string{`"kept"`}, cb.data) + assert.Equal(t, 1, backend.discards) +} + +func TestRunMissionDiscard_Error(t *testing.T) { + rec := &callbackRecorder{} + a3interface.SetCallbackSinkForTest(rec.sink) + t.Cleanup(func() { a3interface.SetCallbackSinkForTest(nil) }) + + state := newTestSaveState() + _, err := triggerMissionDiscard(state, &discardingBackend{discardErr: errors.New("server gone")}) + require.NoError(t, err) + + cb := rec.wait(t, ":MISSION:DISCARDED:") + require.Len(t, cb.data, 2) + assert.Equal(t, `"error"`, cb.data[0]) + assert.Contains(t, cb.data[1], "server gone") +} + +func TestRunMissionDiscard_PanicIsRecovered(t *testing.T) { + rec := &callbackRecorder{} + a3interface.SetCallbackSinkForTest(rec.sink) + t.Cleanup(func() { a3interface.SetCallbackSinkForTest(nil) }) + + state := newTestSaveState() + _, err := triggerMissionDiscard(state, &discardingBackend{discardPanic: true}) + require.NoError(t, err) + + cb := rec.wait(t, ":MISSION:DISCARDED:") + require.Len(t, cb.data, 2) + assert.Equal(t, `"error"`, cb.data[0]) + assert.Contains(t, cb.data[1], "panic") + assert.False(t, state.inFlight.Load(), "inFlight must be cleared even after panic") +} + +func TestRunMissionDiscard_RejectedWhileSaving(t *testing.T) { + rec := &callbackRecorder{} + a3interface.SetCallbackSinkForTest(rec.sink) + t.Cleanup(func() { a3interface.SetCallbackSinkForTest(nil) }) + + state := newTestSaveState() + backend := &discardingBackend{fakeBackend: fakeBackend{endDuration: 200 * time.Millisecond, path: "r.json.gz"}} + + _, err := triggerMissionSave(state, backend, nil, "") + require.NoError(t, err) + _, err = triggerMissionDiscard(state, backend) + require.Error(t, err, "a mission is either saved or discarded") + assert.Contains(t, err.Error(), "already in progress") + + rec.wait(t, ":MISSION:SAVED:") + assert.Zero(t, backend.discards) +} + +func TestTriggerMissionDiscard_NoBackend(t *testing.T) { + _, err := triggerMissionDiscard(newTestSaveState(), nil) + require.Error(t, err) +} diff --git a/cmd/ocap_recorder/storage.go b/cmd/ocap_recorder/storage.go index 8a66614..264b2eb 100644 --- a/cmd/ocap_recorder/storage.go +++ b/cmd/ocap_recorder/storage.go @@ -3,6 +3,7 @@ package main import ( "fmt" "strings" + "sync" "github.com/OCAP2/extension/v5/internal/config" "github.com/OCAP2/extension/v5/internal/storage" @@ -12,21 +13,59 @@ import ( "github.com/OCAP2/extension/v5/pkg/a3interface" ) +// startStorage handles :STORAGE:INIT:. It answers with a :STORAGE:OK: +// callback, or :STORAGE:ERROR: so the addon stops waiting for storage. +func startStorage() { + if err := initStorage(); err != nil { + Logger.Error("Storage initialization failed", "error", err) + if cbErr := a3interface.WriteArmaCallback(ExtensionName, ":STORAGE:ERROR:", err.Error()); cbErr != nil { + Logger.Warn("Failed to send STORAGE:ERROR callback", "error", cbErr) + } + } +} + +// storageInitMu serializes :STORAGE:INIT:, which the addon sends on every +// mission load. +var storageInitMu sync.Mutex + func initStorage() error { Logger.Debug("Received :STORAGE:INIT: call") + storageInitMu.Lock() + defer storageInitMu.Unlock() storageCfg := config.GetStorageConfig() + // The backend lives as long as the process: the next mission reuses it, + // so a mission the game moved on from is ended here instead of staying + // open in a backend nothing uses any more. + if storageBackend != nil { + Logger.Info("Storage already initialized; reusing it", "type", storageCfg.Type) + // A new mission loaded: one still recording was abandoned. It ends + // before :STORAGE:OK:, so the new mission's start cannot come first + // and be taken for it. This runs on startStorage's goroutine. A save + // or discard dispatched before is ending it already; waiting for its + // server would only hold up the next mission. + if a, ok := storageBackend.(storage.Abandonable); ok && !saveState.backendBusy.Load() { + if err := a.AbandonMission(); err != nil { + Logger.Warn("Failed to end the abandoned mission", "error", err) + } + } + if err := a3interface.WriteArmaCallback(ExtensionName, ":STORAGE:OK:", storageCfg.Type); err != nil { + Logger.Warn("Failed to send STORAGE:OK callback", "error", err) + } + return nil + } + backend, err := createStorageBackend(storageCfg) if err != nil { Logger.Error("Failed to create storage backend", "error", err) return err } - storageBackend = backend - if err := storageBackend.Init(); err != nil { + if err := backend.Init(); err != nil { Logger.Error("Failed to initialize storage backend", "error", err) return err } + storageBackend = backend // Initialize worker manager workerManager = worker.NewManager(worker.Dependencies{ diff --git a/cmd/ocap_recorder/storage_test.go b/cmd/ocap_recorder/storage_test.go index b7dd3cc..45d2b69 100644 --- a/cmd/ocap_recorder/storage_test.go +++ b/cmd/ocap_recorder/storage_test.go @@ -5,9 +5,14 @@ import ( "testing" "github.com/OCAP2/extension/v5/internal/config" + "github.com/OCAP2/extension/v5/internal/dispatcher" + "github.com/OCAP2/extension/v5/internal/logging" "github.com/OCAP2/extension/v5/internal/storage" "github.com/OCAP2/extension/v5/internal/storage/websocket" + "github.com/OCAP2/extension/v5/pkg/a3interface" + "github.com/OCAP2/extension/v5/pkg/core" "github.com/spf13/viper" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -41,6 +46,113 @@ func TestWebSocketBackendRequiresSecret(t *testing.T) { require.Error(t, err) } +// isolateStorage gives a test its own storage globals. +func isolateStorage(t *testing.T) *callbackRecorder { + t.Helper() + rec := &callbackRecorder{} + a3interface.SetCallbackSinkForTest(rec.sink) + // Logger is left alone: the package init still reads it. + previousBackend, previousDispatcher, previousWorkers := storageBackend, eventDispatcher, workerManager + t.Cleanup(func() { + a3interface.SetCallbackSinkForTest(nil) + storageBackend, eventDispatcher, workerManager = previousBackend, previousDispatcher, previousWorkers + viper.Reset() + }) + storageBackend = nil + d, err := dispatcher.New(logging.NewDispatcherLogger(Logger)) + require.NoError(t, err) + eventDispatcher = d + return rec +} + +func TestInitStorageReusesBackend(t *testing.T) { + rec := isolateStorage(t) + viper.Set("storage.type", "memory") + viper.Set("storage.memory.outputDir", t.TempDir()) + + require.NoError(t, initStorage()) + first := storageBackend + require.NotNil(t, first) + // The addon initializes storage again on the next mission load. + require.NoError(t, initStorage()) + require.Same(t, first, storageBackend, "the next mission reuses the backend that ends the one left open") + + rec.mu.Lock() + defer rec.mu.Unlock() + oks := 0 + for _, c := range rec.calls { + if c.function == ":STORAGE:OK:" { + oks++ + } + } + require.Equal(t, 2, oks, "every :STORAGE:INIT: is answered") +} + +// abandoningBackend is a fakeBackend that records AbandonMission calls and +// whether :STORAGE:OK: had gone out by then, and SetDiscardAbandoned calls. +type abandoningBackend struct { + fakeBackend + rec *callbackRecorder + abandoned int + okBefore bool + discard []bool +} + +func (a *abandoningBackend) StartMission(*core.Mission, *core.World) error { return nil } + +func (a *abandoningBackend) SetDiscardAbandoned(discard bool) { + a.discard = append(a.discard, discard) +} + +func (a *abandoningBackend) AbandonMission() error { + a.abandoned++ + a.rec.mu.Lock() + defer a.rec.mu.Unlock() + for _, c := range a.rec.calls { + if c.function == ":STORAGE:OK:" { + a.okBefore = true + } + } + return nil +} + +func TestInitStorageAbandonsOpenMission(t *testing.T) { + rec := isolateStorage(t) + viper.Set("storage.type", "websocket") + backend := &abandoningBackend{rec: rec} + storageBackend = backend + + // The next mission loads. + require.NoError(t, initStorage()) + assert.Equal(t, 1, backend.abandoned, "the mission left open is abandoned") + assert.False(t, backend.okBefore, "before :STORAGE:OK:, so the next start cannot come first") + rec.wait(t, ":STORAGE:OK:") + require.Same(t, storage.Backend(backend), storageBackend) +} + +func TestInitStorageLeavesMissionToDispatchedSave(t *testing.T) { + rec := isolateStorage(t) + viper.Set("storage.type", "websocket") + backend := &abandoningBackend{rec: rec} + storageBackend = backend + saveState.backendBusy.Store(true) // a save dispatched before is ending it + t.Cleanup(func() { saveState.backendBusy.Store(false) }) + + require.NoError(t, initStorage()) + assert.Zero(t, backend.abandoned, "the next mission is not held up waiting for that save") + rec.wait(t, ":STORAGE:OK:") +} + +func TestStartStorageReportsFailure(t *testing.T) { + rec := isolateStorage(t) + viper.Set("storage.type", "not-real") + + startStorage() + cb := rec.wait(t, ":STORAGE:ERROR:") + require.Len(t, cb.data, 1) + require.Nil(t, storageBackend) +} + func TestWebSocketURLPreservesPrefix(t *testing.T) { require.Equal(t, "wss://example.test/ocap", httpToWS("https://example.test/ocap/")) } diff --git a/internal/storage/memory/memory.go b/internal/storage/memory/memory.go index 545b003..2f8c748 100644 --- a/internal/storage/memory/memory.go +++ b/internal/storage/memory/memory.go @@ -175,6 +175,22 @@ func (b *Backend) EndMission(tag string) (retErr error) { return nil } +// DiscardMission drops the mission data without exporting it, for a +// recording the addon decided not to save. +func (b *Backend) DiscardMission() error { + b.mu.Lock() + defer b.mu.Unlock() + if b.mission == nil { + return nil // nothing recording; keep the previous export path + } + + b.mission = nil + b.world = nil + b.lastExportPath = "" + b.resetCollections() + return nil +} + // resetCollections clears all entity and event data. // Caller must hold b.mu.Lock. func (b *Backend) resetCollections() { diff --git a/internal/storage/memory/memory_test.go b/internal/storage/memory/memory_test.go index d5072c7..a1ee7e8 100644 --- a/internal/storage/memory/memory_test.go +++ b/internal/storage/memory/memory_test.go @@ -2,6 +2,7 @@ package memory import ( + "os" "strings" "sync" "testing" @@ -20,6 +21,9 @@ var _ storage.Backend = (*Backend)(nil) // Verify Backend implements storage.Uploadable interface var _ storage.Uploadable = (*Backend)(nil) +// Verify Backend implements storage.Discardable interface +var _ storage.Discardable = (*Backend)(nil) + func TestNew(t *testing.T) { cfg := config.MemoryConfig{ OutputDir: "/tmp/test", @@ -896,6 +900,41 @@ func TestEndMissionWithoutStartMission(t *testing.T) { assert.Contains(t, err.Error(), "no mission to end") } +func TestDiscardMissionDropsDataWithoutExport(t *testing.T) { + tmpDir := t.TempDir() + b := New(config.MemoryConfig{OutputDir: tmpDir}, nil) + + require.NoError(t, b.StartMission(&core.Mission{MissionName: "Short", CaptureDelay: 1}, &core.World{WorldName: "Altis"})) + require.NoError(t, b.AddSoldier(&core.Soldier{ID: 1, UnitName: "Alpha 1-1"})) + require.NoError(t, b.RecordGeneralEvent(&core.GeneralEvent{Name: "test"})) + + require.NoError(t, b.DiscardMission()) + + assert.Nil(t, b.mission) + assert.Empty(t, b.soldiers) + assert.Empty(t, b.generalEvents) + assert.Empty(t, b.GetExportedFilePath()) + entries, err := os.ReadDir(tmpDir) + require.NoError(t, err) + assert.Empty(t, entries, "a discarded mission is not exported") + err = b.EndMission("") + require.Error(t, err, "nothing is left to save") + assert.Contains(t, err.Error(), "no mission to end") + + require.NoError(t, b.DiscardMission(), "discarding again is a no-op") +} + +func TestDiscardMissionAfterSaveKeepsExportPath(t *testing.T) { + b := New(config.MemoryConfig{OutputDir: t.TempDir()}, nil) + require.NoError(t, b.StartMission(&core.Mission{MissionName: "Saved", CaptureDelay: 1}, &core.World{WorldName: "Altis"})) + require.NoError(t, b.EndMission("")) + path := b.GetExportedFilePath() + require.NotEmpty(t, path) + + require.NoError(t, b.DiscardMission(), "nothing is recording") + assert.Equal(t, path, b.GetExportedFilePath()) +} + func TestGetExportMetadataWithoutStartMission(t *testing.T) { b := New(config.MemoryConfig{}, nil) diff --git a/internal/storage/storage.go b/internal/storage/storage.go index 1a3982f..84fe968 100644 --- a/internal/storage/storage.go +++ b/internal/storage/storage.go @@ -1,7 +1,11 @@ // internal/storage/storage.go package storage -import "github.com/OCAP2/extension/v5/pkg/core" +import ( + "errors" + + "github.com/OCAP2/extension/v5/pkg/core" +) // Backend is the interface all storage implementations must satisfy type Backend interface { @@ -53,6 +57,37 @@ type Backend interface { SetFocusEnd(frame core.Frame) error } +// Discardable is an optional interface for storage backends that can drop +// the current mission instead of saving it (:MISSION:DISCARD:). Backends +// without it keep what they recorded. +type Discardable interface { + // DiscardMission ends the current mission without saving it. It is a + // no-op when no mission is being recorded. It returns ErrMissionKept + // when the mission ended but its data could not be dropped. + DiscardMission() error +} + +// Abandonable is an optional interface for storage backends that keep a +// mission open outside the process, such as the websocket stream. The next +// mission load ends a mission the game moved on from without saving or +// discarding it. +type Abandonable interface { + // AbandonMission ends the current mission, as interrupted, or drops it + // when it recorded nothing or SetDiscardAbandoned asked for it. It is a + // no-op when no mission is being recorded. + AbandonMission() error + // SetDiscardAbandoned sets whether AbandonMission drops the current + // mission instead of ending it, for a recording the addon would not save + // yet (:MISSION:START: with "discard", until :MISSION:KEEP:). Each + // mission starts without it. It is a no-op when no mission is being + // recorded, and never waits for the backend. + SetDiscardAbandoned(discard bool) +} + +// ErrMissionKept means the storage keeps what was recorded of a discarded +// mission, as a backend without Discardable does. +var ErrMissionKept = errors.New("storage keeps the discarded mission") + // Uploadable is an optional interface for storage backends that produce // files suitable for upload to the OCAP web frontend. type Uploadable interface { diff --git a/internal/storage/websocket/connection.go b/internal/storage/websocket/connection.go index 90e8f99..91aeb04 100644 --- a/internal/storage/websocket/connection.go +++ b/internal/storage/websocket/connection.go @@ -25,13 +25,16 @@ const ( defaultReconnectMax = 30 * time.Second ackChSize = 16 writeWait = 10 * time.Second - ackTimeout = 10 * time.Second ) // handshakeTimeout bounds waiting for the server's answer to resume; tests // lower it. var handshakeTimeout = 10 * time.Second +// ackTimeout bounds waiting for the start_mission acknowledgement; tests +// lower it. +var ackTimeout = 10 * time.Second + var errConnectionClosed = errors.New("websocket connection closed") // connOptions tunes buffering and recovery. Zero values select the defaults. @@ -97,6 +100,10 @@ type connection struct { resumeData []byte // resume envelope for this mission resumable bool // the server acknowledged start with resume support + // protocol is the version the server serving the mission last announced: + // on the start acknowledgement, a resume reply, or a restarted start. + protocol atomic.Uint32 + acked atomic.Uint64 // highest seq the server acknowledged sent atomic.Uint64 // highest seq written on the current socket resumes atomic.Uint64 // replacement connections that resumed the mission @@ -235,14 +242,18 @@ func (c *connection) handshake(conn *ws.Conn) (uint64, uint64, error) { case streaming.ServerResumeOK: c.logger.Info("WebSocket mission resumed", "serverSeq", reply.Seq, "localSeq", c.log.lastSeq()) c.resumes.Add(1) + c.setProtocol(reply.ProtocolVersion) // the server may have changed meanwhile c.ackUpTo(reply.Seq) return reply.Seq, epoch, nil case streaming.ServerResumeCompleted: - // The server saved the mission but its end_mission ack was lost. - c.logger.Info("WebSocket mission already saved by server") + // The server saved or discarded the mission but its end_mission or + // discard_mission ack was lost. It stands in for whichever one is + // awaited; waitAck skips the other. + c.logger.Info("WebSocket mission already finished by server") c.ackUpTo(c.log.lastSeq()) c.deliverAck(streaming.AckMessage{Type: streaming.ServerAck, For: streaming.TypeEndMission}) + c.deliverAck(streaming.AckMessage{Type: streaming.ServerAck, For: streaming.TypeDiscardMission}) return c.log.lastSeq(), epoch, nil default: // resume_unknown @@ -260,11 +271,13 @@ func (c *connection) handshake(conn *ws.Conn) (uint64, uint64, error) { if err := writeData(conn, start); err != nil { return 0, 0, err } - if _, err := readServerMessage(conn, func(m streaming.AckMessage) bool { + ack, err := readServerMessage(conn, func(m streaming.AckMessage) bool { return m.Type == streaming.ServerAck && m.For == streaming.TypeStartMission - }); err != nil { + }) + if err != nil { return 0, 0, fmt.Errorf("restart mission: %w", err) } + c.setProtocol(ack.ProtocolVersion) // Acknowledgements now count from the new session's start. c.acked.Store(1) return 1, epoch, nil @@ -436,6 +449,20 @@ func (c *connection) ackUpTo(seq uint64) { c.progressAt.Store(time.Now().UnixNano()) // progress restarts the stall clock } +// ackRefused stands in for the acknowledgement of a discard_mission that a +// server too old to take it will never send. It never comes from a server. +const ackRefused = "refused" + +// setProtocol records the version the server serving the mission announced +// on a reconnect. A server replaced by one older than ProtocolVersionDiscard +// refuses a discard_mission still waiting for its acknowledgement. +func (c *connection) setProtocol(v uint8) { + old := c.protocol.Swap(uint32(v)) + if old >= streaming.ProtocolVersionDiscard && v < streaming.ProtocolVersionDiscard { + c.deliverAck(streaming.AckMessage{Type: ackRefused, For: streaming.TypeDiscardMission}) + } +} + func (c *connection) deliverAck(m streaming.AckMessage) { select { case c.ackCh <- m: @@ -613,9 +640,13 @@ func (c *connection) waitAck(ackFor string, timeout time.Duration) (streaming.Ac for { select { case ack := <-c.ackCh: - if ack.For == ackFor { - return ack, nil + if ack.For != ackFor { + continue + } + if ack.Type == ackRefused { + return streaming.AckMessage{}, errDiscardUnsupported } + return ack, nil case <-timer.C: return streaming.AckMessage{}, fmt.Errorf("timeout waiting for ack of %q", ackFor) case <-c.done: diff --git a/internal/storage/websocket/discard_test.go b/internal/storage/websocket/discard_test.go new file mode 100644 index 0000000..8230699 --- /dev/null +++ b/internal/storage/websocket/discard_test.go @@ -0,0 +1,665 @@ +package websocket + +import ( + "bufio" + "encoding/json" + "errors" + "log/slog" + "os" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/OCAP2/extension/v5/internal/storage" + "github.com/OCAP2/extension/v5/pkg/core" + "github.com/OCAP2/extension/v5/pkg/streaming" +) + +var _ storage.Discardable = (*Backend)(nil) + +// requireGone waits until path no longer exists. +func requireGone(t *testing.T, path string) { + t.Helper() + require.Eventually(t, func() bool { _, err := os.Stat(path); return os.IsNotExist(err) }, 5*time.Second, 10*time.Millisecond) +} + +// requireNoDiscard checks that no discard_mission reached the server. +func requireNoDiscard(t *testing.T, msgs []streaming.Envelope) { + t.Helper() + for _, env := range msgs { + require.NotEqual(t, streaming.TypeDiscardMission, env.Type) + } +} + +func TestDiscardMissionDropsRecording(t *testing.T) { + rs := newResumeServer(t) + spoolDir := t.TempDir() + b := New(recoveryConfig(rs.url(), spoolDir)) + require.NoError(t, b.Init()) + defer b.Close() + + id := startResumable(t, b) + recordStates(t, b, 1, 5) + require.NoError(t, b.DiscardMission()) + + m := rs.mission(id) + require.True(t, m.discarded) + requireContiguous(t, m, 7) // start, 5 states, discard + assert.Equal(t, streaming.TypeDiscardMission, m.received[6].Type) + var p streaming.DiscardMissionPayload + require.NoError(t, json.Unmarshal(m.received[6].Payload, &p)) + assert.Equal(t, streaming.DiscardMissionPayload{MissionID: id, ProducerID: b.producerID}, p, "the discard names the mission") + requireGone(t, filepath.Join(spoolDir, id)) // a confirmed discard leaves nothing to recover + + // The next mission records as usual. + next := startResumable(t, b) + assert.NotEqual(t, id, next) + recordStates(t, b, 1, 2) + require.NoError(t, b.EndMission("")) + assert.True(t, rs.mission(next).completed) + assert.False(t, rs.mission(next).discarded) +} + +func TestDiscardMissionWithoutMissionIsNoop(t *testing.T) { + rs := newResumeServer(t) + b := New(fastConfig(rs.url())) + require.NoError(t, b.Init()) + defer b.Close() + + require.NoError(t, b.DiscardMission(), "nothing recorded yet") + id := startResumable(t, b) + require.NoError(t, b.EndMission("")) + require.NoError(t, b.DiscardMission(), "the mission was already saved") + m := rs.mission(id) + assert.True(t, m.completed) + assert.False(t, m.discarded) + requireNoDiscard(t, m.received) +} + +func TestDiscardAckLostResolvedByResume(t *testing.T) { + rs := newResumeServer(t) + b := New(fastConfig(rs.url())) + require.NoError(t, b.Init()) + defer b.Close() + + id := startResumable(t, b) + recordStates(t, b, 1, 3) + rs.dropEnd.Store(true) // the discard is applied but its ack never arrives + + done := make(chan error, 1) + go func() { done <- b.DiscardMission() }() + select { + case err := <-done: + require.NoError(t, err, "resume_completed stands in for the lost discard_mission ack") + case <-time.After(5 * time.Second): + t.Fatal("DiscardMission did not complete") + } + assert.True(t, rs.mission(id).discarded) + assert.Equal(t, int32(1), rs.starts.Load(), "the discarded mission is not started again") +} + +func TestDiscardMissionOnServerWithoutDiscard(t *testing.T) { + rs := newResumeServer(t) + rs.noDrop.Store(true) + spoolDir := t.TempDir() + b := New(recoveryConfig(rs.url(), spoolDir)) + require.NoError(t, b.Init()) + defer b.Close() + + id := startResumable(t, b) + recordStates(t, b, 1, 3) + require.Eventually(t, func() bool { return rs.lastSeq(id) == 4 }, 2*time.Second, 5*time.Millisecond) + + require.ErrorIs(t, b.DiscardMission(), storage.ErrMissionKept) + m := rs.mission(id) + assert.False(t, m.completed, "the server keeps the mission until its grace expires") + requireNoDiscard(t, m.received) + requireGone(t, filepath.Join(spoolDir, id)) // recovery must not resend a discarded mission + require.Eventually(t, func() bool { + rs.mu.Lock() + defer rs.mu.Unlock() + return len(rs.conns) == 0 + }, 2*time.Second, 10*time.Millisecond, "the connection is closed so the server stops waiting") + + // The next mission dials a fresh connection. + next := startResumable(t, b) + require.NoError(t, b.EndMission("")) + assert.True(t, rs.mission(next).completed) +} + +func TestDiscardMissionOnLegacyServer(t *testing.T) { + srv, ml := testServer(t) + defer srv.Close() + b := New(Config{URL: wsURL(srv), Secret: "s"}) + require.NoError(t, b.Init()) + defer b.Close() + + require.NoError(t, b.StartMission(&core.Mission{MissionName: "M"}, &core.World{WorldName: "W"})) + require.ErrorIs(t, b.DiscardMission(), errDiscardUnsupported) + requireNoDiscard(t, ml.all()) +} + +// discardUnconfirmed discards the mission while discard_mission is lost in +// transit, so the discard times out and its spool stays behind. +func discardUnconfirmed(t *testing.T, rs *resumeServer, spoolDir string) (*Backend, string) { + t.Helper() + cfg := recoveryConfig(rs.url(), spoolDir) + cfg.SaveTimeout = 200 * time.Millisecond // below the ack stall, so no reconnect resends it + cfg.RecoveryInterval = time.Hour // recovery runs only when the test triggers it + b := New(cfg) + require.NoError(t, b.Init()) + t.Cleanup(func() { _ = b.Close() }) + + id := startResumable(t, b) + recordStates(t, b, 1, 3) + require.Eventually(t, func() bool { return rs.lastSeq(id) == 4 }, 2*time.Second, 5*time.Millisecond) + rs.discard.Store(true) + require.Error(t, b.DiscardMission()) + rs.discard.Store(false) + rs.kill() + require.DirExists(t, filepath.Join(spoolDir, id), "the spool is kept to finish the discard") + return b, id +} + +func TestRecoveryFinishesUnconfirmedDiscard(t *testing.T) { + fastRecovery(t) + rs := newResumeServer(t) + spoolDir := t.TempDir() + b, id := discardUnconfirmed(t, rs, spoolDir) + + b.recoverSpools(b.done) + + m := rs.mission(id) + require.True(t, m.discarded) + requireContiguous(t, m, 5) // nothing is resent: start, 3 states, discard + assert.Zero(t, m.dups) + assert.NoDirExists(t, filepath.Join(spoolDir, id)) +} + +func TestRecoveryDropsPartialOfUnconfirmedDiscard(t *testing.T) { + fastRecovery(t) + rs := newResumeServer(t) + spoolDir := t.TempDir() + b, id := discardUnconfirmed(t, rs, spoolDir) + rs.expire(id) // the producer stayed away past the grace: saved as partial + require.True(t, rs.partial(id)) + + b.recoverSpools(b.done) + + assert.False(t, rs.partial(id), "the partial recording is dropped") + assert.Equal(t, []string{id}, rs.droppedMissions()) + assert.Empty(t, rs.mission(id).received, "the mission is not resent to be discarded") + assert.Equal(t, int32(1), rs.starts.Load()) + assert.NoDirExists(t, filepath.Join(spoolDir, id)) +} + +func TestRecoveryOfDiscardTheServerLost(t *testing.T) { + fastRecovery(t) + rs := newResumeServer(t) + spoolDir := t.TempDir() + b, id := discardUnconfirmed(t, rs, spoolDir) + rs.forget() // the web server restarted + + b.recoverSpools(b.done) + + assert.Equal(t, []string{id}, rs.droppedMissions(), "the discard names the mission") + assert.Equal(t, int32(1), rs.starts.Load(), "the mission is not resent to be discarded") + assert.NoDirExists(t, filepath.Join(spoolDir, id)) +} + +func TestRecoveryOfDiscardOnServerWithoutDiscard(t *testing.T) { + fastRecovery(t) + rs := newResumeServer(t) + rs.noDrop.Store(true) // the web server was downgraded since + spoolDir := t.TempDir() + writeDiscardedSpool(t, filepath.Join(spoolDir, "m-old"), "m-old", "p-old", 3) + + b := New(recoveryConfig(rs.url(), spoolDir)) + require.NoError(t, b.Init()) + defer b.Close() + + requireGone(t, filepath.Join(spoolDir, "m-old")) + assert.Empty(t, rs.droppedMissions(), "an older server is not sent what it cannot take") + assert.Zero(t, rs.starts.Load()) +} + +// writeDiscardedSpool writes a closed spool holding a start_mission, n +// soldier states and a discard_mission. +func writeDiscardedSpool(t *testing.T, dir, missionID, producerID string, n int) { + t.Helper() + sp, err := openSpool(dir, 0, slog.Default()) + require.NoError(t, err) + start, err := marshalEnvelope(streaming.TypeStartMission, streaming.StartMissionPayload{ + Mission: &core.Mission{MissionName: "Discarded", CaptureDelay: 1}, World: &core.World{WorldName: "altis"}, + MissionID: missionID, ProducerID: producerID, ProtocolVersion: streaming.ProtocolVersionDiscard, + }, 1) + require.NoError(t, err) + require.NoError(t, sp.append(1, start)) + for i := 1; i <= n; i++ { + data, err := marshalEnvelope(streaming.TypeSoldierState, &core.SoldierState{SoldierID: 1, CaptureFrame: core.Frame(i)}, uint64(i+1)) + require.NoError(t, err) + require.NoError(t, sp.append(uint64(i+1), data)) + } + discard, err := marshalEnvelope(streaming.TypeDiscardMission, streaming.DiscardMissionPayload{MissionID: missionID, ProducerID: producerID}, uint64(n+2)) + require.NoError(t, err) + require.NoError(t, sp.append(uint64(n+2), discard)) + require.NoError(t, sp.close()) +} + +func TestRecoveryKeepsDiscardWhenResumeTimesOut(t *testing.T) { + fastRecovery(t) + oldHandshake := handshakeTimeout + handshakeTimeout = 100 * time.Millisecond + t.Cleanup(func() { handshakeTimeout = oldHandshake }) + + rs := newResumeServer(t) + rs.legacy.Store(true) // does not answer resume, like a slow server + spoolDir := t.TempDir() + writeDiscardedSpool(t, filepath.Join(spoolDir, "m-kept"), "m-kept", "p-old", 3) + + b := New(recoveryConfig(rs.url(), spoolDir)) + require.NoError(t, b.Init()) + defer b.Close() + + require.Eventually(t, func() bool { return b.recoveryPausedUntil.Load() != 0 }, 5*time.Second, 10*time.Millisecond) + assert.DirExists(t, filepath.Join(spoolDir, "m-kept"), "the discard is retried once the server answers") + assert.Empty(t, rs.mission("m-kept").received) +} + +func TestSpooledDiscard(t *testing.T) { + oldSegment := spoolSegmentBytes + spoolSegmentBytes = 256 // a few records per segment + t.Cleanup(func() { spoolSegmentBytes = oldSegment }) + + dir := filepath.Join(t.TempDir(), "m") + writeTestSpool(t, dir, "m", "p", 10) + paths, _, err := spoolSegments(dir) + require.NoError(t, err) + require.Greater(t, len(paths), 1, "the spool spans several segments") + seq, err := spooledDiscard(paths) + require.NoError(t, err) + assert.Zero(t, seq, "a spool ending with mission data was not discarded") + + discarded := filepath.Join(t.TempDir(), "d") + writeDiscardedSpool(t, discarded, "d", "p", 10) + paths, _, err = spoolSegments(discarded) + require.NoError(t, err) + seq, err = spooledDiscard(paths) + require.NoError(t, err) + assert.Equal(t, uint64(12), seq) +} + +func TestAbandonedMissionEndsAsInterrupted(t *testing.T) { + rs := newResumeServer(t) + spoolDir := t.TempDir() + b := New(recoveryConfig(rs.url(), spoolDir)) + require.NoError(t, b.Init()) + defer b.Close() + + first := startResumable(t, b) + recordStates(t, b, 1, 3) + + // The game moved on to the next mission without saving or discarding. + second := startResumable(t, b) + m := rs.mission(first) + require.True(t, m.completed, "the abandoned mission is ended before the next one starts") + assert.True(t, endPayload(t, m).Interrupted) + requireContiguous(t, m, 5) // start, 3 states, end + assert.False(t, rs.partial(first)) + requireGone(t, filepath.Join(spoolDir, first)) // nothing is left to recover + + recordStates(t, b, 1, 2) + require.NoError(t, b.EndMission("")) + assert.True(t, rs.mission(second).completed) + assert.False(t, rs.partial(second)) + assert.Equal(t, int32(2), rs.starts.Load()) +} + +func TestRecoveredMissionDoesNotSupersedeLiveOne(t *testing.T) { + fastRecovery(t) + rs := newResumeServer(t) + spoolDir := t.TempDir() + b := New(recoveryConfig(rs.url(), spoolDir)) + require.NoError(t, b.Init()) + defer b.Close() + + live := startResumable(t, b) + recordStates(t, b, 1, 3) + // A spool of this producer the server no longer holds, such as a save it + // never confirmed before a web restart; another process sharing the + // spool directory may recover it the same way. + writeTestSpool(t, filepath.Join(spoolDir, "m-left"), "m-left", b.producerID, 4) + + require.Eventually(t, func() bool { return rs.mission("m-left").completed }, 5*time.Second, 10*time.Millisecond) + start := rs.mission("m-left").received[0] + var p streaming.StartMissionPayload + require.NoError(t, json.Unmarshal(start.Payload, &p)) + assert.True(t, p.Recovered, "the resent start is marked recovered") + assert.Equal(t, "m-left", p.MissionID) + assert.False(t, rs.partial(live), "the live mission is left alone") + + recordStates(t, b, 4, 5) + require.NoError(t, b.EndMission("")) + requireContiguous(t, rs.mission(live), 7) // start, 5 states, end +} + +func TestFailedStartLeavesNoMission(t *testing.T) { + rs := newResumeServer(t) + spoolDir := t.TempDir() + b := New(recoveryConfig(rs.url(), spoolDir)) + require.NoError(t, b.Init()) + defer b.Close() + + oldAck := ackTimeout + ackTimeout = 200 * time.Millisecond + t.Cleanup(func() { ackTimeout = oldAck }) + rs.muteStart.Store(true) + require.Error(t, b.StartMission(&core.Mission{MissionName: "M"}, &core.World{WorldName: "W"})) + assert.False(t, b.recording(), "a mission that did not start is not abandoned later") + require.Eventually(t, func() bool { + entries, err := os.ReadDir(spoolDir) + return err == nil && len(entries) == 0 + }, 5*time.Second, 10*time.Millisecond, "its spool is removed") + + rs.muteStart.Store(false) + id := startResumable(t, b) + require.NoError(t, b.EndMission("")) + assert.True(t, rs.mission(id).completed) + assert.Equal(t, int32(2), rs.starts.Load(), "the next start does not end the failed one first") +} + +func TestAbandonedEmptyMissionIsDiscarded(t *testing.T) { + rs := newResumeServer(t) + spoolDir := t.TempDir() + b := New(recoveryConfig(rs.url(), spoolDir)) + require.NoError(t, b.Init()) + defer b.Close() + + // Registered, but the game moved on before anything was recorded. + first := startResumable(t, b) + second := startResumable(t, b) + m := rs.mission(first) + assert.True(t, m.discarded, "an empty recording is not kept") + requireGone(t, filepath.Join(spoolDir, first)) + require.NoError(t, b.EndMission("")) + assert.True(t, rs.mission(second).completed) +} + +func TestAbandonMissionWhenNextMissionLoads(t *testing.T) { + rs := newResumeServer(t) + spoolDir := t.TempDir() + b := New(recoveryConfig(rs.url(), spoolDir)) + require.NoError(t, b.Init()) + defer b.Close() + + require.NoError(t, b.AbandonMission(), "no-op without a mission") + id := startResumable(t, b) + recordStates(t, b, 1, 3) + require.NoError(t, b.AbandonMission()) + m := rs.mission(id) + require.True(t, m.completed) + assert.True(t, endPayload(t, m).Interrupted) + requireGone(t, filepath.Join(spoolDir, id)) + assert.False(t, b.recording()) + + next := startResumable(t, b) + require.NoError(t, b.EndMission("")) + assert.True(t, rs.mission(next).completed) +} + +func TestAbandonedMissionBelowSaveLimitsIsDiscarded(t *testing.T) { + rs := newResumeServer(t) + spoolDir := t.TempDir() + b := New(recoveryConfig(rs.url(), spoolDir)) + require.NoError(t, b.Init()) + defer b.Close() + + // An admin loads the mission, records a little and changes the mission. + short := startResumable(t, b) + b.SetDiscardAbandoned(true) + recordStates(t, b, 1, 3) + require.NoError(t, b.AbandonMission()) + m := rs.mission(short) + assert.True(t, m.discarded, "a recording the addon would not save is dropped") + requireGone(t, filepath.Join(spoolDir, short)) + + // One that met the save limits is kept as partial. + kept := startResumable(t, b) + b.SetDiscardAbandoned(true) + recordStates(t, b, 1, 3) + b.SetDiscardAbandoned(false) + require.NoError(t, b.AbandonMission()) + m = rs.mission(kept) + require.True(t, m.completed) + assert.True(t, endPayload(t, m).Interrupted) + requireNoDiscard(t, m.received) + + // Each mission starts keeping what it records. + b.SetDiscardAbandoned(true) // no mission: no-op + next := startResumable(t, b) + recordStates(t, b, 1, 3) + require.NoError(t, b.AbandonMission()) + m = rs.mission(next) + require.True(t, m.completed) + assert.True(t, endPayload(t, m).Interrupted) +} + +func TestAbandonedMissionBelowSaveLimitsOnServerWithoutDiscard(t *testing.T) { + rs := newResumeServer(t) + rs.noDrop.Store(true) + b := New(fastConfig(rs.url())) + require.NoError(t, b.Init()) + defer b.Close() + + id := startResumable(t, b) + b.SetDiscardAbandoned(true) + recordStates(t, b, 1, 3) + require.NoError(t, b.AbandonMission()) + m := rs.mission(id) + assert.True(t, m.completed, "ended, since this server cannot drop it") + requireNoDiscard(t, m.received) + assert.True(t, endPayload(t, m).Interrupted) +} + +func TestAbandonMissionOnLegacyServerDisconnects(t *testing.T) { + srv, ml := testServer(t) + defer srv.Close() + b := New(Config{URL: wsURL(srv), Secret: "s", SpoolDir: t.TempDir()}) + require.NoError(t, b.Init()) + defer b.Close() + + require.NoError(t, b.StartMission(&core.Mission{MissionName: "M"}, &core.World{WorldName: "W"})) + require.NoError(t, b.AddSoldier(&core.Soldier{ID: 1})) + require.NoError(t, b.AbandonMission()) + for _, env := range ml.all() { + require.NotEqual(t, streaming.TypeEndMission, env.Type, "a legacy server ignores interrupted and would save it whole") + } + assert.False(t, b.recording()) + require.NoError(t, b.StartMission(&core.Mission{MissionName: "N"}, &core.World{WorldName: "W"}), "the next mission dials anew") +} + +func TestRecoveryDefersOwnMissionOnServerWithoutRecovered(t *testing.T) { + fastRecovery(t) + rs := newResumeServer(t) + rs.noDrop.Store(true) // predates recovered starts: it would supersede the live mission + spoolDir := t.TempDir() + b := New(recoveryConfig(rs.url(), spoolDir)) + require.NoError(t, b.Init()) + defer b.Close() + + live := startResumable(t, b) + writeTestSpool(t, filepath.Join(spoolDir, "m-own"), "m-own", b.producerID, 3) + time.Sleep(spoolStaleAfter + 10*recoveryConfig("", "").RecoveryInterval) // several recovery passes + assert.Empty(t, rs.mission("m-own").received, "not resent while this process records") + assert.False(t, rs.partial(live)) + + require.NoError(t, b.EndMission("")) + require.Eventually(t, func() bool { return rs.mission("m-own").completed }, 5*time.Second, 10*time.Millisecond, "resent once it records none") +} + +func TestAbandonedEmptyMissionOnServerWithoutDiscard(t *testing.T) { + rs := newResumeServer(t) + rs.noDrop.Store(true) + b := New(fastConfig(rs.url())) + require.NoError(t, b.Init()) + defer b.Close() + + first := startResumable(t, b) + startResumable(t, b) + m := rs.mission(first) + assert.True(t, m.completed, "ended, since this server cannot drop it") + assert.False(t, m.discarded) + requireNoDiscard(t, m.received) + assert.True(t, endPayload(t, m).Interrupted) +} + +func TestPendingDiscardRefusedAfterServerDowngrade(t *testing.T) { + rs := newResumeServer(t) + b := New(fastConfig(rs.url())) // SaveTimeout 5 s + require.NoError(t, b.Init()) + defer b.Close() + + id := startResumable(t, b) + recordStates(t, b, 1, 3) + require.Eventually(t, func() bool { return rs.lastSeq(id) == 4 }, 2*time.Second, 5*time.Millisecond) + + // The link dies with the discard in flight, and the web server comes back + // without discard support. + rs.discard.Store(true) + done := make(chan error, 1) + start := time.Now() + go func() { done <- b.DiscardMission() }() + b.mu.Lock() + conn := b.conn + b.mu.Unlock() + require.Eventually(t, func() bool { return conn.log.lastSeq() == 5 }, time.Second, time.Millisecond, "the discard is logged") + rs.noDrop.Store(true) + rs.discard.Store(false) + rs.kill() + + select { + case err := <-done: + require.ErrorIs(t, err, storage.ErrMissionKept, "the older server refuses the waiting discard") + assert.Less(t, time.Since(start), 4*time.Second, "without waiting for the save timeout") + case <-time.After(10 * time.Second): + t.Fatal("DiscardMission did not complete") + } + assert.False(t, rs.mission(id).discarded) +} + +func TestRecoveryDeletesDroppedSpool(t *testing.T) { + fastRecovery(t) + rs := newResumeServer(t) + spoolDir := t.TempDir() + dir := filepath.Join(spoolDir, "m-dropped") + writeTestSpool(t, dir, "m-dropped", "p-old", 3) + // Its writer was removing it when recovery took it. + require.NoError(t, markSpoolDropped(dir)) + + b := New(recoveryConfig(rs.url(), spoolDir)) + require.NoError(t, b.Init()) + defer b.Close() + + requireGone(t, dir) + assert.Zero(t, rs.starts.Load(), "a dropped spool is never resent") +} + +func TestAbandonedMissionUnconfirmedKeepsSpool(t *testing.T) { + oldAbandon := abandonTimeout + abandonTimeout = 200 * time.Millisecond // below the ack stall, so no reconnect resends it + t.Cleanup(func() { abandonTimeout = oldAbandon }) + rs := newResumeServer(t) + spoolDir := t.TempDir() + cfg := recoveryConfig(rs.url(), spoolDir) + cfg.RecoveryInterval = time.Hour + b := New(cfg) + require.NoError(t, b.Init()) + defer b.Close() + + first := startResumable(t, b) + recordStates(t, b, 1, 3) + require.Eventually(t, func() bool { return rs.lastSeq(first) == 4 }, 2*time.Second, 5*time.Millisecond) + rs.loseEnd.Store(true) // the interrupted end never arrives + second := startResumable(t, b) + assert.NotEqual(t, first, second) + assert.False(t, rs.mission(first).completed) + assert.DirExists(t, filepath.Join(spoolDir, first), "kept for recovery") + rs.loseEnd.Store(false) + require.NoError(t, b.EndMission("")) + assert.True(t, rs.mission(second).completed) +} + +// failingWriter fails every write, as a full disk would. +type failingWriter struct{} + +func (failingWriter) Write([]byte) (int, error) { return 0, errors.New("no space left on device") } + +func TestUnconfirmedDiscardWithFailedFinalFlushRemovesSpool(t *testing.T) { + fastRecovery(t) + spoolFlushInterval = time.Hour // only the final flush writes the discard + rs := newResumeServer(t) + spoolDir := t.TempDir() + cfg := recoveryConfig(rs.url(), spoolDir) + cfg.SaveTimeout = 200 * time.Millisecond // below the ack stall, so no reconnect reads the spool + b := New(cfg) + require.NoError(t, b.Init()) + defer b.Close() + + id := startResumable(t, b) + recordStates(t, b, 1, 3) + require.Eventually(t, func() bool { return rs.lastSeq(id) == 4 }, 2*time.Second, 5*time.Millisecond) + b.mu.Lock() + sp := b.conn.log.currentSpool() + b.mu.Unlock() + sp.mu.Lock() + sp.flushLocked() + sp.w = bufio.NewWriter(failingWriter{}) // the next flush, the final one, fails + sp.mu.Unlock() + + rs.loseEnd.Store(true) // the discard never arrives + require.Error(t, b.DiscardMission()) + requireGone(t, filepath.Join(spoolDir, id)) + rs.loseEnd.Store(false) + time.Sleep(spoolStaleAfter + 5*cfg.RecoveryInterval) // several recovery passes + assert.Equal(t, int32(1), rs.starts.Load(), "recovery never resends the mission without its discard") + assert.False(t, rs.mission(id).completed) +} + +func TestDiscardAfterServerDowngradeIsNotSent(t *testing.T) { + rs := newResumeServer(t) + b := New(fastConfig(rs.url())) + require.NoError(t, b.Init()) + defer b.Close() + + id := startResumable(t, b) + require.True(t, b.discardable()) + recordStates(t, b, 1, 3) + require.Eventually(t, func() bool { return rs.lastSeq(id) == 4 }, 2*time.Second, 5*time.Millisecond) + + // The web server is replaced by one without discard; the mission resumes. + rs.noDrop.Store(true) + resumes := rs.resumes.Load() + rs.kill() + require.Eventually(t, func() bool { return rs.resumes.Load() > resumes }, 5*time.Second, 10*time.Millisecond) + require.Eventually(t, func() bool { return !b.discardable() }, 5*time.Second, 10*time.Millisecond) + + require.ErrorIs(t, b.DiscardMission(), storage.ErrMissionKept) + requireNoDiscard(t, rs.mission(id).received) +} + +func TestRecoveryOfDiscardAfterServerDowngrade(t *testing.T) { + fastRecovery(t) + rs := newResumeServer(t) + spoolDir := t.TempDir() + b, id := discardUnconfirmed(t, rs, spoolDir) + rs.noDrop.Store(true) // the server holds the session but cannot discard + + b.recoverSpools(b.done) + + assert.False(t, rs.mission(id).discarded) + requireNoDiscard(t, rs.mission(id).received) + assert.NoDirExists(t, filepath.Join(spoolDir, id), "an older server cannot do more") +} diff --git a/internal/storage/websocket/recover.go b/internal/storage/websocket/recover.go index f8b34fc..766a878 100644 --- a/internal/storage/websocket/recover.go +++ b/internal/storage/websocket/recover.go @@ -9,6 +9,7 @@ import ( "io" "os" "path/filepath" + "slices" "sort" "strings" "time" @@ -37,6 +38,12 @@ var ( // errAlreadySaved means the server saved the mission before; the spool // is a leftover and is deleted. errAlreadySaved = errors.New("mission already saved") + // errDiscardFinished means the spool held a mission the addon discarded + // and the server no longer keeps it; the spool is deleted. + errDiscardFinished = errors.New("mission discard finished") + // errRecoveryDeferred means the spooled mission is this process' own and + // waits until it records no other one; the spool is kept. + errRecoveryDeferred = errors.New("mission recovery deferred") ) // recoveryLoop resends missions left in the spool directory: by a game @@ -101,17 +108,27 @@ func (b *Backend) recoverSpools(done <-chan struct{}) { continue } err = b.recoverMission(dir, done) + finished := err == nil || errors.Is(err, errAlreadySaved) || errors.Is(err, errDiscardFinished) || errors.Is(err, errSpoolDiscarded) + if finished { + // Marked and emptied while locked: a process taking the spool + // before it is gone deletes it instead of resending it. + _ = dropSpoolSegments(dir) + } lock.release() // before removal: Windows cannot delete an open file + if finished { + _ = removeSpoolDir(dir) // or the next pass does + } switch { case err == nil: b.recovered.Add(1) b.logger().Info("WebSocket recovered mission from spool", "dir", dir) - _ = os.RemoveAll(dir) case errors.Is(err, errAlreadySaved): b.logger().Info("WebSocket spooled mission was already saved by the server; spool removed", "dir", dir) - _ = os.RemoveAll(dir) + case errors.Is(err, errDiscardFinished): + b.logger().Info("WebSocket discarded mission finished from spool; spool removed", "dir", dir) case errors.Is(err, errSpoolDiscarded): - _ = os.RemoveAll(dir) + case errors.Is(err, errRecoveryDeferred): + b.logger().Debug("WebSocket mission recovery waits until the current mission ends", "dir", dir) case errors.Is(err, errRecoveryUnsupported): // Asking an older server every minute only fills both logs. b.recoveryPausedUntil.Store(time.Now().Add(recoveryUnsupportedPause).UnixNano()) @@ -199,6 +216,9 @@ func scanSegment(path string, fn func(seq uint64, data []byte) error) error { // continues a session that is still waiting, or takes the whole mission when // it no longer has one. func (b *Backend) recoverMission(dir string, done <-chan struct{}) error { + if spoolDropped(dir) { + return errSpoolDiscarded // its writer or a recovery was removing it + } paths, newest, err := spoolSegments(dir) if err != nil { return err @@ -234,6 +254,15 @@ func (b *Backend) recoverMission(dir string, done <-chan struct{}) error { } defer conn.Close() + // A mission the addon discarded is not resent: the server only needs to + // hear the discard it did not confirm. Read once the server is reachable: + // it scans the spool's last segment. + discardSeq, err := spooledDiscard(paths) + if err != nil { + return fmt.Errorf("read end: %w", err) + } + discarding := discardSeq > 0 + // Give up when the backend closes. stop := make(chan struct{}) defer close(stop) @@ -249,7 +278,7 @@ func (b *Backend) recoverMission(dir string, done <-chan struct{}) error { } }() - resume, err := marshalEnvelope(streaming.TypeResume, streaming.ResumePayload{MissionID: payload.MissionID, ProducerID: payload.ProducerID}, 0) + resume, err := marshalEnvelope(streaming.TypeResume, streaming.ResumePayload{MissionID: payload.MissionID, ProducerID: payload.ProducerID, Recovered: true}, 0) if err != nil { return err } @@ -265,47 +294,113 @@ func (b *Backend) recoverMission(dir string, done <-chan struct{}) error { if err != nil { var netErr interface{ Timeout() bool } if errors.As(err, &netErr) && netErr.Timeout() { - return errRecoveryUnsupported // an older server ignores resume + // An older server ignores resume. A discarded spool exists only + // for a server that could discard, so it is kept as well: the + // server is more likely slow than downgraded. + return errRecoveryUnsupported } return fmt.Errorf("resume: %w", err) } _ = conn.SetReadDeadline(time.Time{}) + if discarding { + switch { + case reply.Type == streaming.ServerResumeCompleted: + return errDiscardFinished // the server confirmed the discard before + case reply.ProtocolVersion < streaming.ProtocolVersionDiscard: + // The server was replaced by an older one: nothing more it can do, + // and it saves what it holds as partial. + return errDiscardFinished + } + // resume_ok: the server still holds the session. resume_unknown: it + // may still keep the mission, saved as partial after the producer + // went away; the discard names it. + } + from := uint64(1) // resume_unknown: the whole mission, start included switch reply.Type { case streaming.ServerResumeCompleted: return errAlreadySaved case streaming.ServerResumeOK: from = reply.Seq + 1 + case streaming.ServerResumeUnknown: + // A server older than ProtocolVersionDiscard takes any start as the + // producer's next mission and saves its live one as partial. This + // process' own mission waits until it records none. + if !discarding && payload.ProducerID == b.producerID && b.recording() && !b.discardable() { + return errRecoveryDeferred + } } - b.logger().Info("WebSocket recovering mission", "missionId", payload.MissionID, "resumeFrom", from, "serverHasIt", reply.Type == streaming.ServerResumeOK) // Read acknowledgements concurrently: the server acknowledges as it goes, // and closes the connection if it cannot save the mission. + ackFor := streaming.TypeEndMission + if discarding { + ackFor = streaming.TypeDiscardMission + } ended := make(chan error, 1) go func() { for { _, raw, err := conn.ReadMessage() if err != nil { - ended <- fmt.Errorf("connection lost before the mission was saved: %w", err) + ended <- fmt.Errorf("connection lost before the mission was finished: %w", err) return } var m streaming.AckMessage - if json.Unmarshal(raw, &m) == nil && m.Type == streaming.ServerAck && m.For == streaming.TypeEndMission { + if json.Unmarshal(raw, &m) == nil && m.Type == streaming.ServerAck && m.For == ackFor { ended <- nil return } } }() + if discarding { + b.logger().Info("WebSocket finishing a discarded mission", "missionId", payload.MissionID, "serverHasIt", reply.Type == streaming.ServerResumeOK) + // On a resumed session, numbered after everything the server holds, + // so it is not taken for a duplicate. Without one, it is not numbered. + var seq uint64 + if reply.Type == streaming.ServerResumeOK { + seq = max(discardSeq, reply.Seq+1) + } + discardPayload := streaming.DiscardMissionPayload{MissionID: payload.MissionID, ProducerID: payload.ProducerID} + discard, err := marshalEnvelope(streaming.TypeDiscardMission, discardPayload, seq) + if err != nil { + return err + } + if err := writeData(conn, discard); err != nil { + return err + } + if err := b.awaitRecovered(ended, done); err != nil { + return err + } + return errDiscardFinished + } + b.logger().Info("WebSocket recovering mission", "missionId", payload.MissionID, "resumeFrom", from, "serverHasIt", reply.Type == streaming.ServerResumeOK) + var lastSeq uint64 var lastData []byte + first := true pace := newPacer(b.cfg.connOptions().withDefaults().resendRate) if err := scanSpool(paths, func(seq uint64, data []byte) error { lastSeq, lastData = seq, data if seq < from { + first = false return nil } + if first { + // A resent start must not take the producer's live mission + // for one it left behind. An older server ignores the mark: + // checked again right before the start, as a mission may have + // started during the handshake. + first = false + if payload.ProducerID == b.producerID && b.recording() && !b.discardable() { + return errRecoveryDeferred + } + var err error + if data, err = markRecovered(data); err != nil { + return fmt.Errorf("mark start recovered: %w", err) + } + } if !pace.wait(len(data), done, nil) { return errConnectionClosed } @@ -330,17 +425,62 @@ func (b *Backend) recoverMission(dir string, done <-chan struct{}) error { return err } } + return b.awaitRecovered(ended, done) +} - timeout := b.cfg.SaveTimeout - if timeout <= 0 { - timeout = 10 * time.Minute - } +// awaitRecovered waits for the server to acknowledge the recovered mission's +// end or discard. +func (b *Backend) awaitRecovered(ended <-chan error, done <-chan struct{}) error { select { case err := <-ended: return err - case <-time.After(timeout): - return fmt.Errorf("timeout waiting for the server to save the recovered mission") + case <-time.After(b.saveTimeout()): + return fmt.Errorf("timeout waiting for the server to finish the recovered mission") case <-done: return errConnectionClosed } } + +// markRecovered sets Recovered on a spooled start_mission envelope, keeping +// every other field as recorded. +func markRecovered(data []byte) ([]byte, error) { + var env streaming.Envelope + if err := json.Unmarshal(data, &env); err != nil { + return nil, err + } + var fields map[string]json.RawMessage + if err := json.Unmarshal(env.Payload, &fields); err != nil { + return nil, err + } + fields["recovered"] = json.RawMessage("true") + raw, err := json.Marshal(fields) + if err != nil { + return nil, err + } + return encodeEnvelope(env.Type, raw, env.Seq) +} + +// spooledDiscard returns the sequence number of the spool's discard_mission +// record, or 0 when the spool does not end with one. +func spooledDiscard(paths []string) (uint64, error) { + // Only the last record matters; a rotated segment may still be empty. + for _, path := range slices.Backward(paths) { + var lastSeq uint64 + var lastData []byte + if err := scanSegment(path, func(seq uint64, data []byte) error { + lastSeq, lastData = seq, data + return nil + }); err != nil { + return 0, err + } + if lastData == nil { + continue + } + var last streaming.Envelope + if json.Unmarshal(lastData, &last) != nil || last.Type != streaming.TypeDiscardMission { + return 0, nil + } + return lastSeq, nil + } + return 0, nil +} diff --git a/internal/storage/websocket/resume_test.go b/internal/storage/websocket/resume_test.go index 34f9683..5cf1ddf 100644 --- a/internal/storage/websocket/resume_test.go +++ b/internal/storage/websocket/resume_test.go @@ -27,11 +27,19 @@ type resumeServer struct { mu sync.Mutex missions map[string]*fakeMission conns map[*ws.Conn]string // live connection → mission id ("" before start/resume) + // partials holds missions saved as partial once their session ended + // without end_mission: superseded by their producer's next mission, or + // expired. The server no longer resumes them. + partials map[string]*fakeMission + dropped []string // missions whose partials a session-less discard_mission dropped discard atomic.Bool // drop envelopes as if lost in transit; the link stays dead noAck atomic.Bool // never send cumulative acks dropEnd atomic.Bool // save end_mission, then close without acknowledging it legacy atomic.Bool // behave like a web server without resume support + noDrop atomic.Bool // support resume but predate discard_mission + muteStart atomic.Bool // never acknowledge start_mission + loseEnd atomic.Bool // drop end_mission and discard_mission as if lost in transit resumes atomic.Int32 starts atomic.Int32 conns2ndP atomic.Int32 // connections that began with resume @@ -44,11 +52,13 @@ type fakeMission struct { dups int gaps int completed bool + discarded bool // finished by discard_mission instead of end_mission + recovered bool // started by a resend from the spool } func newResumeServer(t *testing.T) *resumeServer { t.Helper() - rs := &resumeServer{t: t, missions: map[string]*fakeMission{}, conns: map[*ws.Conn]string{}} + rs := &resumeServer{t: t, missions: map[string]*fakeMission{}, conns: map[*ws.Conn]string{}, partials: map[string]*fakeMission{}} upgrader := ws.Upgrader{} rs.srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { c, err := upgrader.Upgrade(w, r, nil) @@ -99,10 +109,32 @@ func (rs *resumeServer) serve(c *ws.Conn) { var p streaming.StartMissionPayload require.NoError(rs.t, json.Unmarshal(env.Payload, &p)) rs.starts.Add(1) + if m != nil && !m.completed { + return // a connection feeds one mission: "mission already active" + } + if rs.muteStart.Load() { + continue + } rs.mu.Lock() + // A producer records one mission at a time: its other + // unfinished missions are saved as partial. A server that + // knows recovered starts neither supersedes with one nor + // supersedes one. + knowsRecovered := !rs.noDrop.Load() + for id, other := range rs.missions { + if p.Recovered && knowsRecovered { + break + } + if id != p.MissionID && other.producer == p.ProducerID && !other.completed && !(other.recovered && knowsRecovered) { + if other.lastSeq > 1 || !knowsRecovered { + rs.partials[id] = other // a newer server drops one with nothing after its start + } + delete(rs.missions, id) + } + } m = rs.missions[p.MissionID] if m == nil { - m = &fakeMission{producer: p.ProducerID, lastSeq: env.Seq} + m = &fakeMission{producer: p.ProducerID, lastSeq: env.Seq, recovered: p.Recovered} rs.missions[p.MissionID] = m } else if env.Seq > m.lastSeq { m.lastSeq = env.Seq @@ -110,7 +142,10 @@ func (rs *resumeServer) serve(c *ws.Conn) { m.received = append(m.received, env) rs.conns[c] = p.MissionID rs.mu.Unlock() - ack := streaming.AckMessage{Type: streaming.ServerAck, For: env.Type, Seq: env.Seq, ProtocolVersion: streaming.ProtocolVersionResume} + ack := streaming.AckMessage{Type: streaming.ServerAck, For: env.Type, Seq: env.Seq, ProtocolVersion: streaming.ProtocolVersionDiscard} + if rs.noDrop.Load() { + ack.ProtocolVersion = streaming.ProtocolVersionResume + } if rs.legacy.Load() { ack = streaming.AckMessage{Type: streaming.ServerAck, For: env.Type} } @@ -139,12 +174,29 @@ func (rs *resumeServer) serve(c *ws.Conn) { m = nil } rs.mu.Unlock() + if !rs.noDrop.Load() { + reply.ProtocolVersion = streaming.ProtocolVersionDiscard + } if c.WriteJSON(reply) != nil { return } default: - if m == nil { + if m == nil && env.Type == streaming.TypeDiscardMission && !rs.noDrop.Load() { + // No session: drop what the server kept of the mission. + var p streaming.DiscardMissionPayload + require.NoError(rs.t, json.Unmarshal(env.Payload, &p)) + rs.mu.Lock() + delete(rs.partials, p.MissionID) + rs.dropped = append(rs.dropped, p.MissionID) + rs.mu.Unlock() + _ = c.WriteJSON(streaming.AckMessage{Type: streaming.ServerAck, For: env.Type}) + return + } + if m == nil || (env.Type == streaming.TypeDiscardMission && rs.noDrop.Load()) { + continue // an older server ignores discard_mission + } + if (env.Type == streaming.TypeEndMission || env.Type == streaming.TypeDiscardMission) && rs.loseEnd.Load() { continue } rs.mu.Lock() @@ -159,12 +211,14 @@ func (rs *resumeServer) serve(c *ws.Conn) { m.lastSeq = env.Seq m.received = append(m.received, env) last := m.lastSeq - if env.Type == streaming.TypeEndMission { + finished := env.Type == streaming.TypeEndMission || env.Type == streaming.TypeDiscardMission + if finished { m.completed = true + m.discarded = env.Type == streaming.TypeDiscardMission } rs.mu.Unlock() - if env.Type == streaming.TypeEndMission { + if finished { if rs.dropEnd.Load() { return // saved, but the acknowledgement never arrives } @@ -207,6 +261,31 @@ func (rs *resumeServer) forget() { rs.missions = map[string]*fakeMission{} } +// expire saves an unfinished mission as partial, as the web server does when +// its producer stays away past the resume grace. +func (rs *resumeServer) expire(id string) { + rs.mu.Lock() + defer rs.mu.Unlock() + if m := rs.missions[id]; m != nil { + rs.partials[id] = m + delete(rs.missions, id) + } +} + +// partial reports whether the server keeps a partial recording of the mission. +func (rs *resumeServer) partial(id string) bool { + rs.mu.Lock() + defer rs.mu.Unlock() + return rs.partials[id] != nil +} + +// droppedMissions lists the missions session-less discards dropped. +func (rs *resumeServer) droppedMissions() []string { + rs.mu.Lock() + defer rs.mu.Unlock() + return append([]string(nil), rs.dropped...) +} + func (rs *resumeServer) mission(id string) fakeMission { rs.mu.Lock() defer rs.mu.Unlock() diff --git a/internal/storage/websocket/spool.go b/internal/storage/websocket/spool.go index 8c4b1e3..a03d07b 100644 --- a/internal/storage/websocket/spool.go +++ b/internal/storage/websocket/spool.go @@ -292,32 +292,153 @@ func readSegment(path string, from, to uint64, maxBytes int, out *[]logEntry, by // close flushes and closes the spool, keeping its files. func (s *spool) close() error { + _, err := s.shut(false, false) + return err +} + +// keepIntact closes a spool kept for recovery to finish the mission with the +// record it ends with, such as discard_mission. If any record, the last one +// included, did not reach the disk, the spool is removed instead, as remove +// does: recovery would resend the mission without it, and the server save +// it. It reports whether the spool was kept. +func (s *spool) keepIntact() bool { + dropped, err := s.shut(false, true) + if !dropped { + if err != nil { + s.logger.Warn("WebSocket spool closed with an error", "dir", s.dir, "error", err) + } + return true + } + s.logger.Warn("WebSocket spool incomplete on disk; removing it so recovery does not resend the mission", "dir", s.dir, "error", err) + if rerr := removeSpoolDir(s.dir); rerr != nil && !spoolDropped(s.dir) { + s.logger.Warn("WebSocket spool not removed", "dir", s.dir, "error", rerr) + } + return false +} + +// remove closes the spool and deletes its files. It is marked dropped and its +// segments go before the lock is released, so recovery never takes up a +// spool being removed: it would resend a mission that was saved, discarded or +// never started. Recovery deletes a dropped spool it finds, so a directory +// another process holds open for a moment is not an error. +func (s *spool) remove() error { + _, _ = s.shut(true, false) + if err := removeSpoolDir(s.dir); err != nil { + if spoolDropped(s.dir) { + s.logger.Debug("WebSocket spool left for recovery to delete", "dir", s.dir, "error", err) + return nil + } + return err + } + return nil +} + +// removeSpoolDir deletes a dropped spool's directory. The dropped marker goes +// last: while anything else is left, recovery still knows not to resend it. +func removeSpoolDir(dir string) error { + entries, err := os.ReadDir(dir) + if err != nil { + if os.IsNotExist(err) { + return nil + } + return err + } + var first error + for _, e := range entries { + if e.Name() == spoolDroppedFile { + continue + } + if err := os.RemoveAll(filepath.Join(dir, e.Name())); err != nil && first == nil { + first = err + } + } + if first != nil { + return first + } + if err := os.Remove(filepath.Join(dir, spoolDroppedFile)); err != nil && !os.IsNotExist(err) { + return err + } + return os.Remove(dir) +} + +// dropSpoolSegments marks the spool in dir dropped and deletes its segments. +// Call it while holding the spool's lock. +func dropSpoolSegments(dir string) error { + if err := markSpoolDropped(dir); err != nil { + return err + } + paths, _, err := spoolSegments(dir) + if err != nil { + return err + } + var first error + for _, p := range paths { + if err := os.Remove(p); err != nil && first == nil { + first = err + } + } + return first +} + +// spoolDroppedFile marks a spool whose mission needs nothing more; recovery +// deletes it. +const spoolDroppedFile = "dropped" + +// markSpoolDropped marks the spool in dir dropped. Call it while holding the +// spool's lock. +func markSpoolDropped(dir string) error { + return os.WriteFile(filepath.Join(dir, spoolDroppedFile), nil, 0o644) +} + +func spoolDropped(dir string) bool { + _, err := os.Stat(filepath.Join(dir, spoolDroppedFile)) + return err == nil +} + +// shut closes the spool, deleting its segments first when drop is set, or +// when intact is set and a record did not reach the disk; all under the +// spool's lock, so recovery never takes up an incomplete one. It reports +// whether it deleted them. +func (s *spool) shut(drop, intact bool) (bool, error) { s.mu.Lock() defer s.mu.Unlock() if s.closed { - return nil + return false, nil } s.closed = true close(s.done) var err error if s.file != nil { - if s.ioErr == nil { + if s.ioErr == nil && !drop { err = s.w.Flush() + if intact && err == nil { + err = s.file.Sync() + } } if cerr := s.file.Close(); err == nil { err = cerr } } + if intact && (err != nil || s.failed != nil) { + if err == nil { + err = s.failed + } + drop = true + } + if drop { + if merr := markSpoolDropped(s.dir); err == nil && merr != nil { + err = merr + } + for _, seg := range s.segments { + if rerr := os.Remove(seg.path); err == nil && rerr != nil && !os.IsNotExist(rerr) { + err = rerr + } + } + } // The mission is complete on disk; recovery may take it from here. if rerr := os.Remove(filepath.Join(s.dir, spoolLiveFile)); err == nil && rerr != nil && !os.IsNotExist(rerr) { err = rerr } s.lock.release() - return err -} - -// remove closes the spool and deletes its files. -func (s *spool) remove() error { - _ = s.close() - return os.RemoveAll(s.dir) + return drop, err } diff --git a/internal/storage/websocket/web_integration_test.go b/internal/storage/websocket/web_integration_test.go index 2d22cb7..6217284 100644 --- a/internal/storage/websocket/web_integration_test.go +++ b/internal/storage/websocket/web_integration_test.go @@ -89,3 +89,39 @@ func TestWebIngestionIntegration(t *testing.T) { } t.Fatal("web server did not publish the acknowledged mission") } + +// A discarded mission must leave no operation behind on the web server. +func TestWebDiscardIntegration(t *testing.T) { + base := os.Getenv("OCAP_TEST_WEB_URL") + if base == "" { + t.Skip("set OCAP_TEST_WEB_URL to a disposable local web server") + } + secret := os.Getenv("OCAP_TEST_WEB_SECRET") + b := New(Config{URL: strings.Replace(base, "http", "ws", 1) + "/api/v1/stream", Secret: secret}) + require.NoError(t, b.Init()) + defer b.Close() + name := fmt.Sprintf("Proxy Discard %d", time.Now().UnixNano()) + require.NoError(t, b.StartMission(&core.Mission{MissionName: name, CaptureDelay: 1}, &core.World{WorldName: "altis"})) + require.NoError(t, b.AddSoldier(&core.Soldier{ID: 1, JoinFrame: 1, UnitName: "Proxy Player", IsPlayer: true, Side: "WEST", ClassName: "B_Soldier_F"})) + require.NoError(t, b.RecordSoldierState(&core.SoldierState{SoldierID: 1, CaptureFrame: 1, Lifestate: 1, IsPlayer: true, UnitName: "Proxy Player", Side: "WEST"})) + require.NoError(t, b.DiscardMission()) + // The next mission still records. + require.NoError(t, b.StartMission(&core.Mission{MissionName: name + " kept", CaptureDelay: 1}, &core.World{WorldName: "altis"})) + require.NoError(t, b.RecordTelemetryEvent(&core.TelemetryEvent{CaptureFrame: 1, FpsAverage: 50})) + require.NoError(t, b.EndMission("")) + + resp, err := http.Get(base + "/api/v1/operations") + require.NoError(t, err) + defer resp.Body.Close() + require.Equal(t, http.StatusOK, resp.StatusCode) + var ops []struct { + Name string `json:"mission_name"` + } + require.NoError(t, json.NewDecoder(resp.Body).Decode(&ops)) + kept := false + for _, op := range ops { + require.NotEqual(t, name, op.Name, "the discarded mission was published") + kept = kept || op.Name == name+" kept" + } + require.True(t, kept, "web server did not publish the mission after the discarded one") +} diff --git a/internal/storage/websocket/websocket.go b/internal/storage/websocket/websocket.go index 19e1967..977c4cc 100644 --- a/internal/storage/websocket/websocket.go +++ b/internal/storage/websocket/websocket.go @@ -3,6 +3,7 @@ package websocket import ( "crypto/rand" "encoding/json" + "errors" "fmt" "log/slog" "path/filepath" @@ -10,6 +11,7 @@ import ( "sync/atomic" "time" + "github.com/OCAP2/extension/v5/internal/storage" "github.com/OCAP2/extension/v5/pkg/core" "github.com/OCAP2/extension/v5/pkg/streaming" ) @@ -57,18 +59,21 @@ func (cfg Config) connOptions() connOptions { // Backend streams mission data over WebSocket to the OCAP2 web server. // It implements storage.Backend but not storage.Uploadable. type Backend struct { - missionMu sync.Mutex - mu sync.Mutex - restart bool - active bool // a mission has started and not ended - conn *connection - cfg Config - producerID string // identifies this extension process to the server - missionID string // identifies the current recording - nextMarkerID atomic.Uint64 - done chan struct{} // closed by Close; stops spool recovery - closeOnce sync.Once - loops sync.WaitGroup // recovery and stats loops + missionMu sync.Mutex + mu sync.Mutex + restart bool + active bool // a mission has started and not ended + // discardAbandoned makes abandonMission drop the mission; see + // SetDiscardAbandoned. + discardAbandoned bool + conn *connection + cfg Config + producerID string // identifies this extension process to the server + missionID string // identifies the current recording + nextMarkerID atomic.Uint64 + done chan struct{} // closed by Close; stops spool recovery + closeOnce sync.Once + loops sync.WaitGroup // recovery and stats loops // Totals of connections replaced between missions; see Stats. reconnects, resumes, restarts atomic.Uint64 @@ -210,6 +215,11 @@ func (b *Backend) sendEnvelopeAndWait(msgType string, payload any, timeout time. func (b *Backend) StartMission(mission *core.Mission, world *core.World) error { b.missionMu.Lock() defer b.missionMu.Unlock() + + // Normally done when the next mission loads (AbandonMission); this + // covers a start without one. + b.abandonMission() + b.mu.Lock() // A completed mission's server closes its socket. Dial a fresh connection // before accepting the next mission instead of racing the reconnect loop. @@ -227,13 +237,15 @@ func (b *Backend) StartMission(mission *core.Mission, world *core.World) error { } conn := b.conn b.missionID = newID() + b.discardAbandoned = false missionID := b.missionID b.mu.Unlock() + conn.protocol.Store(0) base := uint8(1) raw, err := marshalPayload(streaming.TypeStartMission, streaming.StartMissionPayload{ Mission: mission, World: world, FrameBase: &base, - MissionID: missionID, ProducerID: b.producerID, ProtocolVersion: streaming.ProtocolVersionResume, + MissionID: missionID, ProducerID: b.producerID, ProtocolVersion: streaming.ProtocolVersionDiscard, }) if err != nil { return err @@ -266,18 +278,89 @@ func (b *Backend) StartMission(mission *core.Mission, world *core.World) error { ack, err := conn.waitAck(streaming.TypeStartMission, ackTimeout) if err != nil { + // Nothing records a mission that did not start: forget it, so the + // next start does not take it for an abandoned one. A server that + // started it late loses its producer and drops the empty session. + if sp := b.clearMission(); sp != nil { + removeSpool(sp) + } + _ = conn.close() return err } + // Stored first: a reconnect from here on records what it finds. + conn.protocol.Store(uint32(ack.ProtocolVersion)) resumable := ack.ProtocolVersion >= streaming.ProtocolVersionResume conn.setResumable(resumable) if resumable { conn.ackUpTo(max(ack.Seq, 1)) b.resumeRecovery() } - slog.Default().Info("WebSocket mission started", "missionId", missionID, "resumable", resumable) + slog.Default().Info("WebSocket mission started", "missionId", missionID, "resumable", resumable, "discardable", b.discardable()) return nil } +// AbandonMission ends a mission the game moved on from without saving or +// discarding it, for example after an admin changed the mission. It ends as +// interrupted, so the server saves what it got as partial and nothing is left +// to recover. One that recorded nothing after its start, or one the addon +// would not save yet (SetDiscardAbandoned), is dropped instead, when the +// server can. A server without resume support saves a mission as partial +// when its producer disconnects, so the connection is closed instead. +// It is a no-op when no mission is recording. +func (b *Backend) AbandonMission() error { + b.missionMu.Lock() + defer b.missionMu.Unlock() + return b.abandonMission() +} + +// abandonMission is AbandonMission with missionMu held. +func (b *Backend) abandonMission() error { + if !b.recording() { + return nil + } + b.mu.Lock() + conn, missionID, discard := b.conn, b.missionID, b.discardAbandoned + b.mu.Unlock() + switch { + case !conn.isResumable(): + slog.Default().Warn("WebSocket mission abandoned without save or discard; disconnecting, the server saves it as partial", "missionId", missionID) + if sp := b.clearMission(); sp != nil { + removeSpool(sp) + } + return conn.close() + case conn.log.lastSeq() <= 1 && b.discardable(): + slog.Default().Warn("WebSocket mission abandoned before it recorded anything; discarding it", "missionId", missionID) + return b.discardMission(abandonTimeout) + case discard && b.discardable(): + slog.Default().Warn("WebSocket mission abandoned before it met the addon's save limits; discarding it", "missionId", missionID) + return b.discardMission(abandonTimeout) + default: + slog.Default().Warn("WebSocket mission abandoned without save or discard; ending it as interrupted", "missionId", missionID) + return b.endMission(streaming.EndMissionPayload{Interrupted: true}, abandonTimeout) + } +} + +// SetDiscardAbandoned sets whether AbandonMission drops the current mission, +// for a recording the addon would not save yet. Each mission starts without +// it. It does not take missionMu, so a save holding it never blocks the +// game's thread here. +func (b *Backend) SetDiscardAbandoned(discard bool) { + b.mu.Lock() + defer b.mu.Unlock() + if b.active { + b.discardAbandoned = discard + } +} + +// discardable reports whether the server serving the mission accepts +// discard_mission, as it last announced. +func (b *Backend) discardable() bool { + b.mu.Lock() + conn := b.conn + b.mu.Unlock() + return conn.protocol.Load() >= streaming.ProtocolVersionDiscard +} + // EndMission sends end_mission and waits for server ack. // tag is the save-time override from :MISSION:SAVE:; empty keeps the // mission-start tag on the web side and sends a null payload for byte @@ -290,36 +373,146 @@ func (b *Backend) EndMission(tag string) error { if tag != "" { payload = streaming.EndMissionPayload{Tag: tag} } - timeout := b.cfg.SaveTimeout - if timeout <= 0 { - timeout = 10 * time.Minute - } + return b.endMission(payload, b.saveTimeout()) +} + +// abandonTimeout bounds the wait for the server to save an abandoned mission +// before the next one starts; past it, the spool is kept for recovery. Tests +// lower it. +var abandonTimeout = 30 * time.Second + +// endMission sends end_mission, waits for its ack and forgets the mission. +// Caller holds missionMu. +func (b *Backend) endMission(payload any, timeout time.Duration) error { err := b.sendEnvelopeAndWait(streaming.TypeEndMission, payload, timeout) // Clear mission state regardless of error. + sp := b.clearMission() + + // The spool is needed until the server has saved the mission. + if sp != nil { + if err == nil { + removeSpool(sp) + } else { + _ = sp.close() + slog.Default().Warn("WebSocket mission not confirmed by the server; spool kept", "dir", sp.dir, "error", err) + } + } + + return err +} + +// errDiscardUnsupported means the server serving the mission predates +// discard_mission. +var errDiscardUnsupported = fmt.Errorf("websocket server cannot discard missions: %w", storage.ErrMissionKept) + +// DiscardMission ends the mission without saving it: it sends +// discard_mission, and the server drops the recording. A server older than +// ProtocolVersionDiscard cannot drop it. The mission is then abandoned: the +// connection is closed and nothing is kept for recovery, but that server +// still saves what it received as partial, as it did before discard existed, +// and DiscardMission returns storage.ErrMissionKept. +func (b *Backend) DiscardMission() error { + b.missionMu.Lock() + defer b.missionMu.Unlock() + return b.discardMission(b.saveTimeout()) +} + +// discardMission sends discard_mission, waits for its ack and forgets the +// mission. Caller holds missionMu. +func (b *Backend) discardMission(timeout time.Duration) error { + b.mu.Lock() + active, missionID := b.active, b.missionID + b.mu.Unlock() + if !active { + return nil + } + + b.mu.Lock() + conn := b.conn + b.mu.Unlock() + conn.drainAcks() + // A reconnect that finds the server replaced by an older one lowers the + // version, and refuses a discard already waiting for its ack. + discardable := b.discardable() + err := errDiscardUnsupported + logged := false // discard_mission is in the log, and so in a healthy spool + if discardable { + payload := streaming.DiscardMissionPayload{MissionID: missionID, ProducerID: b.producerID} + err = conn.send(func(seq uint64) ([]byte, error) { + return marshalEnvelope(streaming.TypeDiscardMission, payload, seq) + }) + if err == nil { + logged = true + _, err = conn.waitAck(streaming.TypeDiscardMission, timeout) + } + if errors.Is(err, errDiscardUnsupported) { + discardable = false + } + } + + // A spool that holds discard_mission lets recovery finish the discard. + // Without it, recovery would resend the mission and the server save it, + // so one whose last records do not reach the disk is removed instead. + sp := b.clearMission() + keep := err != nil && discardable && logged && sp != nil && sp.healthy() + if sp != nil { + if keep { + keep = sp.keepIntact() + } else { + removeSpool(sp) + } + } + + switch { + case err == nil: + slog.Default().Info("WebSocket mission discarded", "missionId", missionID) + case !discardable: + _ = conn.close() // the server stops waiting for the mission; StartMission dials anew + slog.Default().Warn("WebSocket server cannot discard missions; it saves what it received as partial", "missionId", missionID) + case keep: + slog.Default().Warn("WebSocket mission discard not confirmed by the server; spool kept to finish it", "missionId", missionID, "dir", sp.dir, "error", err) + default: + slog.Default().Warn("WebSocket mission discard not confirmed by the server", "missionId", missionID, "error", err) + } + return err +} + +// clearMission forgets the current mission after end_mission or +// discard_mission and returns its spool, if any, for the caller to keep or +// remove. +func (b *Backend) clearMission() *spool { b.mu.Lock() conn := b.conn b.restart = true b.active = false + b.discardAbandoned = false b.missionID = "" b.mu.Unlock() sp := conn.log.currentSpool() conn.endMission() b.nextMarkerID.Store(0) + return sp +} - // The spool is needed until the server has saved the mission. - if sp != nil { - if err == nil { - if rerr := sp.remove(); rerr != nil { - slog.Default().Warn("WebSocket spool not removed", "dir", sp.dir, "error", rerr) - } - } else { - _ = sp.close() - slog.Default().Warn("WebSocket mission not confirmed by the server; spool kept", "dir", sp.dir, "error", err) - } +// recording reports whether a mission has started and not ended. +func (b *Backend) recording() bool { + b.mu.Lock() + defer b.mu.Unlock() + return b.active +} + +func (b *Backend) saveTimeout() time.Duration { + if b.cfg.SaveTimeout <= 0 { + return 10 * time.Minute } + return b.cfg.SaveTimeout +} - return err +func removeSpool(sp *spool) { + if err := sp.remove(); err != nil { + slog.Default().Warn("WebSocket spool not removed", "dir", sp.dir, "error", err) + } } func (b *Backend) AddSoldier(s *core.Soldier) error { diff --git a/internal/storage/websocket/websocket_test.go b/internal/storage/websocket/websocket_test.go index 0a88b63..5a70fbf 100644 --- a/internal/storage/websocket/websocket_test.go +++ b/internal/storage/websocket/websocket_test.go @@ -463,7 +463,7 @@ func TestStartMissionRegistersStart(t *testing.T) { require.NoError(t, json.Unmarshal(msgs[0].Payload, &payload)) assert.NotEmpty(t, payload.MissionID) assert.Equal(t, b.producerID, payload.ProducerID) - assert.Equal(t, uint8(streaming.ProtocolVersionResume), payload.ProtocolVersion) + assert.Equal(t, uint8(streaming.ProtocolVersionDiscard), payload.ProtocolVersion) require.NoError(t, b.EndMission("")) diff --git a/pkg/streaming/messages.go b/pkg/streaming/messages.go index 31c7e33..ed9852a 100644 --- a/pkg/streaming/messages.go +++ b/pkg/streaming/messages.go @@ -63,6 +63,29 @@ type Envelope struct { // envelopes, cumulative acknowledgements and mission resume. const ProtocolVersionResume = 2 +// ProtocolVersionDiscard is the first protocol version whose servers accept +// discard_mission. +const ProtocolVersionDiscard = 3 + +// TypeDiscardMission ends a mission without saving it, in place of +// end_mission: the addon decided the recording does not meet its save limits, +// for example the minimum mission duration. The server drops the recording +// and acknowledges it like end_mission. On a session it is numbered like any +// mission envelope; without one, it is not numbered and the payload alone +// names the mission. It is sent only to servers that acknowledged +// start_mission, or answered resume, with ProtocolVersionDiscard or later. +// Saved missions never carry it, so it is not part of AllMessageTypes. Its +// payload is DiscardMissionPayload. +const TypeDiscardMission = "discard_mission" + +// DiscardMissionPayload names the discarded mission. A server that no longer +// holds the mission's session, for example because it saved it as partial +// after the producer went away, drops what it kept of the mission by id. +type DiscardMissionPayload struct { + MissionID string `json:"missionId"` + ProducerID string `json:"producerId"` +} + // Connection control messages. They are not mission data, so they are not // part of AllMessageTypes. const ( @@ -82,7 +105,8 @@ const ( // mission, for example after a web server restart. ServerResumeUnknown = "resume_unknown" // ServerResumeCompleted answers resume for a mission the server has - // already saved. It stands in for the end_mission acknowledgement. + // already saved, or discarded. It stands in for the end_mission or + // discard_mission acknowledgement. ServerResumeCompleted = "resume_completed" ) @@ -93,7 +117,8 @@ type AckMessage struct { // Seq is the highest envelope the server has received (ack, resume_ok). Seq uint64 `json:"seq,omitempty"` // ProtocolVersion is set on the start_mission acknowledgement by servers - // that support resume. Older servers omit it. + // that support resume, and on resume replies by servers that support + // discard. Older servers omit it. ProtocolVersion uint8 `json:"protocolVersion,omitempty"` } @@ -108,12 +133,23 @@ type StartMissionPayload struct { MissionID string `json:"missionId,omitempty"` // unique per recording ProducerID string `json:"producerId,omitempty"` // unique per extension process ProtocolVersion uint8 `json:"protocolVersion,omitempty"` + // Recovered marks a start resent from the spool, for a mission the + // producer no longer records. A server takes any other start as the + // producer's next mission and saves the producer's unfinished ones as + // partial; servers at ProtocolVersionDiscard or later leave them alone + // for a recovered start. + Recovered bool `json:"recovered,omitempty"` } // ResumePayload identifies the mission a replacement connection continues. type ResumePayload struct { MissionID string `json:"missionId"` ProducerID string `json:"producerId"` + // Recovered marks a resume from the spool, for a mission the producer + // no longer records; like StartMissionPayload.Recovered, it keeps the + // session from being taken for one the producer's next start + // supersedes. + Recovered bool `json:"recovered,omitempty"` } // EndMissionPayload carries finalization hints for end_mission.