From a7d2febff16ebf51fb0f88527309f331f3bf1868 Mon Sep 17 00:00:00 2001 From: Smith <13200576+smitt14ua@users.noreply.github.com> Date: Mon, 26 Oct 2026 21:37:18 +0200 Subject: [PATCH 01/10] feat(websocket): discard missions the addon does not save The addon only enforces its save limits (minimum mission time) by not sending :MISSION:SAVE:. With the websocket backend that is not enough: the web server already holds the streamed mission and saves it as partial once the producer goes away, and spool recovery ends it as interrupted. Short missions the addon chose not to keep ended up on the web server anyway. - :MISSION:DISCARD: ends the recording without saving it and answers with a :MISSION:DISCARDED: callback (ok, kept, error). Backends opt in through storage.Discardable; memory drops its data, postgres and sqlite keep what they wrote. - Protocol version 3 adds discard_mission. The producer advertises it in start_mission and only sends it to servers that acknowledge version 3. Older servers keep the previous behavior: the mission is abandoned and the server saves it as partial. - A discard that is not confirmed keeps the spool, which now ends with discard_mission. Recovery then only repeats the discard instead of resending the mission; a server that no longer holds the mission, or cannot resume, has nothing left to discard. - resume_completed stands in for a lost discard_mission ack as it does for end_mission. - A mission abandoned by the next StartMission closes its spool, so recovery can end it while the process runs instead of after it exits. --- CLAUDE.md | 2 +- README.md | 9 + cmd/ocap_recorder/main.go | 7 + cmd/ocap_recorder/mission_save.go | 59 ++++ cmd/ocap_recorder/mission_save_test.go | 105 +++++++ internal/storage/memory/memory.go | 13 + internal/storage/memory/memory_test.go | 28 ++ internal/storage/storage.go | 9 + internal/storage/websocket/connection.go | 7 +- internal/storage/websocket/discard_test.go | 276 ++++++++++++++++++ internal/storage/websocket/recover.go | 88 +++++- internal/storage/websocket/resume_test.go | 13 +- .../storage/websocket/web_integration_test.go | 36 +++ internal/storage/websocket/websocket.go | 123 ++++++-- internal/storage/websocket/websocket_test.go | 2 +- pkg/streaming/messages.go | 13 + 16 files changed, 756 insertions(+), 34 deletions(-) create mode 100644 internal/storage/websocket/discard_test.go diff --git a/CLAUDE.md b/CLAUDE.md index 73b18b5..a91dffe 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -78,7 +78,7 @@ All commands follow a `:RESOURCE:ACTION:` naming convention. New commands must u | `:PLACED:CREATE:`, `:PLACED:EVENT:` | Placed object (mine/explosive) lifecycle | | `:ACE3:DEATH:`, `:ACE3:UNCONSCIOUS:` | ACE3 integration events | | `:SYS:INIT:`, `:STORAGE:INIT:` | Initialize extension and storage | -| `:MISSION:START:`, `:MISSION:SAVE:` | Begin/end recording | +| `:MISSION:START:`, `:MISSION:SAVE:`, `:MISSION:DISCARD:` | Begin/end recording, or end it unsaved | ### Data Flow diff --git a/README.md b/README.md index f39901f..eb8a35f 100644 --- a/README.md +++ b/README.md @@ -162,6 +162,14 @@ mission: saved as `partial`, with everything recorded up to the last flush (at most about a second is lost). - A mission whose save was not confirmed is saved as recorded. +- A mission whose discard was not confirmed is not resent; the web service is + only told again to drop it. + +When the addon ends a recording that does not meet its save limits (for +example one shorter than `OCAP_settings_minMissionTime`), it sends +`:MISSION:DISCARD:`, and the proxy tells the web service to drop the recording +(protocol version 3) and deletes its spool. A web service that predates +discard keeps saving such recordings as `partial`. Game servers can share one `spoolDir`, for example several instances started from the same directory. Each spool has an operating system file lock @@ -268,6 +276,7 @@ All commands follow a `:RESOURCE:ACTION:` naming convention. New commands must u | `:STORAGE:INIT:` | Initialize storage backend, ungate buffered handlers | | `:MISSION:START:` | Start recording mission | | `:MISSION:SAVE:` | End recording, flush data, upload if configured | +| `:MISSION:DISCARD:` | End recording without saving it (`memory`, `websocket`); answers with a `:MISSION:DISCARDED:` callback | | `:MISSION:FOCUS_START:` | Set playback focus start frame | | `:MISSION:FOCUS_END:` | Set playback focus end frame | | `:SYS:VERSION:` | Get extension version | diff --git a/cmd/ocap_recorder/main.go b/cmd/ocap_recorder/main.go index bbaabcd..ac5fd98 100644 --- a/cmd/ocap_recorder/main.go +++ b/cmd/ocap_recorder/main.go @@ -416,4 +416,11 @@ func registerLifecycleHandlers(d *dispatcher.Dispatcher) { Logger.Info("Received :MISSION:SAVE: command, queuing async save", "tag", tag) return triggerMissionSave(saveState, storageBackend, apiClient, tag) }) + + // The addon ended a recording without saving it, for example one shorter + // than its minimum mission time. + d.Register(":MISSION:DISCARD:", func(e dispatcher.Event) (any, error) { + Logger.Info("Received :MISSION:DISCARD: command, queuing async discard") + return triggerMissionDiscard(saveState, storageBackend) + }) } diff --git a/cmd/ocap_recorder/mission_save.go b/cmd/ocap_recorder/mission_save.go index 62d7ae8..3a90ead 100644 --- a/cmd/ocap_recorder/mission_save.go +++ b/cmd/ocap_recorder/mission_save.go @@ -46,6 +46,65 @@ func triggerMissionSave(state *missionSaveState, backend storage.Backend, client return "queued", nil } +// triggerMissionDiscard is the logic behind the :MISSION:DISCARD: dispatcher +// handler: the addon ended a recording that does not meet its save limits. +// It shares the in-flight guard with saves, as both end the mission. +// +// Contract: +// - Returns "queued" + nil error when a goroutine was spawned; it reports +// completion via a ":MISSION:DISCARDED:" callback: ["ok"], ["kept"] for +// a backend that keeps what it recorded, or ["error", message]. +// - Returns a non-nil error if a save or discard is already in progress. +func triggerMissionDiscard(state *missionSaveState, backend storage.Backend) (any, error) { + if backend == nil { + return nil, fmt.Errorf("no storage backend configured") + } + if !state.inFlight.CompareAndSwap(false, true) { + return nil, fmt.Errorf("save already in progress") + } + + go runMissionDiscardWorker(state, backend) + return "queued", nil +} + +// runMissionDiscardWorker performs DiscardMission, recovers from panics and +// dispatches the :MISSION:DISCARDED: callback. +func runMissionDiscardWorker(state *missionSaveState, backend storage.Backend) { + defer state.inFlight.Store(false) + + d, ok := backend.(storage.Discardable) + if !ok { + if Logger != nil { + Logger.Info("Storage backend keeps discarded missions") + } + _ = a3interface.WriteArmaCallback(ExtensionName, ":MISSION:DISCARDED:", "kept") + return + } + + err := func() (err error) { + defer func() { + if r := recover(); r != nil { + err = fmt.Errorf("panic during mission discard: %v", r) + if Logger != nil { + Logger.Error("panic during mission discard", "panic", r, "stack", string(debug.Stack())) + } + } + }() + return d.DiscardMission() + }() + if err != nil { + if Logger != nil { + Logger.Error("Failed to discard mission in storage backend", "error", err) + } + _ = a3interface.WriteArmaCallback(ExtensionName, ":MISSION:DISCARDED:", "error", err.Error()) + return + } + if Logger != nil { + Logger.Info("Mission recording discarded") + } + _ = a3interface.WriteArmaCallback(ExtensionName, ":MISSION:DISCARDED:", "ok") +} + // runMissionSaveWorker performs EndMission + optional upload, recovers from // panics, flushes OTel, and dispatches the :MISSION:SAVED: callback. func runMissionSaveWorker(state *missionSaveState, backend storage.Backend, client *api.Client, tag string) { diff --git a/cmd/ocap_recorder/mission_save_test.go b/cmd/ocap_recorder/mission_save_test.go index 742d64e..0ba9f52 100644 --- a/cmd/ocap_recorder/mission_save_test.go +++ b/cmd/ocap_recorder/mission_save_test.go @@ -240,3 +240,108 @@ type panickingBackend struct{ fakeBackend } func (p *panickingBackend) EndMission(tag string) error { panic(fmt.Errorf("boom")) } + +// discardingBackend is a fakeBackend that also implements storage.Discardable. +type discardingBackend struct { + fakeBackend + discardErr error + discardPanic bool + discards int +} + +func (d *discardingBackend) DiscardMission() error { + d.discards++ + if d.discardPanic { + panic("boom") + } + return d.discardErr +} + +var _ storage.Discardable = (*discardingBackend)(nil) + +func TestRunMissionDiscard_Success(t *testing.T) { + rec := &callbackRecorder{} + a3interface.SetCallbackSinkForTest(rec.sink) + t.Cleanup(func() { a3interface.SetCallbackSinkForTest(nil) }) + + state := newTestSaveState() + backend := &discardingBackend{} + + result, err := triggerMissionDiscard(state, backend) + require.NoError(t, err) + assert.Equal(t, "queued", result) + + cb := rec.wait(t, ":MISSION:DISCARDED:") + assert.Equal(t, []string{`"ok"`}, cb.data) + assert.Equal(t, 1, backend.discards) + assert.Empty(t, backend.endTag, "a discarded mission is not saved") + assert.False(t, state.inFlight.Load()) +} + +func TestRunMissionDiscard_BackendKeepsData(t *testing.T) { + rec := &callbackRecorder{} + a3interface.SetCallbackSinkForTest(rec.sink) + t.Cleanup(func() { a3interface.SetCallbackSinkForTest(nil) }) + + state := newTestSaveState() + _, err := triggerMissionDiscard(state, &fakeBackend{}) + require.NoError(t, err) + + cb := rec.wait(t, ":MISSION:DISCARDED:") + assert.Equal(t, []string{`"kept"`}, cb.data) + assert.False(t, state.inFlight.Load()) +} + +func TestRunMissionDiscard_Error(t *testing.T) { + rec := &callbackRecorder{} + a3interface.SetCallbackSinkForTest(rec.sink) + t.Cleanup(func() { a3interface.SetCallbackSinkForTest(nil) }) + + state := newTestSaveState() + _, err := triggerMissionDiscard(state, &discardingBackend{discardErr: errors.New("server gone")}) + require.NoError(t, err) + + cb := rec.wait(t, ":MISSION:DISCARDED:") + require.Len(t, cb.data, 2) + assert.Equal(t, `"error"`, cb.data[0]) + assert.Contains(t, cb.data[1], "server gone") +} + +func TestRunMissionDiscard_PanicIsRecovered(t *testing.T) { + rec := &callbackRecorder{} + a3interface.SetCallbackSinkForTest(rec.sink) + t.Cleanup(func() { a3interface.SetCallbackSinkForTest(nil) }) + + state := newTestSaveState() + _, err := triggerMissionDiscard(state, &discardingBackend{discardPanic: true}) + require.NoError(t, err) + + cb := rec.wait(t, ":MISSION:DISCARDED:") + require.Len(t, cb.data, 2) + assert.Equal(t, `"error"`, cb.data[0]) + assert.Contains(t, cb.data[1], "panic") + assert.False(t, state.inFlight.Load(), "inFlight must be cleared even after panic") +} + +func TestRunMissionDiscard_RejectedWhileSaving(t *testing.T) { + rec := &callbackRecorder{} + a3interface.SetCallbackSinkForTest(rec.sink) + t.Cleanup(func() { a3interface.SetCallbackSinkForTest(nil) }) + + state := newTestSaveState() + backend := &discardingBackend{fakeBackend: fakeBackend{endDuration: 200 * time.Millisecond, path: "r.json.gz"}} + + _, err := triggerMissionSave(state, backend, nil, "") + require.NoError(t, err) + _, err = triggerMissionDiscard(state, backend) + require.Error(t, err, "a mission is either saved or discarded") + assert.Contains(t, err.Error(), "save already in progress") + + rec.wait(t, ":MISSION:SAVED:") + assert.Zero(t, backend.discards) +} + +func TestTriggerMissionDiscard_NoBackend(t *testing.T) { + _, err := triggerMissionDiscard(newTestSaveState(), nil) + require.Error(t, err) +} diff --git a/internal/storage/memory/memory.go b/internal/storage/memory/memory.go index 545b003..1be5e44 100644 --- a/internal/storage/memory/memory.go +++ b/internal/storage/memory/memory.go @@ -175,6 +175,19 @@ func (b *Backend) EndMission(tag string) (retErr error) { return nil } +// DiscardMission drops the mission data without exporting it, for a +// recording the addon decided not to save. +func (b *Backend) DiscardMission() error { + b.mu.Lock() + defer b.mu.Unlock() + + b.mission = nil + b.world = nil + b.lastExportPath = "" + b.resetCollections() + return nil +} + // resetCollections clears all entity and event data. // Caller must hold b.mu.Lock. func (b *Backend) resetCollections() { diff --git a/internal/storage/memory/memory_test.go b/internal/storage/memory/memory_test.go index d5072c7..74ba407 100644 --- a/internal/storage/memory/memory_test.go +++ b/internal/storage/memory/memory_test.go @@ -2,6 +2,7 @@ package memory import ( + "os" "strings" "sync" "testing" @@ -20,6 +21,9 @@ var _ storage.Backend = (*Backend)(nil) // Verify Backend implements storage.Uploadable interface var _ storage.Uploadable = (*Backend)(nil) +// Verify Backend implements storage.Discardable interface +var _ storage.Discardable = (*Backend)(nil) + func TestNew(t *testing.T) { cfg := config.MemoryConfig{ OutputDir: "/tmp/test", @@ -896,6 +900,30 @@ func TestEndMissionWithoutStartMission(t *testing.T) { assert.Contains(t, err.Error(), "no mission to end") } +func TestDiscardMissionDropsDataWithoutExport(t *testing.T) { + tmpDir := t.TempDir() + b := New(config.MemoryConfig{OutputDir: tmpDir}, nil) + + require.NoError(t, b.StartMission(&core.Mission{MissionName: "Short", CaptureDelay: 1}, &core.World{WorldName: "Altis"})) + require.NoError(t, b.AddSoldier(&core.Soldier{ID: 1, UnitName: "Alpha 1-1"})) + require.NoError(t, b.RecordGeneralEvent(&core.GeneralEvent{Name: "test"})) + + require.NoError(t, b.DiscardMission()) + + assert.Nil(t, b.mission) + assert.Empty(t, b.soldiers) + assert.Empty(t, b.generalEvents) + assert.Empty(t, b.GetExportedFilePath()) + entries, err := os.ReadDir(tmpDir) + require.NoError(t, err) + assert.Empty(t, entries, "a discarded mission is not exported") + err = b.EndMission("") + require.Error(t, err, "nothing is left to save") + assert.Contains(t, err.Error(), "no mission to end") + + require.NoError(t, b.DiscardMission(), "discarding again is a no-op") +} + func TestGetExportMetadataWithoutStartMission(t *testing.T) { b := New(config.MemoryConfig{}, nil) diff --git a/internal/storage/storage.go b/internal/storage/storage.go index 1a3982f..8f2b13a 100644 --- a/internal/storage/storage.go +++ b/internal/storage/storage.go @@ -53,6 +53,15 @@ type Backend interface { SetFocusEnd(frame core.Frame) error } +// Discardable is an optional interface for storage backends that can drop +// the current mission instead of saving it (:MISSION:DISCARD:). Backends +// without it keep what they recorded. +type Discardable interface { + // DiscardMission ends the current mission without saving it. It is a + // no-op when no mission is being recorded. + DiscardMission() error +} + // Uploadable is an optional interface for storage backends that produce // files suitable for upload to the OCAP web frontend. type Uploadable interface { diff --git a/internal/storage/websocket/connection.go b/internal/storage/websocket/connection.go index 90e8f99..c61cda1 100644 --- a/internal/storage/websocket/connection.go +++ b/internal/storage/websocket/connection.go @@ -239,10 +239,13 @@ func (c *connection) handshake(conn *ws.Conn) (uint64, uint64, error) { return reply.Seq, epoch, nil case streaming.ServerResumeCompleted: - // The server saved the mission but its end_mission ack was lost. - c.logger.Info("WebSocket mission already saved by server") + // The server saved or discarded the mission but its end_mission or + // discard_mission ack was lost. It stands in for whichever one is + // awaited; waitAck skips the other. + c.logger.Info("WebSocket mission already finished by server") c.ackUpTo(c.log.lastSeq()) c.deliverAck(streaming.AckMessage{Type: streaming.ServerAck, For: streaming.TypeEndMission}) + c.deliverAck(streaming.AckMessage{Type: streaming.ServerAck, For: streaming.TypeDiscardMission}) return c.log.lastSeq(), epoch, nil default: // resume_unknown diff --git a/internal/storage/websocket/discard_test.go b/internal/storage/websocket/discard_test.go new file mode 100644 index 0000000..f01b9d8 --- /dev/null +++ b/internal/storage/websocket/discard_test.go @@ -0,0 +1,276 @@ +package websocket + +import ( + "log/slog" + "os" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/OCAP2/extension/v5/internal/storage" + "github.com/OCAP2/extension/v5/pkg/core" + "github.com/OCAP2/extension/v5/pkg/streaming" +) + +var _ storage.Discardable = (*Backend)(nil) + +// requireGone waits until path no longer exists. +func requireGone(t *testing.T, path string) { + t.Helper() + require.Eventually(t, func() bool { _, err := os.Stat(path); return os.IsNotExist(err) }, 5*time.Second, 10*time.Millisecond) +} + +// requireNoDiscard checks that no discard_mission reached the server. +func requireNoDiscard(t *testing.T, msgs []streaming.Envelope) { + t.Helper() + for _, env := range msgs { + require.NotEqual(t, streaming.TypeDiscardMission, env.Type) + } +} + +func TestDiscardMissionDropsRecording(t *testing.T) { + rs := newResumeServer(t) + spoolDir := t.TempDir() + b := New(recoveryConfig(rs.url(), spoolDir)) + require.NoError(t, b.Init()) + defer b.Close() + + id := startResumable(t, b) + recordStates(t, b, 1, 5) + require.NoError(t, b.DiscardMission()) + + m := rs.mission(id) + require.True(t, m.discarded) + requireContiguous(t, m, 7) // start, 5 states, discard + assert.Equal(t, streaming.TypeDiscardMission, m.received[6].Type) + assert.NoDirExists(t, filepath.Join(spoolDir, id), "a confirmed discard leaves nothing to recover") + + // The next mission records as usual. + next := startResumable(t, b) + assert.NotEqual(t, id, next) + recordStates(t, b, 1, 2) + require.NoError(t, b.EndMission("")) + assert.True(t, rs.mission(next).completed) + assert.False(t, rs.mission(next).discarded) +} + +func TestDiscardMissionWithoutMissionIsNoop(t *testing.T) { + rs := newResumeServer(t) + b := New(fastConfig(rs.url())) + require.NoError(t, b.Init()) + defer b.Close() + + require.NoError(t, b.DiscardMission(), "nothing recorded yet") + id := startResumable(t, b) + require.NoError(t, b.EndMission("")) + require.NoError(t, b.DiscardMission(), "the mission was already saved") + m := rs.mission(id) + assert.True(t, m.completed) + assert.False(t, m.discarded) + requireNoDiscard(t, m.received) +} + +func TestDiscardAckLostResolvedByResume(t *testing.T) { + rs := newResumeServer(t) + b := New(fastConfig(rs.url())) + require.NoError(t, b.Init()) + defer b.Close() + + id := startResumable(t, b) + recordStates(t, b, 1, 3) + rs.dropEnd.Store(true) // the discard is applied but its ack never arrives + + done := make(chan error, 1) + go func() { done <- b.DiscardMission() }() + select { + case err := <-done: + require.NoError(t, err, "resume_completed stands in for the lost discard_mission ack") + case <-time.After(5 * time.Second): + t.Fatal("DiscardMission did not complete") + } + assert.True(t, rs.mission(id).discarded) + assert.Equal(t, int32(1), rs.starts.Load(), "the discarded mission is not started again") +} + +func TestDiscardMissionOnServerWithoutDiscard(t *testing.T) { + rs := newResumeServer(t) + rs.noDrop.Store(true) + spoolDir := t.TempDir() + b := New(recoveryConfig(rs.url(), spoolDir)) + require.NoError(t, b.Init()) + defer b.Close() + + id := startResumable(t, b) + recordStates(t, b, 1, 3) + require.Eventually(t, func() bool { return rs.lastSeq(id) == 4 }, 2*time.Second, 5*time.Millisecond) + + require.ErrorIs(t, b.DiscardMission(), errDiscardUnsupported) + m := rs.mission(id) + assert.False(t, m.completed, "the server keeps the mission until its grace expires") + requireNoDiscard(t, m.received) + assert.NoDirExists(t, filepath.Join(spoolDir, id), "recovery must not resend a discarded mission") + require.Eventually(t, func() bool { + rs.mu.Lock() + defer rs.mu.Unlock() + return len(rs.conns) == 0 + }, 2*time.Second, 10*time.Millisecond, "the connection is closed so the server stops waiting") + + // The next mission dials a fresh connection. + next := startResumable(t, b) + require.NoError(t, b.EndMission("")) + assert.True(t, rs.mission(next).completed) +} + +func TestDiscardMissionOnLegacyServer(t *testing.T) { + srv, ml := testServer(t) + defer srv.Close() + b := New(Config{URL: wsURL(srv), Secret: "s"}) + require.NoError(t, b.Init()) + defer b.Close() + + require.NoError(t, b.StartMission(&core.Mission{MissionName: "M"}, &core.World{WorldName: "W"})) + require.ErrorIs(t, b.DiscardMission(), errDiscardUnsupported) + requireNoDiscard(t, ml.all()) +} + +// discardUnconfirmed discards the mission while discard_mission is lost in +// transit, so the discard times out and its spool stays behind. +func discardUnconfirmed(t *testing.T, rs *resumeServer, spoolDir string) (*Backend, string) { + t.Helper() + cfg := recoveryConfig(rs.url(), spoolDir) + cfg.SaveTimeout = 200 * time.Millisecond // below the ack stall, so no reconnect resends it + cfg.RecoveryInterval = time.Hour // recovery runs only when the test triggers it + b := New(cfg) + require.NoError(t, b.Init()) + t.Cleanup(func() { _ = b.Close() }) + + id := startResumable(t, b) + recordStates(t, b, 1, 3) + require.Eventually(t, func() bool { return rs.lastSeq(id) == 4 }, 2*time.Second, 5*time.Millisecond) + rs.discard.Store(true) + require.Error(t, b.DiscardMission()) + rs.discard.Store(false) + rs.kill() + require.DirExists(t, filepath.Join(spoolDir, id), "the spool is kept to finish the discard") + return b, id +} + +func TestRecoveryFinishesUnconfirmedDiscard(t *testing.T) { + fastRecovery(t) + rs := newResumeServer(t) + spoolDir := t.TempDir() + b, id := discardUnconfirmed(t, rs, spoolDir) + + b.recoverSpools(b.done) + + m := rs.mission(id) + require.True(t, m.discarded) + requireContiguous(t, m, 5) // nothing is resent: start, 3 states, discard + assert.Zero(t, m.dups) + assert.NoDirExists(t, filepath.Join(spoolDir, id)) +} + +func TestRecoveryOfDiscardTheServerLost(t *testing.T) { + fastRecovery(t) + rs := newResumeServer(t) + spoolDir := t.TempDir() + b, id := discardUnconfirmed(t, rs, spoolDir) + rs.forget() // the web server restarted and holds nothing to discard + + b.recoverSpools(b.done) + + assert.Empty(t, rs.mission(id).received, "the mission is not resent to be discarded") + assert.Equal(t, int32(1), rs.starts.Load()) + assert.NoDirExists(t, filepath.Join(spoolDir, id)) +} + +// writeDiscardedSpool writes a closed spool holding a start_mission, n +// soldier states and a discard_mission. +func writeDiscardedSpool(t *testing.T, dir, missionID, producerID string, n int) { + t.Helper() + sp, err := openSpool(dir, 0, slog.Default()) + require.NoError(t, err) + start, err := marshalEnvelope(streaming.TypeStartMission, streaming.StartMissionPayload{ + Mission: &core.Mission{MissionName: "Discarded", CaptureDelay: 1}, World: &core.World{WorldName: "altis"}, + MissionID: missionID, ProducerID: producerID, ProtocolVersion: streaming.ProtocolVersionDiscard, + }, 1) + require.NoError(t, err) + require.NoError(t, sp.append(1, start)) + for i := 1; i <= n; i++ { + data, err := marshalEnvelope(streaming.TypeSoldierState, &core.SoldierState{SoldierID: 1, CaptureFrame: core.Frame(i)}, uint64(i+1)) + require.NoError(t, err) + require.NoError(t, sp.append(uint64(i+1), data)) + } + discard, err := marshalEnvelope(streaming.TypeDiscardMission, nil, uint64(n+2)) + require.NoError(t, err) + require.NoError(t, sp.append(uint64(n+2), discard)) + require.NoError(t, sp.close()) +} + +func TestRecoveryDropsDiscardForServerWithoutResume(t *testing.T) { + fastRecovery(t) + oldHandshake := handshakeTimeout + handshakeTimeout = 100 * time.Millisecond + t.Cleanup(func() { handshakeTimeout = oldHandshake }) + + rs := newResumeServer(t) + rs.legacy.Store(true) + spoolDir := t.TempDir() + writeDiscardedSpool(t, filepath.Join(spoolDir, "m-dropped"), "m-dropped", "p-old", 3) + + b := New(recoveryConfig(rs.url(), spoolDir)) + require.NoError(t, b.Init()) + defer b.Close() + + requireGone(t, filepath.Join(spoolDir, "m-dropped")) + assert.Empty(t, rs.mission("m-dropped").received) + assert.Zero(t, b.recoveryPausedUntil.Load(), "a discard does not pause recovery of other missions") +} + +func TestSpooledDiscard(t *testing.T) { + oldSegment := spoolSegmentBytes + spoolSegmentBytes = 256 // a few records per segment + t.Cleanup(func() { spoolSegmentBytes = oldSegment }) + + dir := filepath.Join(t.TempDir(), "m") + writeTestSpool(t, dir, "m", "p", 10) + paths, _, err := spoolSegments(dir) + require.NoError(t, err) + require.Greater(t, len(paths), 1, "the spool spans several segments") + seq, err := spooledDiscard(paths) + require.NoError(t, err) + assert.Zero(t, seq, "a spool ending with mission data was not discarded") + + discarded := filepath.Join(t.TempDir(), "d") + writeDiscardedSpool(t, discarded, "d", "p", 10) + paths, _, err = spoolSegments(discarded) + require.NoError(t, err) + seq, err = spooledDiscard(paths) + require.NoError(t, err) + assert.Equal(t, uint64(12), seq) +} + +func TestAbandonedMissionIsRecoveredWhileProcessRuns(t *testing.T) { + fastRecovery(t) + rs := newResumeServer(t) + spoolDir := t.TempDir() + b := New(recoveryConfig(rs.url(), spoolDir)) + require.NoError(t, b.Init()) + defer b.Close() + + first := startResumable(t, b) + recordStates(t, b, 1, 3) + require.Eventually(t, func() bool { return rs.lastSeq(first) == 4 }, 2*time.Second, 5*time.Millisecond) + + // The game moved on to the next mission without saving or discarding. + second := startResumable(t, b) + require.Eventually(t, func() bool { return rs.mission(first).completed }, 5*time.Second, 10*time.Millisecond, + "the abandoned spool is released for recovery") + assert.True(t, endPayload(t, rs.mission(first)).Interrupted) + requireGone(t, filepath.Join(spoolDir, first)) + assert.DirExists(t, filepath.Join(spoolDir, second), "the live mission is left alone") + require.NoError(t, b.EndMission("")) +} diff --git a/internal/storage/websocket/recover.go b/internal/storage/websocket/recover.go index f8b34fc..c4918da 100644 --- a/internal/storage/websocket/recover.go +++ b/internal/storage/websocket/recover.go @@ -9,6 +9,7 @@ import ( "io" "os" "path/filepath" + "slices" "sort" "strings" "time" @@ -37,6 +38,9 @@ var ( // errAlreadySaved means the server saved the mission before; the spool // is a leftover and is deleted. errAlreadySaved = errors.New("mission already saved") + // errDiscardFinished means the spool held a mission the addon discarded + // and the server no longer keeps it; the spool is deleted. + errDiscardFinished = errors.New("mission discard finished") ) // recoveryLoop resends missions left in the spool directory: by a game @@ -110,6 +114,9 @@ func (b *Backend) recoverSpools(done <-chan struct{}) { case errors.Is(err, errAlreadySaved): b.logger().Info("WebSocket spooled mission was already saved by the server; spool removed", "dir", dir) _ = os.RemoveAll(dir) + case errors.Is(err, errDiscardFinished): + b.logger().Info("WebSocket discarded mission finished from spool; spool removed", "dir", dir) + _ = os.RemoveAll(dir) case errors.Is(err, errSpoolDiscarded): _ = os.RemoveAll(dir) case errors.Is(err, errRecoveryUnsupported): @@ -225,6 +232,13 @@ func (b *Backend) recoverMission(dir string, done <-chan struct{}) error { b.logger().Warn("WebSocket spool does not start with a resumable mission; discarding", "dir", dir) return errSpoolDiscarded } + // A mission the addon discarded is not resent: the server only needs to + // hear the discard it did not confirm. + discardSeq, err := spooledDiscard(paths) + if err != nil { + return fmt.Errorf("read end: %w", err) + } + discarding := discardSeq > 0 c := newConnection(b.logger(), b.cfg.connOptions()) c.wsURL, c.secret = b.cfg.URL, b.cfg.Secret @@ -265,12 +279,21 @@ func (b *Backend) recoverMission(dir string, done <-chan struct{}) error { if err != nil { var netErr interface{ Timeout() bool } if errors.As(err, &netErr) && netErr.Timeout() { + if discarding { + return errSpoolDiscarded // an older server cannot discard it either + } return errRecoveryUnsupported // an older server ignores resume } return fmt.Errorf("resume: %w", err) } _ = conn.SetReadDeadline(time.Time{}) + // resume_completed: the server confirmed the discard before; + // resume_unknown: it no longer holds the mission. Neither needs more. + if discarding && reply.Type != streaming.ServerResumeOK { + return errDiscardFinished + } + from := uint64(1) // resume_unknown: the whole mission, start included switch reply.Type { case streaming.ServerResumeCompleted: @@ -278,26 +301,47 @@ func (b *Backend) recoverMission(dir string, done <-chan struct{}) error { case streaming.ServerResumeOK: from = reply.Seq + 1 } - b.logger().Info("WebSocket recovering mission", "missionId", payload.MissionID, "resumeFrom", from, "serverHasIt", reply.Type == streaming.ServerResumeOK) // Read acknowledgements concurrently: the server acknowledges as it goes, // and closes the connection if it cannot save the mission. + ackFor := streaming.TypeEndMission + if discarding { + ackFor = streaming.TypeDiscardMission + } ended := make(chan error, 1) go func() { for { _, raw, err := conn.ReadMessage() if err != nil { - ended <- fmt.Errorf("connection lost before the mission was saved: %w", err) + ended <- fmt.Errorf("connection lost before the mission was finished: %w", err) return } var m streaming.AckMessage - if json.Unmarshal(raw, &m) == nil && m.Type == streaming.ServerAck && m.For == streaming.TypeEndMission { + if json.Unmarshal(raw, &m) == nil && m.Type == streaming.ServerAck && m.For == ackFor { ended <- nil return } } }() + if discarding { + b.logger().Info("WebSocket finishing a discarded mission", "missionId", payload.MissionID) + // Numbered after everything the server holds, so it is not taken + // for a duplicate. + discard, err := marshalEnvelope(streaming.TypeDiscardMission, nil, max(discardSeq, reply.Seq+1)) + if err != nil { + return err + } + if err := writeData(conn, discard); err != nil { + return err + } + if err := b.awaitRecovered(ended, done); err != nil { + return err + } + return errDiscardFinished + } + b.logger().Info("WebSocket recovering mission", "missionId", payload.MissionID, "resumeFrom", from, "serverHasIt", reply.Type == streaming.ServerResumeOK) + var lastSeq uint64 var lastData []byte pace := newPacer(b.cfg.connOptions().withDefaults().resendRate) @@ -330,17 +374,43 @@ func (b *Backend) recoverMission(dir string, done <-chan struct{}) error { return err } } + return b.awaitRecovered(ended, done) +} - timeout := b.cfg.SaveTimeout - if timeout <= 0 { - timeout = 10 * time.Minute - } +// awaitRecovered waits for the server to acknowledge the recovered mission's +// end or discard. +func (b *Backend) awaitRecovered(ended <-chan error, done <-chan struct{}) error { select { case err := <-ended: return err - case <-time.After(timeout): - return fmt.Errorf("timeout waiting for the server to save the recovered mission") + case <-time.After(b.saveTimeout()): + return fmt.Errorf("timeout waiting for the server to finish the recovered mission") case <-done: return errConnectionClosed } } + +// spooledDiscard returns the sequence number of the spool's discard_mission +// record, or 0 when the spool does not end with one. +func spooledDiscard(paths []string) (uint64, error) { + // Only the last record matters; a rotated segment may still be empty. + for _, path := range slices.Backward(paths) { + var lastSeq uint64 + var lastData []byte + if err := scanSegment(path, func(seq uint64, data []byte) error { + lastSeq, lastData = seq, data + return nil + }); err != nil { + return 0, err + } + if lastData == nil { + continue + } + var last streaming.Envelope + if json.Unmarshal(lastData, &last) != nil || last.Type != streaming.TypeDiscardMission { + return 0, nil + } + return lastSeq, nil + } + return 0, nil +} diff --git a/internal/storage/websocket/resume_test.go b/internal/storage/websocket/resume_test.go index 34f9683..06a7858 100644 --- a/internal/storage/websocket/resume_test.go +++ b/internal/storage/websocket/resume_test.go @@ -32,6 +32,7 @@ type resumeServer struct { noAck atomic.Bool // never send cumulative acks dropEnd atomic.Bool // save end_mission, then close without acknowledging it legacy atomic.Bool // behave like a web server without resume support + noDrop atomic.Bool // support resume but predate discard_mission resumes atomic.Int32 starts atomic.Int32 conns2ndP atomic.Int32 // connections that began with resume @@ -44,6 +45,7 @@ type fakeMission struct { dups int gaps int completed bool + discarded bool // finished by discard_mission instead of end_mission } func newResumeServer(t *testing.T) *resumeServer { @@ -110,7 +112,10 @@ func (rs *resumeServer) serve(c *ws.Conn) { m.received = append(m.received, env) rs.conns[c] = p.MissionID rs.mu.Unlock() - ack := streaming.AckMessage{Type: streaming.ServerAck, For: env.Type, Seq: env.Seq, ProtocolVersion: streaming.ProtocolVersionResume} + ack := streaming.AckMessage{Type: streaming.ServerAck, For: env.Type, Seq: env.Seq, ProtocolVersion: streaming.ProtocolVersionDiscard} + if rs.noDrop.Load() { + ack.ProtocolVersion = streaming.ProtocolVersionResume + } if rs.legacy.Load() { ack = streaming.AckMessage{Type: streaming.ServerAck, For: env.Type} } @@ -159,12 +164,14 @@ func (rs *resumeServer) serve(c *ws.Conn) { m.lastSeq = env.Seq m.received = append(m.received, env) last := m.lastSeq - if env.Type == streaming.TypeEndMission { + finished := env.Type == streaming.TypeEndMission || env.Type == streaming.TypeDiscardMission + if finished { m.completed = true + m.discarded = env.Type == streaming.TypeDiscardMission } rs.mu.Unlock() - if env.Type == streaming.TypeEndMission { + if finished { if rs.dropEnd.Load() { return // saved, but the acknowledgement never arrives } diff --git a/internal/storage/websocket/web_integration_test.go b/internal/storage/websocket/web_integration_test.go index 2d22cb7..6217284 100644 --- a/internal/storage/websocket/web_integration_test.go +++ b/internal/storage/websocket/web_integration_test.go @@ -89,3 +89,39 @@ func TestWebIngestionIntegration(t *testing.T) { } t.Fatal("web server did not publish the acknowledged mission") } + +// A discarded mission must leave no operation behind on the web server. +func TestWebDiscardIntegration(t *testing.T) { + base := os.Getenv("OCAP_TEST_WEB_URL") + if base == "" { + t.Skip("set OCAP_TEST_WEB_URL to a disposable local web server") + } + secret := os.Getenv("OCAP_TEST_WEB_SECRET") + b := New(Config{URL: strings.Replace(base, "http", "ws", 1) + "/api/v1/stream", Secret: secret}) + require.NoError(t, b.Init()) + defer b.Close() + name := fmt.Sprintf("Proxy Discard %d", time.Now().UnixNano()) + require.NoError(t, b.StartMission(&core.Mission{MissionName: name, CaptureDelay: 1}, &core.World{WorldName: "altis"})) + require.NoError(t, b.AddSoldier(&core.Soldier{ID: 1, JoinFrame: 1, UnitName: "Proxy Player", IsPlayer: true, Side: "WEST", ClassName: "B_Soldier_F"})) + require.NoError(t, b.RecordSoldierState(&core.SoldierState{SoldierID: 1, CaptureFrame: 1, Lifestate: 1, IsPlayer: true, UnitName: "Proxy Player", Side: "WEST"})) + require.NoError(t, b.DiscardMission()) + // The next mission still records. + require.NoError(t, b.StartMission(&core.Mission{MissionName: name + " kept", CaptureDelay: 1}, &core.World{WorldName: "altis"})) + require.NoError(t, b.RecordTelemetryEvent(&core.TelemetryEvent{CaptureFrame: 1, FpsAverage: 50})) + require.NoError(t, b.EndMission("")) + + resp, err := http.Get(base + "/api/v1/operations") + require.NoError(t, err) + defer resp.Body.Close() + require.Equal(t, http.StatusOK, resp.StatusCode) + var ops []struct { + Name string `json:"mission_name"` + } + require.NoError(t, json.NewDecoder(resp.Body).Decode(&ops)) + kept := false + for _, op := range ops { + require.NotEqual(t, name, op.Name, "the discarded mission was published") + kept = kept || op.Name == name+" kept" + } + require.True(t, kept, "web server did not publish the mission after the discarded one") +} diff --git a/internal/storage/websocket/websocket.go b/internal/storage/websocket/websocket.go index 19e1967..f174161 100644 --- a/internal/storage/websocket/websocket.go +++ b/internal/storage/websocket/websocket.go @@ -3,6 +3,7 @@ package websocket import ( "crypto/rand" "encoding/json" + "errors" "fmt" "log/slog" "path/filepath" @@ -61,6 +62,7 @@ type Backend struct { mu sync.Mutex restart bool active bool // a mission has started and not ended + discardable bool // the server acknowledged start with discard support conn *connection cfg Config producerID string // identifies this extension process to the server @@ -226,14 +228,26 @@ func (b *Backend) StartMission(mission *core.Mission, world *core.World) error { b.restart = false } conn := b.conn + abandoned := b.active // the previous mission was neither saved nor discarded b.missionID = newID() missionID := b.missionID + b.discardable = false b.mu.Unlock() + // Its spool stays on disk, complete, for recovery to end as interrupted. + // Closing it releases the lock that would otherwise keep recovery away + // until the process exits. + if abandoned { + if old := conn.log.currentSpool(); old != nil { + _ = old.close() + slog.Default().Warn("WebSocket mission abandoned without save or discard; spool kept for recovery", "dir", old.dir) + } + } + base := uint8(1) raw, err := marshalPayload(streaming.TypeStartMission, streaming.StartMissionPayload{ Mission: mission, World: world, FrameBase: &base, - MissionID: missionID, ProducerID: b.producerID, ProtocolVersion: streaming.ProtocolVersionResume, + MissionID: missionID, ProducerID: b.producerID, ProtocolVersion: streaming.ProtocolVersionDiscard, }) if err != nil { return err @@ -274,7 +288,11 @@ func (b *Backend) StartMission(mission *core.Mission, world *core.World) error { conn.ackUpTo(max(ack.Seq, 1)) b.resumeRecovery() } - slog.Default().Info("WebSocket mission started", "missionId", missionID, "resumable", resumable) + discardable := ack.ProtocolVersion >= streaming.ProtocolVersionDiscard + b.mu.Lock() + b.discardable = discardable + b.mu.Unlock() + slog.Default().Info("WebSocket mission started", "missionId", missionID, "resumable", resumable, "discardable", discardable) return nil } @@ -290,36 +308,105 @@ func (b *Backend) EndMission(tag string) error { if tag != "" { payload = streaming.EndMissionPayload{Tag: tag} } - timeout := b.cfg.SaveTimeout - if timeout <= 0 { - timeout = 10 * time.Minute - } - err := b.sendEnvelopeAndWait(streaming.TypeEndMission, payload, timeout) + err := b.sendEnvelopeAndWait(streaming.TypeEndMission, payload, b.saveTimeout()) // Clear mission state regardless of error. + sp := b.clearMission() + + // The spool is needed until the server has saved the mission. + if sp != nil { + if err == nil { + removeSpool(sp) + } else { + _ = sp.close() + slog.Default().Warn("WebSocket mission not confirmed by the server; spool kept", "dir", sp.dir, "error", err) + } + } + + return err +} + +// errDiscardUnsupported means the server predates discard_mission. +var errDiscardUnsupported = errors.New("websocket server cannot discard missions") + +// DiscardMission ends the mission without saving it: it sends +// discard_mission, and the server drops the recording. A server older than +// ProtocolVersionDiscard cannot drop it. The mission is then abandoned: the +// connection is closed and nothing is kept for recovery, but that server +// still saves what it received as partial, as it did before discard existed. +func (b *Backend) DiscardMission() error { + b.missionMu.Lock() + defer b.missionMu.Unlock() + + b.mu.Lock() + active, discardable, missionID := b.active, b.discardable, b.missionID + b.mu.Unlock() + if !active { + return nil + } + + err := errDiscardUnsupported + if discardable { + err = b.sendEnvelopeAndWait(streaming.TypeDiscardMission, nil, b.saveTimeout()) + } + + // A spool that holds discard_mission lets recovery finish the discard. + // Without it, recovery would resend the mission and the server save it. + sp := b.clearMission() + keep := err != nil && discardable && sp != nil && sp.healthy() + if sp != nil { + if keep { + _ = sp.close() + } else { + removeSpool(sp) + } + } + + switch { + case err == nil: + slog.Default().Info("WebSocket mission discarded", "missionId", missionID) + case !discardable: + b.mu.Lock() + conn := b.conn + b.mu.Unlock() + _ = conn.close() // the server stops waiting for the mission; StartMission dials anew + slog.Default().Warn("WebSocket server cannot discard missions; it saves what it received as partial", "missionId", missionID) + case keep: + slog.Default().Warn("WebSocket mission discard not confirmed by the server; spool kept to finish it", "missionId", missionID, "dir", sp.dir, "error", err) + default: + slog.Default().Warn("WebSocket mission discard not confirmed by the server", "missionId", missionID, "error", err) + } + return err +} + +// clearMission forgets the current mission after end_mission or +// discard_mission and returns its spool, if any, for the caller to keep or +// remove. +func (b *Backend) clearMission() *spool { b.mu.Lock() conn := b.conn b.restart = true b.active = false + b.discardable = false b.missionID = "" b.mu.Unlock() sp := conn.log.currentSpool() conn.endMission() b.nextMarkerID.Store(0) + return sp +} - // The spool is needed until the server has saved the mission. - if sp != nil { - if err == nil { - if rerr := sp.remove(); rerr != nil { - slog.Default().Warn("WebSocket spool not removed", "dir", sp.dir, "error", rerr) - } - } else { - _ = sp.close() - slog.Default().Warn("WebSocket mission not confirmed by the server; spool kept", "dir", sp.dir, "error", err) - } +func (b *Backend) saveTimeout() time.Duration { + if b.cfg.SaveTimeout <= 0 { + return 10 * time.Minute } + return b.cfg.SaveTimeout +} - return err +func removeSpool(sp *spool) { + if err := sp.remove(); err != nil { + slog.Default().Warn("WebSocket spool not removed", "dir", sp.dir, "error", err) + } } func (b *Backend) AddSoldier(s *core.Soldier) error { diff --git a/internal/storage/websocket/websocket_test.go b/internal/storage/websocket/websocket_test.go index 0a88b63..5a70fbf 100644 --- a/internal/storage/websocket/websocket_test.go +++ b/internal/storage/websocket/websocket_test.go @@ -463,7 +463,7 @@ func TestStartMissionRegistersStart(t *testing.T) { require.NoError(t, json.Unmarshal(msgs[0].Payload, &payload)) assert.NotEmpty(t, payload.MissionID) assert.Equal(t, b.producerID, payload.ProducerID) - assert.Equal(t, uint8(streaming.ProtocolVersionResume), payload.ProtocolVersion) + assert.Equal(t, uint8(streaming.ProtocolVersionDiscard), payload.ProtocolVersion) require.NoError(t, b.EndMission("")) diff --git a/pkg/streaming/messages.go b/pkg/streaming/messages.go index 31c7e33..e0109fb 100644 --- a/pkg/streaming/messages.go +++ b/pkg/streaming/messages.go @@ -63,6 +63,19 @@ type Envelope struct { // envelopes, cumulative acknowledgements and mission resume. const ProtocolVersionResume = 2 +// ProtocolVersionDiscard is the first protocol version whose servers accept +// discard_mission. +const ProtocolVersionDiscard = 3 + +// TypeDiscardMission ends a mission without saving it, in place of +// end_mission: the addon decided the recording does not meet its save limits, +// for example the minimum mission duration. The server drops the recording +// and acknowledges it like end_mission. It is numbered like any mission +// envelope, and is sent only to servers that acknowledged start_mission with +// ProtocolVersionDiscard or later. Saved missions never carry it, so it is +// not part of AllMessageTypes. +const TypeDiscardMission = "discard_mission" + // Connection control messages. They are not mission data, so they are not // part of AllMessageTypes. const ( From 8cb43bf1c624c310d807c5359533568f090e55f9 Mon Sep 17 00:00:00 2001 From: Smith <13200576+smitt14ua@users.noreply.github.com> Date: Mon, 26 Oct 2026 23:40:41 +0200 Subject: [PATCH 02/10] fix(websocket): finish discards the server saved as partial, keep live missions safe from recovery Review fixes: - discard_mission carries the mission and producer ids. When recovery of an unconfirmed discard gets resume_unknown from a protocol 3 server, it sends the discard without a session, so the server drops a partial recording it saved after the producer went away. Older servers are not sent it. - A spool whose resume handshake timed out is kept and recovery paused, instead of deleting a discard the server never heard. - Recovery does not resend this process' own mission in full while it records another: the server would take that start for the producer's next mission and save the live one as partial. It waits until the mission ends. - A server that cannot discard reports "kept" (storage.ErrMissionKept) instead of "error". - :MISSION:START: answers :MISSION:FAILED: when the mission does not start, so the addon stops waiting for it. - memory DiscardMission is a no-op without a mission; the discard worker flushes OTel like the save worker; README recovery notes updated. --- README.md | 19 ++++-- cmd/ocap_recorder/main.go | 14 +++- cmd/ocap_recorder/main_test.go | 35 ++++++++++ cmd/ocap_recorder/mission_save.go | 73 ++++++++++++--------- cmd/ocap_recorder/mission_save_test.go | 18 ++++- internal/storage/memory/memory.go | 3 + internal/storage/memory/memory_test.go | 11 ++++ internal/storage/storage.go | 13 +++- internal/storage/websocket/discard_test.go | 76 ++++++++++++++++++---- internal/storage/websocket/recover.go | 63 ++++++++++++------ internal/storage/websocket/resume_test.go | 54 ++++++++++++++- internal/storage/websocket/websocket.go | 20 ++++-- pkg/streaming/messages.go | 13 +++- 13 files changed, 328 insertions(+), 84 deletions(-) create mode 100644 cmd/ocap_recorder/main_test.go diff --git a/README.md b/README.md index eb8a35f..de63ebb 100644 --- a/README.md +++ b/README.md @@ -153,17 +153,22 @@ paced. `0` disables pacing. per mission (0 = unlimited). Once the limit is reached, spooling stops for that mission, and a later full resend is partial. -The spool also survives the game server. If it crashes mid-mission, or a save -is not confirmed, the mission stays in `spoolDir`. The next extension process -checks `spoolDir` at startup and then every minute, resends such missions to -the web service, and deletes each spool once the web service has saved its -mission: +The spool also survives the game server. If it crashes mid-mission, or a +mission ends without a confirmed save, the mission stays in `spoolDir`. The +extension checks `spoolDir` at startup and then every minute, resends such +missions to the web service, and deletes each spool once the web service has +saved its mission. The running process recovers its own missions too, but +resends one in full only while it is not recording another, since the web +service would take that start for its next mission: - A mission cut short by a crash is ended on the crashed process' behalf and saved as `partial`, with everything recorded up to the last flush (at most about a second is lost). +- A mission the game moved on from without saving or discarding it is ended + the same way, as `partial`. - A mission whose save was not confirmed is saved as recorded. - A mission whose discard was not confirmed is not resent; the web service is - only told again to drop it. + only told again to drop it, including any `partial` recording it saved of + the mission in the meantime. When the addon ends a recording that does not meet its save limits (for example one shorter than `OCAP_settings_minMissionTime`), it sends @@ -274,7 +279,7 @@ All commands follow a `:RESOURCE:ACTION:` naming convention. New commands must u |---------|---------| | `:SYS:INIT:` | Initialize extension, send `:SYS:READY:` callback | | `:STORAGE:INIT:` | Initialize storage backend, ungate buffered handlers | -| `:MISSION:START:` | Start recording mission | +| `:MISSION:START:` | Start recording mission; answers with a `:MISSION:OK:` callback, or `:MISSION:FAILED:` with the error | | `:MISSION:SAVE:` | End recording, flush data, upload if configured | | `:MISSION:DISCARD:` | End recording without saving it (`memory`, `websocket`); answers with a `:MISSION:DISCARDED:` callback | | `:MISSION:FOCUS_START:` | Set playback focus start frame | diff --git a/cmd/ocap_recorder/main.go b/cmd/ocap_recorder/main.go index ac5fd98..21bb588 100644 --- a/cmd/ocap_recorder/main.go +++ b/cmd/ocap_recorder/main.go @@ -312,9 +312,21 @@ func initAPIClient() { // handleNewMission handles the :MISSION:START: command: parses mission data, // delegates DB persistence to the storage backend, and sets runtime state. +// It answers with a :MISSION:OK: callback, or :MISSION:FAILED: with the error, +// so the addon never waits for a mission that did not start. func handleNewMission(e dispatcher.Event) (any, error) { + result, err := startNewMission(e) + if err != nil { + if cbErr := a3interface.WriteArmaCallback(ExtensionName, ":MISSION:FAILED:", err.Error()); cbErr != nil { + Logger.Warn("Failed to send MISSION:FAILED callback", "error", cbErr) + } + } + return result, err +} + +func startNewMission(e dispatcher.Event) (any, error) { if parserService == nil { - return nil, nil + return nil, fmt.Errorf("extension not initialized") } // 1. Parse (returns core types) diff --git a/cmd/ocap_recorder/main_test.go b/cmd/ocap_recorder/main_test.go new file mode 100644 index 0000000..a12a7a9 --- /dev/null +++ b/cmd/ocap_recorder/main_test.go @@ -0,0 +1,35 @@ +package main + +import ( + "log/slog" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/OCAP2/extension/v5/internal/dispatcher" + "github.com/OCAP2/extension/v5/internal/parser" + "github.com/OCAP2/extension/v5/pkg/a3interface" +) + +func TestHandleNewMission_FailureIsReported(t *testing.T) { + rec := &callbackRecorder{} + a3interface.SetCallbackSinkForTest(rec.sink) + t.Cleanup(func() { a3interface.SetCallbackSinkForTest(nil) }) + previousLogger, previousParser := Logger, parserService + t.Cleanup(func() { Logger, parserService = previousLogger, previousParser }) + Logger = slog.Default() + + parserService = nil + _, err := handleNewMission(dispatcher.Event{}) + require.Error(t, err) + cb := rec.wait(t, ":MISSION:FAILED:") + require.Len(t, cb.data, 1) + assert.Contains(t, cb.data[0], "not initialized") + + rec.calls = nil + parserService = parser.NewParser(Logger) + _, err = handleNewMission(dispatcher.Event{Args: []string{"not json"}}) + require.Error(t, err) + rec.wait(t, ":MISSION:FAILED:") +} diff --git a/cmd/ocap_recorder/mission_save.go b/cmd/ocap_recorder/mission_save.go index 3a90ead..23e415f 100644 --- a/cmd/ocap_recorder/mission_save.go +++ b/cmd/ocap_recorder/mission_save.go @@ -2,6 +2,7 @@ package main import ( "context" + "errors" "fmt" "runtime/debug" "sync/atomic" @@ -60,49 +61,64 @@ func triggerMissionDiscard(state *missionSaveState, backend storage.Backend) (an return nil, fmt.Errorf("no storage backend configured") } if !state.inFlight.CompareAndSwap(false, true) { - return nil, fmt.Errorf("save already in progress") + return nil, fmt.Errorf("mission save or discard already in progress") } go runMissionDiscardWorker(state, backend) return "queued", nil } -// runMissionDiscardWorker performs DiscardMission, recovers from panics and -// dispatches the :MISSION:DISCARDED: callback. +// runMissionDiscardWorker performs DiscardMission, recovers from panics, +// flushes OTel, and dispatches the :MISSION:DISCARDED: callback. func runMissionDiscardWorker(state *missionSaveState, backend storage.Backend) { defer state.inFlight.Store(false) - d, ok := backend.(storage.Discardable) - if !ok { + err := storage.ErrMissionKept + if d, ok := backend.(storage.Discardable); ok { + err = func() (err error) { + defer func() { + if r := recover(); r != nil { + err = fmt.Errorf("panic during mission discard: %v", r) + if Logger != nil { + Logger.Error("panic during mission discard", "panic", r, "stack", string(debug.Stack())) + } + } + }() + return d.DiscardMission() + }() + } + + flushOTel() + + switch { + case errors.Is(err, storage.ErrMissionKept): if Logger != nil { - Logger.Info("Storage backend keeps discarded missions") + Logger.Info("Storage keeps the discarded mission", "reason", err) } _ = a3interface.WriteArmaCallback(ExtensionName, ":MISSION:DISCARDED:", "kept") - return - } - - err := func() (err error) { - defer func() { - if r := recover(); r != nil { - err = fmt.Errorf("panic during mission discard: %v", r) - if Logger != nil { - Logger.Error("panic during mission discard", "panic", r, "stack", string(debug.Stack())) - } - } - }() - return d.DiscardMission() - }() - if err != nil { + case err != nil: if Logger != nil { Logger.Error("Failed to discard mission in storage backend", "error", err) } _ = a3interface.WriteArmaCallback(ExtensionName, ":MISSION:DISCARDED:", "error", err.Error()) + default: + if Logger != nil { + Logger.Info("Mission recording discarded") + } + _ = a3interface.WriteArmaCallback(ExtensionName, ":MISSION:DISCARDED:", "ok") + } +} + +// flushOTel flushes OTel data at mission end if the provider is available. +func flushOTel() { + if OTelProvider == nil { return } - if Logger != nil { - Logger.Info("Mission recording discarded") + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if err := OTelProvider.Flush(ctx); err != nil && Logger != nil { + Logger.Warn("Failed to flush OTel data", "error", err) } - _ = a3interface.WriteArmaCallback(ExtensionName, ":MISSION:DISCARDED:", "ok") } // runMissionSaveWorker performs EndMission + optional upload, recovers from @@ -171,14 +187,7 @@ func runMissionSaveWorker(state *missionSaveState, backend storage.Backend, clie } }() - // Flush OTel data if provider is available. - if OTelProvider != nil { - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - if err := OTelProvider.Flush(ctx); err != nil && Logger != nil { - Logger.Warn("Failed to flush OTel data", "error", err) - } - } + flushOTel() // Build callback payload. switch { diff --git a/cmd/ocap_recorder/mission_save_test.go b/cmd/ocap_recorder/mission_save_test.go index 0ba9f52..7bd2cd1 100644 --- a/cmd/ocap_recorder/mission_save_test.go +++ b/cmd/ocap_recorder/mission_save_test.go @@ -292,6 +292,22 @@ func TestRunMissionDiscard_BackendKeepsData(t *testing.T) { assert.False(t, state.inFlight.Load()) } +func TestRunMissionDiscard_StorageKeepsMission(t *testing.T) { + rec := &callbackRecorder{} + a3interface.SetCallbackSinkForTest(rec.sink) + t.Cleanup(func() { a3interface.SetCallbackSinkForTest(nil) }) + + // For example a websocket server too old to drop it. + state := newTestSaveState() + backend := &discardingBackend{discardErr: fmt.Errorf("old server: %w", storage.ErrMissionKept)} + _, err := triggerMissionDiscard(state, backend) + require.NoError(t, err) + + cb := rec.wait(t, ":MISSION:DISCARDED:") + assert.Equal(t, []string{`"kept"`}, cb.data) + assert.Equal(t, 1, backend.discards) +} + func TestRunMissionDiscard_Error(t *testing.T) { rec := &callbackRecorder{} a3interface.SetCallbackSinkForTest(rec.sink) @@ -335,7 +351,7 @@ func TestRunMissionDiscard_RejectedWhileSaving(t *testing.T) { require.NoError(t, err) _, err = triggerMissionDiscard(state, backend) require.Error(t, err, "a mission is either saved or discarded") - assert.Contains(t, err.Error(), "save already in progress") + assert.Contains(t, err.Error(), "already in progress") rec.wait(t, ":MISSION:SAVED:") assert.Zero(t, backend.discards) diff --git a/internal/storage/memory/memory.go b/internal/storage/memory/memory.go index 1be5e44..2f8c748 100644 --- a/internal/storage/memory/memory.go +++ b/internal/storage/memory/memory.go @@ -180,6 +180,9 @@ func (b *Backend) EndMission(tag string) (retErr error) { func (b *Backend) DiscardMission() error { b.mu.Lock() defer b.mu.Unlock() + if b.mission == nil { + return nil // nothing recording; keep the previous export path + } b.mission = nil b.world = nil diff --git a/internal/storage/memory/memory_test.go b/internal/storage/memory/memory_test.go index 74ba407..a1ee7e8 100644 --- a/internal/storage/memory/memory_test.go +++ b/internal/storage/memory/memory_test.go @@ -924,6 +924,17 @@ func TestDiscardMissionDropsDataWithoutExport(t *testing.T) { require.NoError(t, b.DiscardMission(), "discarding again is a no-op") } +func TestDiscardMissionAfterSaveKeepsExportPath(t *testing.T) { + b := New(config.MemoryConfig{OutputDir: t.TempDir()}, nil) + require.NoError(t, b.StartMission(&core.Mission{MissionName: "Saved", CaptureDelay: 1}, &core.World{WorldName: "Altis"})) + require.NoError(t, b.EndMission("")) + path := b.GetExportedFilePath() + require.NotEmpty(t, path) + + require.NoError(t, b.DiscardMission(), "nothing is recording") + assert.Equal(t, path, b.GetExportedFilePath()) +} + func TestGetExportMetadataWithoutStartMission(t *testing.T) { b := New(config.MemoryConfig{}, nil) diff --git a/internal/storage/storage.go b/internal/storage/storage.go index 8f2b13a..98011e0 100644 --- a/internal/storage/storage.go +++ b/internal/storage/storage.go @@ -1,7 +1,11 @@ // internal/storage/storage.go package storage -import "github.com/OCAP2/extension/v5/pkg/core" +import ( + "errors" + + "github.com/OCAP2/extension/v5/pkg/core" +) // Backend is the interface all storage implementations must satisfy type Backend interface { @@ -58,10 +62,15 @@ type Backend interface { // without it keep what they recorded. type Discardable interface { // DiscardMission ends the current mission without saving it. It is a - // no-op when no mission is being recorded. + // no-op when no mission is being recorded. It returns ErrMissionKept + // when the mission ended but its data could not be dropped. DiscardMission() error } +// ErrMissionKept means a discarded mission ended, but its storage keeps what +// was recorded, as a backend without Discardable does. +var ErrMissionKept = errors.New("storage keeps the discarded mission") + // Uploadable is an optional interface for storage backends that produce // files suitable for upload to the OCAP web frontend. type Uploadable interface { diff --git a/internal/storage/websocket/discard_test.go b/internal/storage/websocket/discard_test.go index f01b9d8..1d74ab1 100644 --- a/internal/storage/websocket/discard_test.go +++ b/internal/storage/websocket/discard_test.go @@ -1,6 +1,7 @@ package websocket import ( + "encoding/json" "log/slog" "os" "path/filepath" @@ -46,6 +47,9 @@ func TestDiscardMissionDropsRecording(t *testing.T) { require.True(t, m.discarded) requireContiguous(t, m, 7) // start, 5 states, discard assert.Equal(t, streaming.TypeDiscardMission, m.received[6].Type) + var p streaming.DiscardMissionPayload + require.NoError(t, json.Unmarshal(m.received[6].Payload, &p)) + assert.Equal(t, streaming.DiscardMissionPayload{MissionID: id, ProducerID: b.producerID}, p, "the discard names the mission") assert.NoDirExists(t, filepath.Join(spoolDir, id), "a confirmed discard leaves nothing to recover") // The next mission records as usual. @@ -107,7 +111,7 @@ func TestDiscardMissionOnServerWithoutDiscard(t *testing.T) { recordStates(t, b, 1, 3) require.Eventually(t, func() bool { return rs.lastSeq(id) == 4 }, 2*time.Second, 5*time.Millisecond) - require.ErrorIs(t, b.DiscardMission(), errDiscardUnsupported) + require.ErrorIs(t, b.DiscardMission(), storage.ErrMissionKept) m := rs.mission(id) assert.False(t, m.completed, "the server keeps the mission until its grace expires") requireNoDiscard(t, m.received) @@ -173,20 +177,53 @@ func TestRecoveryFinishesUnconfirmedDiscard(t *testing.T) { assert.NoDirExists(t, filepath.Join(spoolDir, id)) } -func TestRecoveryOfDiscardTheServerLost(t *testing.T) { +func TestRecoveryDropsPartialOfUnconfirmedDiscard(t *testing.T) { fastRecovery(t) rs := newResumeServer(t) spoolDir := t.TempDir() b, id := discardUnconfirmed(t, rs, spoolDir) - rs.forget() // the web server restarted and holds nothing to discard + rs.expire(id) // the producer stayed away past the grace: saved as partial + require.True(t, rs.partial(id)) b.recoverSpools(b.done) + assert.False(t, rs.partial(id), "the partial recording is dropped") + assert.Equal(t, []string{id}, rs.droppedMissions()) assert.Empty(t, rs.mission(id).received, "the mission is not resent to be discarded") assert.Equal(t, int32(1), rs.starts.Load()) assert.NoDirExists(t, filepath.Join(spoolDir, id)) } +func TestRecoveryOfDiscardTheServerLost(t *testing.T) { + fastRecovery(t) + rs := newResumeServer(t) + spoolDir := t.TempDir() + b, id := discardUnconfirmed(t, rs, spoolDir) + rs.forget() // the web server restarted + + b.recoverSpools(b.done) + + assert.Equal(t, []string{id}, rs.droppedMissions(), "the discard names the mission") + assert.Equal(t, int32(1), rs.starts.Load(), "the mission is not resent to be discarded") + assert.NoDirExists(t, filepath.Join(spoolDir, id)) +} + +func TestRecoveryOfDiscardOnServerWithoutDiscard(t *testing.T) { + fastRecovery(t) + rs := newResumeServer(t) + rs.noDrop.Store(true) // the web server was downgraded since + spoolDir := t.TempDir() + writeDiscardedSpool(t, filepath.Join(spoolDir, "m-old"), "m-old", "p-old", 3) + + b := New(recoveryConfig(rs.url(), spoolDir)) + require.NoError(t, b.Init()) + defer b.Close() + + requireGone(t, filepath.Join(spoolDir, "m-old")) + assert.Empty(t, rs.droppedMissions(), "an older server is not sent what it cannot take") + assert.Zero(t, rs.starts.Load()) +} + // writeDiscardedSpool writes a closed spool holding a start_mission, n // soldier states and a discard_mission. func writeDiscardedSpool(t *testing.T, dir, missionID, producerID string, n int) { @@ -204,30 +241,30 @@ func writeDiscardedSpool(t *testing.T, dir, missionID, producerID string, n int) require.NoError(t, err) require.NoError(t, sp.append(uint64(i+1), data)) } - discard, err := marshalEnvelope(streaming.TypeDiscardMission, nil, uint64(n+2)) + discard, err := marshalEnvelope(streaming.TypeDiscardMission, streaming.DiscardMissionPayload{MissionID: missionID, ProducerID: producerID}, uint64(n+2)) require.NoError(t, err) require.NoError(t, sp.append(uint64(n+2), discard)) require.NoError(t, sp.close()) } -func TestRecoveryDropsDiscardForServerWithoutResume(t *testing.T) { +func TestRecoveryKeepsDiscardWhenResumeTimesOut(t *testing.T) { fastRecovery(t) oldHandshake := handshakeTimeout handshakeTimeout = 100 * time.Millisecond t.Cleanup(func() { handshakeTimeout = oldHandshake }) rs := newResumeServer(t) - rs.legacy.Store(true) + rs.legacy.Store(true) // does not answer resume, like a slow server spoolDir := t.TempDir() - writeDiscardedSpool(t, filepath.Join(spoolDir, "m-dropped"), "m-dropped", "p-old", 3) + writeDiscardedSpool(t, filepath.Join(spoolDir, "m-kept"), "m-kept", "p-old", 3) b := New(recoveryConfig(rs.url(), spoolDir)) require.NoError(t, b.Init()) defer b.Close() - requireGone(t, filepath.Join(spoolDir, "m-dropped")) - assert.Empty(t, rs.mission("m-dropped").received) - assert.Zero(t, b.recoveryPausedUntil.Load(), "a discard does not pause recovery of other missions") + require.Eventually(t, func() bool { return b.recoveryPausedUntil.Load() != 0 }, 5*time.Second, 10*time.Millisecond) + assert.DirExists(t, filepath.Join(spoolDir, "m-kept"), "the discard is retried once the server answers") + assert.Empty(t, rs.mission("m-kept").received) } func TestSpooledDiscard(t *testing.T) { @@ -266,11 +303,22 @@ func TestAbandonedMissionIsRecoveredWhileProcessRuns(t *testing.T) { require.Eventually(t, func() bool { return rs.lastSeq(first) == 4 }, 2*time.Second, 5*time.Millisecond) // The game moved on to the next mission without saving or discarding. + // The server saves the abandoned one as partial. second := startResumable(t, b) - require.Eventually(t, func() bool { return rs.mission(first).completed }, 5*time.Second, 10*time.Millisecond, - "the abandoned spool is released for recovery") + require.True(t, rs.partial(first)) + + // Resending it in full would supersede the live mission: it waits. + time.Sleep(spoolStaleAfter + 10*recoveryConfig("", "").RecoveryInterval) // several recovery passes + assert.Equal(t, int32(2), rs.starts.Load(), "the abandoned mission is not resent while another records") + assert.False(t, rs.partial(second), "the live mission is left alone") + assert.DirExists(t, filepath.Join(spoolDir, first)) + + recordStates(t, b, 1, 2) + require.NoError(t, b.EndMission("")) + assert.True(t, rs.mission(second).completed) + + // Between missions it is resent and ended as interrupted. + require.Eventually(t, func() bool { return rs.mission(first).completed }, 5*time.Second, 10*time.Millisecond) assert.True(t, endPayload(t, rs.mission(first)).Interrupted) requireGone(t, filepath.Join(spoolDir, first)) - assert.DirExists(t, filepath.Join(spoolDir, second), "the live mission is left alone") - require.NoError(t, b.EndMission("")) } diff --git a/internal/storage/websocket/recover.go b/internal/storage/websocket/recover.go index c4918da..5c33913 100644 --- a/internal/storage/websocket/recover.go +++ b/internal/storage/websocket/recover.go @@ -41,6 +41,9 @@ var ( // errDiscardFinished means the spool held a mission the addon discarded // and the server no longer keeps it; the spool is deleted. errDiscardFinished = errors.New("mission discard finished") + // errRecoveryDeferred means the spooled mission is this process' own and + // must wait until it records no other mission; the spool is kept. + errRecoveryDeferred = errors.New("mission recovery deferred") ) // recoveryLoop resends missions left in the spool directory: by a game @@ -119,6 +122,8 @@ func (b *Backend) recoverSpools(done <-chan struct{}) { _ = os.RemoveAll(dir) case errors.Is(err, errSpoolDiscarded): _ = os.RemoveAll(dir) + case errors.Is(err, errRecoveryDeferred): + b.logger().Debug("WebSocket mission recovery waits until the current mission ends", "dir", dir) case errors.Is(err, errRecoveryUnsupported): // Asking an older server every minute only fills both logs. b.recoveryPausedUntil.Store(time.Now().Add(recoveryUnsupportedPause).UnixNano()) @@ -232,13 +237,6 @@ func (b *Backend) recoverMission(dir string, done <-chan struct{}) error { b.logger().Warn("WebSocket spool does not start with a resumable mission; discarding", "dir", dir) return errSpoolDiscarded } - // A mission the addon discarded is not resent: the server only needs to - // hear the discard it did not confirm. - discardSeq, err := spooledDiscard(paths) - if err != nil { - return fmt.Errorf("read end: %w", err) - } - discarding := discardSeq > 0 c := newConnection(b.logger(), b.cfg.connOptions()) c.wsURL, c.secret = b.cfg.URL, b.cfg.Secret @@ -248,6 +246,15 @@ func (b *Backend) recoverMission(dir string, done <-chan struct{}) error { } defer conn.Close() + // A mission the addon discarded is not resent: the server only needs to + // hear the discard it did not confirm. Read once the server is reachable: + // it scans the spool's last segment. + discardSeq, err := spooledDiscard(paths) + if err != nil { + return fmt.Errorf("read end: %w", err) + } + discarding := discardSeq > 0 + // Give up when the backend closes. stop := make(chan struct{}) defer close(stop) @@ -279,19 +286,25 @@ func (b *Backend) recoverMission(dir string, done <-chan struct{}) error { if err != nil { var netErr interface{ Timeout() bool } if errors.As(err, &netErr) && netErr.Timeout() { - if discarding { - return errSpoolDiscarded // an older server cannot discard it either - } - return errRecoveryUnsupported // an older server ignores resume + // An older server ignores resume. A discarded spool exists only + // for a server that could discard, so it is kept as well: the + // server is more likely slow than downgraded. + return errRecoveryUnsupported } return fmt.Errorf("resume: %w", err) } _ = conn.SetReadDeadline(time.Time{}) - // resume_completed: the server confirmed the discard before; - // resume_unknown: it no longer holds the mission. Neither needs more. - if discarding && reply.Type != streaming.ServerResumeOK { - return errDiscardFinished + if discarding { + switch { + case reply.Type == streaming.ServerResumeCompleted: + return errDiscardFinished // the server confirmed the discard before + case reply.Type == streaming.ServerResumeUnknown && reply.ProtocolVersion < streaming.ProtocolVersionDiscard: + return errDiscardFinished // nothing more an older server can do + } + // resume_ok: the server still holds the session. resume_unknown: it + // may still keep the mission, saved as partial after the producer + // went away; the discard names it. } from := uint64(1) // resume_unknown: the whole mission, start included @@ -300,6 +313,13 @@ func (b *Backend) recoverMission(dir string, done <-chan struct{}) error { return errAlreadySaved case streaming.ServerResumeOK: from = reply.Seq + 1 + case streaming.ServerResumeUnknown: + // The server takes a start as its producer's next mission and saves + // the producer's live one as partial. This process' own mission + // waits until it records none. + if !discarding && payload.ProducerID == b.producerID && b.recording() { + return errRecoveryDeferred + } } // Read acknowledgements concurrently: the server acknowledges as it goes, @@ -325,10 +345,15 @@ func (b *Backend) recoverMission(dir string, done <-chan struct{}) error { }() if discarding { - b.logger().Info("WebSocket finishing a discarded mission", "missionId", payload.MissionID) - // Numbered after everything the server holds, so it is not taken - // for a duplicate. - discard, err := marshalEnvelope(streaming.TypeDiscardMission, nil, max(discardSeq, reply.Seq+1)) + b.logger().Info("WebSocket finishing a discarded mission", "missionId", payload.MissionID, "serverHasIt", reply.Type == streaming.ServerResumeOK) + // On a resumed session, numbered after everything the server holds, + // so it is not taken for a duplicate. Without one, it is not numbered. + var seq uint64 + if reply.Type == streaming.ServerResumeOK { + seq = max(discardSeq, reply.Seq+1) + } + discardPayload := streaming.DiscardMissionPayload{MissionID: payload.MissionID, ProducerID: payload.ProducerID} + discard, err := marshalEnvelope(streaming.TypeDiscardMission, discardPayload, seq) if err != nil { return err } diff --git a/internal/storage/websocket/resume_test.go b/internal/storage/websocket/resume_test.go index 06a7858..deadf64 100644 --- a/internal/storage/websocket/resume_test.go +++ b/internal/storage/websocket/resume_test.go @@ -27,6 +27,11 @@ type resumeServer struct { mu sync.Mutex missions map[string]*fakeMission conns map[*ws.Conn]string // live connection → mission id ("" before start/resume) + // partials holds missions saved as partial once their session ended + // without end_mission: superseded by their producer's next mission, or + // expired. The server no longer resumes them. + partials map[string]*fakeMission + dropped []string // missions whose partials a session-less discard_mission dropped discard atomic.Bool // drop envelopes as if lost in transit; the link stays dead noAck atomic.Bool // never send cumulative acks @@ -50,7 +55,7 @@ type fakeMission struct { func newResumeServer(t *testing.T) *resumeServer { t.Helper() - rs := &resumeServer{t: t, missions: map[string]*fakeMission{}, conns: map[*ws.Conn]string{}} + rs := &resumeServer{t: t, missions: map[string]*fakeMission{}, conns: map[*ws.Conn]string{}, partials: map[string]*fakeMission{}} upgrader := ws.Upgrader{} rs.srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { c, err := upgrader.Upgrade(w, r, nil) @@ -102,6 +107,14 @@ func (rs *resumeServer) serve(c *ws.Conn) { require.NoError(rs.t, json.Unmarshal(env.Payload, &p)) rs.starts.Add(1) rs.mu.Lock() + // A producer records one mission at a time: its other + // unfinished missions are saved as partial. + for id, other := range rs.missions { + if id != p.MissionID && other.producer == p.ProducerID && !other.completed { + rs.partials[id] = other + delete(rs.missions, id) + } + } m = rs.missions[p.MissionID] if m == nil { m = &fakeMission{producer: p.ProducerID, lastSeq: env.Seq} @@ -144,11 +157,25 @@ func (rs *resumeServer) serve(c *ws.Conn) { m = nil } rs.mu.Unlock() + if !rs.noDrop.Load() { + reply.ProtocolVersion = streaming.ProtocolVersionDiscard + } if c.WriteJSON(reply) != nil { return } default: + if m == nil && env.Type == streaming.TypeDiscardMission && !rs.noDrop.Load() { + // No session: drop what the server kept of the mission. + var p streaming.DiscardMissionPayload + require.NoError(rs.t, json.Unmarshal(env.Payload, &p)) + rs.mu.Lock() + delete(rs.partials, p.MissionID) + rs.dropped = append(rs.dropped, p.MissionID) + rs.mu.Unlock() + _ = c.WriteJSON(streaming.AckMessage{Type: streaming.ServerAck, For: env.Type}) + return + } if m == nil { continue } @@ -214,6 +241,31 @@ func (rs *resumeServer) forget() { rs.missions = map[string]*fakeMission{} } +// expire saves an unfinished mission as partial, as the web server does when +// its producer stays away past the resume grace. +func (rs *resumeServer) expire(id string) { + rs.mu.Lock() + defer rs.mu.Unlock() + if m := rs.missions[id]; m != nil { + rs.partials[id] = m + delete(rs.missions, id) + } +} + +// partial reports whether the server keeps a partial recording of the mission. +func (rs *resumeServer) partial(id string) bool { + rs.mu.Lock() + defer rs.mu.Unlock() + return rs.partials[id] != nil +} + +// droppedMissions lists the missions session-less discards dropped. +func (rs *resumeServer) droppedMissions() []string { + rs.mu.Lock() + defer rs.mu.Unlock() + return append([]string(nil), rs.dropped...) +} + func (rs *resumeServer) mission(id string) fakeMission { rs.mu.Lock() defer rs.mu.Unlock() diff --git a/internal/storage/websocket/websocket.go b/internal/storage/websocket/websocket.go index f174161..65f648c 100644 --- a/internal/storage/websocket/websocket.go +++ b/internal/storage/websocket/websocket.go @@ -3,7 +3,6 @@ package websocket import ( "crypto/rand" "encoding/json" - "errors" "fmt" "log/slog" "path/filepath" @@ -11,6 +10,7 @@ import ( "sync/atomic" "time" + "github.com/OCAP2/extension/v5/internal/storage" "github.com/OCAP2/extension/v5/pkg/core" "github.com/OCAP2/extension/v5/pkg/streaming" ) @@ -326,14 +326,16 @@ func (b *Backend) EndMission(tag string) error { return err } -// errDiscardUnsupported means the server predates discard_mission. -var errDiscardUnsupported = errors.New("websocket server cannot discard missions") +// errDiscardUnsupported means the server predates discard_mission, or never +// acknowledged the mission start that would tell. +var errDiscardUnsupported = fmt.Errorf("websocket server cannot discard missions: %w", storage.ErrMissionKept) // DiscardMission ends the mission without saving it: it sends // discard_mission, and the server drops the recording. A server older than // ProtocolVersionDiscard cannot drop it. The mission is then abandoned: the // connection is closed and nothing is kept for recovery, but that server -// still saves what it received as partial, as it did before discard existed. +// still saves what it received as partial, as it did before discard existed, +// and DiscardMission returns storage.ErrMissionKept. func (b *Backend) DiscardMission() error { b.missionMu.Lock() defer b.missionMu.Unlock() @@ -347,7 +349,8 @@ func (b *Backend) DiscardMission() error { err := errDiscardUnsupported if discardable { - err = b.sendEnvelopeAndWait(streaming.TypeDiscardMission, nil, b.saveTimeout()) + payload := streaming.DiscardMissionPayload{MissionID: missionID, ProducerID: b.producerID} + err = b.sendEnvelopeAndWait(streaming.TypeDiscardMission, payload, b.saveTimeout()) } // A spool that holds discard_mission lets recovery finish the discard. @@ -396,6 +399,13 @@ func (b *Backend) clearMission() *spool { return sp } +// recording reports whether a mission has started and not ended. +func (b *Backend) recording() bool { + b.mu.Lock() + defer b.mu.Unlock() + return b.active +} + func (b *Backend) saveTimeout() time.Duration { if b.cfg.SaveTimeout <= 0 { return 10 * time.Minute diff --git a/pkg/streaming/messages.go b/pkg/streaming/messages.go index e0109fb..aa5a814 100644 --- a/pkg/streaming/messages.go +++ b/pkg/streaming/messages.go @@ -73,9 +73,17 @@ const ProtocolVersionDiscard = 3 // and acknowledges it like end_mission. It is numbered like any mission // envelope, and is sent only to servers that acknowledged start_mission with // ProtocolVersionDiscard or later. Saved missions never carry it, so it is -// not part of AllMessageTypes. +// not part of AllMessageTypes. Its payload is DiscardMissionPayload. const TypeDiscardMission = "discard_mission" +// DiscardMissionPayload names the discarded mission. A server that no longer +// holds the mission's session, for example because it saved it as partial +// after the producer went away, drops what it kept of the mission by id. +type DiscardMissionPayload struct { + MissionID string `json:"missionId"` + ProducerID string `json:"producerId"` +} + // Connection control messages. They are not mission data, so they are not // part of AllMessageTypes. const ( @@ -106,7 +114,8 @@ type AckMessage struct { // Seq is the highest envelope the server has received (ack, resume_ok). Seq uint64 `json:"seq,omitempty"` // ProtocolVersion is set on the start_mission acknowledgement by servers - // that support resume. Older servers omit it. + // that support resume, and on resume replies by servers that support + // discard. Older servers omit it. ProtocolVersion uint8 `json:"protocolVersion,omitempty"` } From cd71933d63fd849a6e33a32906cd115fc619f7e6 Mon Sep 17 00:00:00 2001 From: Smith <13200576+smitt14ua@users.noreply.github.com> Date: Tue, 27 Oct 2026 00:08:24 +0200 Subject: [PATCH 03/10] fix(websocket): end abandoned missions instead of recovering them, mark resent starts recovered Second review round: - An abandoned mission (the game moved on without save or discard) is now ended as interrupted before the next mission starts, on its own connection, and the next mission dials a fresh one. Closing its spool for recovery let another process sharing spoolDir resend it in full, and the server took that start for the producer's next mission and saved the live one as partial. The spool stays for recovery only if the end is not confirmed within 30 s. The own-process deferral is gone. - A start resent in full from the spool carries "recovered": true, so a protocol 3 server does not supersede the producer's live mission with it. This covers unconfirmed saves too, and siblings sharing spoolDir. - A start whose ack never came forgets the mission and removes its spool, so the next start does not end it as abandoned. - The save and discard workers clear the in-flight guard before the callback, so the addon may act on it at once. - :MISSION:START: also answers :MISSION:FAILED: when it panics. - Doc fixes: discard numbering, resume_completed, README recovery notes. - The fake server rejects a second start on a connection that feeds a mission, as the web server does, and honors recovered starts. --- README.md | 14 ++-- cmd/ocap_recorder/main.go | 17 +++-- cmd/ocap_recorder/mission_save.go | 11 ++-- internal/storage/websocket/connection.go | 5 +- internal/storage/websocket/discard_test.go | 75 +++++++++++++++++----- internal/storage/websocket/recover.go | 42 ++++++++---- internal/storage/websocket/resume_test.go | 13 +++- internal/storage/websocket/websocket.go | 37 +++++++---- pkg/streaming/messages.go | 19 ++++-- 9 files changed, 173 insertions(+), 60 deletions(-) diff --git a/README.md b/README.md index de63ebb..1ab0c83 100644 --- a/README.md +++ b/README.md @@ -157,19 +157,23 @@ The spool also survives the game server. If it crashes mid-mission, or a mission ends without a confirmed save, the mission stays in `spoolDir`. The extension checks `spoolDir` at startup and then every minute, resends such missions to the web service, and deletes each spool once the web service has -saved its mission. The running process recovers its own missions too, but -resends one in full only while it is not recording another, since the web -service would take that start for its next mission: +saved its mission. A mission resent in full is marked as recovered, so a web +service at protocol version 3 does not take it for the producer's next +mission and save the mission it records now as `partial`: - A mission cut short by a crash is ended on the crashed process' behalf and saved as `partial`, with everything recorded up to the last flush (at most about a second is lost). -- A mission the game moved on from without saving or discarding it is ended - the same way, as `partial`. - A mission whose save was not confirmed is saved as recorded. - A mission whose discard was not confirmed is not resent; the web service is only told again to drop it, including any `partial` recording it saved of the mission in the meantime. +When the game moves on to the next mission without saving or discarding the +current one, for example after an admin `#missions`, the proxy ends it as +interrupted before the next one starts, and the web service saves it as +`partial`. Only if that is not confirmed within 30 seconds does its spool stay +behind for recovery. + When the addon ends a recording that does not meet its save limits (for example one shorter than `OCAP_settings_minMissionTime`), it sends `:MISSION:DISCARD:`, and the proxy tells the web service to drop the recording diff --git a/cmd/ocap_recorder/main.go b/cmd/ocap_recorder/main.go index 21bb588..82d66e3 100644 --- a/cmd/ocap_recorder/main.go +++ b/cmd/ocap_recorder/main.go @@ -314,13 +314,22 @@ func initAPIClient() { // delegates DB persistence to the storage backend, and sets runtime state. // It answers with a :MISSION:OK: callback, or :MISSION:FAILED: with the error, // so the addon never waits for a mission that did not start. -func handleNewMission(e dispatcher.Event) (any, error) { - result, err := startNewMission(e) - if err != nil { - if cbErr := a3interface.WriteArmaCallback(ExtensionName, ":MISSION:FAILED:", err.Error()); cbErr != nil { +func handleNewMission(e dispatcher.Event) (result any, err error) { + reportFailure := func(msg string) { + if cbErr := a3interface.WriteArmaCallback(ExtensionName, ":MISSION:FAILED:", msg); cbErr != nil { Logger.Warn("Failed to send MISSION:FAILED callback", "error", cbErr) } } + defer func() { + if r := recover(); r != nil { + reportFailure(fmt.Sprintf("panic during mission start: %v", r)) + panic(r) // the dispatcher logs it + } + }() + result, err = startNewMission(e) + if err != nil { + reportFailure(err.Error()) + } return result, err } diff --git a/cmd/ocap_recorder/mission_save.go b/cmd/ocap_recorder/mission_save.go index 23e415f..1c7561b 100644 --- a/cmd/ocap_recorder/mission_save.go +++ b/cmd/ocap_recorder/mission_save.go @@ -71,8 +71,6 @@ func triggerMissionDiscard(state *missionSaveState, backend storage.Backend) (an // runMissionDiscardWorker performs DiscardMission, recovers from panics, // flushes OTel, and dispatches the :MISSION:DISCARDED: callback. func runMissionDiscardWorker(state *missionSaveState, backend storage.Backend) { - defer state.inFlight.Store(false) - err := storage.ErrMissionKept if d, ok := backend.(storage.Discardable); ok { err = func() (err error) { @@ -89,6 +87,9 @@ func runMissionDiscardWorker(state *missionSaveState, backend storage.Backend) { } flushOTel() + // Cleared before the callback: the addon may start the next mission's + // save or discard as soon as it hears this one ended. + state.inFlight.Store(false) switch { case errors.Is(err, storage.ErrMissionKept): @@ -124,9 +125,6 @@ func flushOTel() { // runMissionSaveWorker performs EndMission + optional upload, recovers from // panics, flushes OTel, and dispatches the :MISSION:SAVED: callback. func runMissionSaveWorker(state *missionSaveState, backend storage.Backend, client *api.Client, tag string) { - // Always clear the in-flight flag before returning. - defer state.inFlight.Store(false) - start := time.Now() var ( @@ -188,6 +186,9 @@ func runMissionSaveWorker(state *missionSaveState, backend storage.Backend, clie }() flushOTel() + // Cleared before the callback: the addon may start the next mission's + // save or discard as soon as it hears this one ended. + state.inFlight.Store(false) // Build callback payload. switch { diff --git a/internal/storage/websocket/connection.go b/internal/storage/websocket/connection.go index c61cda1..65852c2 100644 --- a/internal/storage/websocket/connection.go +++ b/internal/storage/websocket/connection.go @@ -25,13 +25,16 @@ const ( defaultReconnectMax = 30 * time.Second ackChSize = 16 writeWait = 10 * time.Second - ackTimeout = 10 * time.Second ) // handshakeTimeout bounds waiting for the server's answer to resume; tests // lower it. var handshakeTimeout = 10 * time.Second +// ackTimeout bounds waiting for the start_mission acknowledgement; tests +// lower it. +var ackTimeout = 10 * time.Second + var errConnectionClosed = errors.New("websocket connection closed") // connOptions tunes buffering and recovery. Zero values select the defaults. diff --git a/internal/storage/websocket/discard_test.go b/internal/storage/websocket/discard_test.go index 1d74ab1..92f7675 100644 --- a/internal/storage/websocket/discard_test.go +++ b/internal/storage/websocket/discard_test.go @@ -290,8 +290,7 @@ func TestSpooledDiscard(t *testing.T) { assert.Equal(t, uint64(12), seq) } -func TestAbandonedMissionIsRecoveredWhileProcessRuns(t *testing.T) { - fastRecovery(t) +func TestAbandonedMissionEndsAsInterrupted(t *testing.T) { rs := newResumeServer(t) spoolDir := t.TempDir() b := New(recoveryConfig(rs.url(), spoolDir)) @@ -300,25 +299,71 @@ func TestAbandonedMissionIsRecoveredWhileProcessRuns(t *testing.T) { first := startResumable(t, b) recordStates(t, b, 1, 3) - require.Eventually(t, func() bool { return rs.lastSeq(first) == 4 }, 2*time.Second, 5*time.Millisecond) // The game moved on to the next mission without saving or discarding. - // The server saves the abandoned one as partial. second := startResumable(t, b) - require.True(t, rs.partial(first)) - - // Resending it in full would supersede the live mission: it waits. - time.Sleep(spoolStaleAfter + 10*recoveryConfig("", "").RecoveryInterval) // several recovery passes - assert.Equal(t, int32(2), rs.starts.Load(), "the abandoned mission is not resent while another records") - assert.False(t, rs.partial(second), "the live mission is left alone") - assert.DirExists(t, filepath.Join(spoolDir, first)) + m := rs.mission(first) + require.True(t, m.completed, "the abandoned mission is ended before the next one starts") + assert.True(t, endPayload(t, m).Interrupted) + requireContiguous(t, m, 5) // start, 3 states, end + assert.False(t, rs.partial(first)) + assert.NoDirExists(t, filepath.Join(spoolDir, first), "nothing is left to recover") recordStates(t, b, 1, 2) require.NoError(t, b.EndMission("")) assert.True(t, rs.mission(second).completed) + assert.False(t, rs.partial(second)) + assert.Equal(t, int32(2), rs.starts.Load()) +} + +func TestRecoveredMissionDoesNotSupersedeLiveOne(t *testing.T) { + fastRecovery(t) + rs := newResumeServer(t) + spoolDir := t.TempDir() + b := New(recoveryConfig(rs.url(), spoolDir)) + require.NoError(t, b.Init()) + defer b.Close() + + live := startResumable(t, b) + recordStates(t, b, 1, 3) + // A spool of this producer the server no longer holds, such as a save it + // never confirmed before a web restart; another process sharing the + // spool directory may recover it the same way. + writeTestSpool(t, filepath.Join(spoolDir, "m-left"), "m-left", b.producerID, 4) + + require.Eventually(t, func() bool { return rs.mission("m-left").completed }, 5*time.Second, 10*time.Millisecond) + start := rs.mission("m-left").received[0] + var p streaming.StartMissionPayload + require.NoError(t, json.Unmarshal(start.Payload, &p)) + assert.True(t, p.Recovered, "the resent start is marked recovered") + assert.Equal(t, "m-left", p.MissionID) + assert.False(t, rs.partial(live), "the live mission is left alone") + + recordStates(t, b, 4, 5) + require.NoError(t, b.EndMission("")) + requireContiguous(t, rs.mission(live), 7) // start, 5 states, end +} + +func TestFailedStartLeavesNoMission(t *testing.T) { + rs := newResumeServer(t) + spoolDir := t.TempDir() + b := New(recoveryConfig(rs.url(), spoolDir)) + require.NoError(t, b.Init()) + defer b.Close() + + oldAck := ackTimeout + ackTimeout = 200 * time.Millisecond + t.Cleanup(func() { ackTimeout = oldAck }) + rs.muteStart.Store(true) + require.Error(t, b.StartMission(&core.Mission{MissionName: "M"}, &core.World{WorldName: "W"})) + assert.False(t, b.recording(), "a mission that did not start is not abandoned later") + entries, err := os.ReadDir(spoolDir) + require.NoError(t, err) + assert.Empty(t, entries, "its spool is removed") - // Between missions it is resent and ended as interrupted. - require.Eventually(t, func() bool { return rs.mission(first).completed }, 5*time.Second, 10*time.Millisecond) - assert.True(t, endPayload(t, rs.mission(first)).Interrupted) - requireGone(t, filepath.Join(spoolDir, first)) + rs.muteStart.Store(false) + id := startResumable(t, b) + require.NoError(t, b.EndMission("")) + assert.True(t, rs.mission(id).completed) + assert.Equal(t, int32(2), rs.starts.Load(), "the next start does not end the failed one first") } diff --git a/internal/storage/websocket/recover.go b/internal/storage/websocket/recover.go index 5c33913..c73882b 100644 --- a/internal/storage/websocket/recover.go +++ b/internal/storage/websocket/recover.go @@ -41,9 +41,6 @@ var ( // errDiscardFinished means the spool held a mission the addon discarded // and the server no longer keeps it; the spool is deleted. errDiscardFinished = errors.New("mission discard finished") - // errRecoveryDeferred means the spooled mission is this process' own and - // must wait until it records no other mission; the spool is kept. - errRecoveryDeferred = errors.New("mission recovery deferred") ) // recoveryLoop resends missions left in the spool directory: by a game @@ -122,8 +119,6 @@ func (b *Backend) recoverSpools(done <-chan struct{}) { _ = os.RemoveAll(dir) case errors.Is(err, errSpoolDiscarded): _ = os.RemoveAll(dir) - case errors.Is(err, errRecoveryDeferred): - b.logger().Debug("WebSocket mission recovery waits until the current mission ends", "dir", dir) case errors.Is(err, errRecoveryUnsupported): // Asking an older server every minute only fills both logs. b.recoveryPausedUntil.Store(time.Now().Add(recoveryUnsupportedPause).UnixNano()) @@ -313,13 +308,6 @@ func (b *Backend) recoverMission(dir string, done <-chan struct{}) error { return errAlreadySaved case streaming.ServerResumeOK: from = reply.Seq + 1 - case streaming.ServerResumeUnknown: - // The server takes a start as its producer's next mission and saves - // the producer's live one as partial. This process' own mission - // waits until it records none. - if !discarding && payload.ProducerID == b.producerID && b.recording() { - return errRecoveryDeferred - } } // Read acknowledgements concurrently: the server acknowledges as it goes, @@ -369,12 +357,23 @@ func (b *Backend) recoverMission(dir string, done <-chan struct{}) error { var lastSeq uint64 var lastData []byte + first := true pace := newPacer(b.cfg.connOptions().withDefaults().resendRate) if err := scanSpool(paths, func(seq uint64, data []byte) error { lastSeq, lastData = seq, data if seq < from { + first = false return nil } + if first { + // A resent start must not take the producer's live mission + // for one it left behind. + first = false + var err error + if data, err = markRecovered(data); err != nil { + return fmt.Errorf("mark start recovered: %w", err) + } + } if !pace.wait(len(data), done, nil) { return errConnectionClosed } @@ -415,6 +414,25 @@ func (b *Backend) awaitRecovered(ended <-chan error, done <-chan struct{}) error } } +// markRecovered sets Recovered on a spooled start_mission envelope, keeping +// every other field as recorded. +func markRecovered(data []byte) ([]byte, error) { + var env streaming.Envelope + if err := json.Unmarshal(data, &env); err != nil { + return nil, err + } + var fields map[string]json.RawMessage + if err := json.Unmarshal(env.Payload, &fields); err != nil { + return nil, err + } + fields["recovered"] = json.RawMessage("true") + raw, err := json.Marshal(fields) + if err != nil { + return nil, err + } + return encodeEnvelope(env.Type, raw, env.Seq) +} + // spooledDiscard returns the sequence number of the spool's discard_mission // record, or 0 when the spool does not end with one. func spooledDiscard(paths []string) (uint64, error) { diff --git a/internal/storage/websocket/resume_test.go b/internal/storage/websocket/resume_test.go index deadf64..6e05492 100644 --- a/internal/storage/websocket/resume_test.go +++ b/internal/storage/websocket/resume_test.go @@ -38,6 +38,7 @@ type resumeServer struct { dropEnd atomic.Bool // save end_mission, then close without acknowledging it legacy atomic.Bool // behave like a web server without resume support noDrop atomic.Bool // support resume but predate discard_mission + muteStart atomic.Bool // never acknowledge start_mission resumes atomic.Int32 starts atomic.Int32 conns2ndP atomic.Int32 // connections that began with resume @@ -106,10 +107,20 @@ func (rs *resumeServer) serve(c *ws.Conn) { var p streaming.StartMissionPayload require.NoError(rs.t, json.Unmarshal(env.Payload, &p)) rs.starts.Add(1) + if m != nil && !m.completed { + return // a connection feeds one mission: "mission already active" + } + if rs.muteStart.Load() { + continue + } rs.mu.Lock() // A producer records one mission at a time: its other - // unfinished missions are saved as partial. + // unfinished missions are saved as partial, unless the start + // resends a mission from the spool. for id, other := range rs.missions { + if p.Recovered && !rs.noDrop.Load() { + break + } if id != p.MissionID && other.producer == p.ProducerID && !other.completed { rs.partials[id] = other delete(rs.missions, id) diff --git a/internal/storage/websocket/websocket.go b/internal/storage/websocket/websocket.go index 65f648c..7ce462c 100644 --- a/internal/storage/websocket/websocket.go +++ b/internal/storage/websocket/websocket.go @@ -212,6 +212,15 @@ func (b *Backend) sendEnvelopeAndWait(msgType string, payload any, timeout time. func (b *Backend) StartMission(mission *core.Mission, world *core.World) error { b.missionMu.Lock() defer b.missionMu.Unlock() + + // The game moved on without saving or discarding the previous mission. + // It ends as interrupted, so the server saves what it got as partial and + // nothing is left to recover, and this mission gets a fresh connection. + if b.recording() { + slog.Default().Warn("WebSocket mission abandoned without save or discard; ending it as interrupted") + _ = b.endMission(streaming.EndMissionPayload{Interrupted: true}, abandonTimeout) + } + b.mu.Lock() // A completed mission's server closes its socket. Dial a fresh connection // before accepting the next mission instead of racing the reconnect loop. @@ -228,22 +237,11 @@ func (b *Backend) StartMission(mission *core.Mission, world *core.World) error { b.restart = false } conn := b.conn - abandoned := b.active // the previous mission was neither saved nor discarded b.missionID = newID() missionID := b.missionID b.discardable = false b.mu.Unlock() - // Its spool stays on disk, complete, for recovery to end as interrupted. - // Closing it releases the lock that would otherwise keep recovery away - // until the process exits. - if abandoned { - if old := conn.log.currentSpool(); old != nil { - _ = old.close() - slog.Default().Warn("WebSocket mission abandoned without save or discard; spool kept for recovery", "dir", old.dir) - } - } - base := uint8(1) raw, err := marshalPayload(streaming.TypeStartMission, streaming.StartMissionPayload{ Mission: mission, World: world, FrameBase: &base, @@ -280,6 +278,11 @@ func (b *Backend) StartMission(mission *core.Mission, world *core.World) error { ack, err := conn.waitAck(streaming.TypeStartMission, ackTimeout) if err != nil { + // Nothing records a mission that did not start: forget it, so the + // next start does not take it for an abandoned one. + if sp := b.clearMission(); sp != nil { + removeSpool(sp) + } return err } resumable := ack.ProtocolVersion >= streaming.ProtocolVersionResume @@ -308,7 +311,17 @@ func (b *Backend) EndMission(tag string) error { if tag != "" { payload = streaming.EndMissionPayload{Tag: tag} } - err := b.sendEnvelopeAndWait(streaming.TypeEndMission, payload, b.saveTimeout()) + return b.endMission(payload, b.saveTimeout()) +} + +// abandonTimeout bounds the wait for the server to save an abandoned mission +// before the next one starts; past it, the spool is kept for recovery. +const abandonTimeout = 30 * time.Second + +// endMission sends end_mission, waits for its ack and forgets the mission. +// Caller holds missionMu. +func (b *Backend) endMission(payload any, timeout time.Duration) error { + err := b.sendEnvelopeAndWait(streaming.TypeEndMission, payload, timeout) // Clear mission state regardless of error. sp := b.clearMission() diff --git a/pkg/streaming/messages.go b/pkg/streaming/messages.go index aa5a814..17139ba 100644 --- a/pkg/streaming/messages.go +++ b/pkg/streaming/messages.go @@ -70,10 +70,12 @@ const ProtocolVersionDiscard = 3 // TypeDiscardMission ends a mission without saving it, in place of // end_mission: the addon decided the recording does not meet its save limits, // for example the minimum mission duration. The server drops the recording -// and acknowledges it like end_mission. It is numbered like any mission -// envelope, and is sent only to servers that acknowledged start_mission with -// ProtocolVersionDiscard or later. Saved missions never carry it, so it is -// not part of AllMessageTypes. Its payload is DiscardMissionPayload. +// and acknowledges it like end_mission. On a session it is numbered like any +// mission envelope; without one, it is not numbered and the payload alone +// names the mission. It is sent only to servers that acknowledged +// start_mission, or answered resume, with ProtocolVersionDiscard or later. +// Saved missions never carry it, so it is not part of AllMessageTypes. Its +// payload is DiscardMissionPayload. const TypeDiscardMission = "discard_mission" // DiscardMissionPayload names the discarded mission. A server that no longer @@ -103,7 +105,8 @@ const ( // mission, for example after a web server restart. ServerResumeUnknown = "resume_unknown" // ServerResumeCompleted answers resume for a mission the server has - // already saved. It stands in for the end_mission acknowledgement. + // already saved, or discarded. It stands in for the end_mission or + // discard_mission acknowledgement. ServerResumeCompleted = "resume_completed" ) @@ -130,6 +133,12 @@ type StartMissionPayload struct { MissionID string `json:"missionId,omitempty"` // unique per recording ProducerID string `json:"producerId,omitempty"` // unique per extension process ProtocolVersion uint8 `json:"protocolVersion,omitempty"` + // Recovered marks a start resent from the spool, for a mission the + // producer no longer records. A server takes any other start as the + // producer's next mission and saves the producer's unfinished ones as + // partial; servers at ProtocolVersionDiscard or later leave them alone + // for a recovered start. + Recovered bool `json:"recovered,omitempty"` } // ResumePayload identifies the mission a replacement connection continues. From 6cad419f840fa257e2398a78773115dc50b5898b Mon Sep 17 00:00:00 2001 From: Smith <13200576+smitt14ua@users.noreply.github.com> Date: Tue, 27 Oct 2026 00:37:34 +0200 Subject: [PATCH 04/10] fix(websocket): track the server's protocol per connection, remove spools under their lock Third review round: - Whether the server can discard comes from what it last announced: the start ack, a resume reply or a restarted start's ack. A server replaced by an older one mid-mission is never sent discard_mission, and recovery gives up a discard when resume comes from a server below version 3. - An abandoned mission that recorded nothing after its start is discarded instead of saved as an empty partial. - spool.remove() deletes the segments before releasing the lock: recovery could take a spool being removed and resend a mission that was saved, discarded or never started (on Windows the delete then failed). - :STORAGE:INIT: answers :STORAGE:ERROR: when storage cannot start. - abandonTimeout is a var for tests; tests for the downgrade, an empty or unconfirmed abandoned mission, and :MISSION:FAILED: on a panic. The fake server ignores discard_mission when it predates it, like the web server. --- README.md | 5 +- cmd/ocap_recorder/main.go | 4 ++ cmd/ocap_recorder/main_test.go | 25 +++++++ internal/storage/websocket/connection.go | 11 ++- internal/storage/websocket/discard_test.go | 78 ++++++++++++++++++++++ internal/storage/websocket/recover.go | 6 +- internal/storage/websocket/resume_test.go | 6 +- internal/storage/websocket/spool.go | 28 ++++++-- internal/storage/websocket/websocket.go | 51 +++++++++----- 9 files changed, 185 insertions(+), 29 deletions(-) diff --git a/README.md b/README.md index 1ab0c83..edc1fae 100644 --- a/README.md +++ b/README.md @@ -172,7 +172,8 @@ When the game moves on to the next mission without saving or discarding the current one, for example after an admin `#missions`, the proxy ends it as interrupted before the next one starts, and the web service saves it as `partial`. Only if that is not confirmed within 30 seconds does its spool stay -behind for recovery. +behind for recovery. One that recorded nothing after its start is discarded +instead, when the web service supports it. When the addon ends a recording that does not meet its save limits (for example one shorter than `OCAP_settings_minMissionTime`), it sends @@ -282,7 +283,7 @@ All commands follow a `:RESOURCE:ACTION:` naming convention. New commands must u | Command | Purpose | |---------|---------| | `:SYS:INIT:` | Initialize extension, send `:SYS:READY:` callback | -| `:STORAGE:INIT:` | Initialize storage backend, ungate buffered handlers | +| `:STORAGE:INIT:` | Initialize storage backend, ungate buffered handlers; answers with a `:STORAGE:OK:` callback, or `:STORAGE:ERROR:` with the error | | `:MISSION:START:` | Start recording mission; answers with a `:MISSION:OK:` callback, or `:MISSION:FAILED:` with the error | | `:MISSION:SAVE:` | End recording, flush data, upload if configured | | `:MISSION:DISCARD:` | End recording without saving it (`memory`, `websocket`); answers with a `:MISSION:DISCARDED:` callback | diff --git a/cmd/ocap_recorder/main.go b/cmd/ocap_recorder/main.go index 82d66e3..7689d61 100644 --- a/cmd/ocap_recorder/main.go +++ b/cmd/ocap_recorder/main.go @@ -384,6 +384,10 @@ func registerLifecycleHandlers(d *dispatcher.Dispatcher) { go func() { if err := initStorage(); err != nil { Logger.Error("Storage initialization failed", "error", err) + // The addon stops waiting for storage to record. + if cbErr := a3interface.WriteArmaCallback(ExtensionName, ":STORAGE:ERROR:", err.Error()); cbErr != nil { + Logger.Warn("Failed to send STORAGE:ERROR callback", "error", cbErr) + } } }() return nil, nil diff --git a/cmd/ocap_recorder/main_test.go b/cmd/ocap_recorder/main_test.go index a12a7a9..6ca9e63 100644 --- a/cmd/ocap_recorder/main_test.go +++ b/cmd/ocap_recorder/main_test.go @@ -10,6 +10,7 @@ import ( "github.com/OCAP2/extension/v5/internal/dispatcher" "github.com/OCAP2/extension/v5/internal/parser" "github.com/OCAP2/extension/v5/pkg/a3interface" + "github.com/OCAP2/extension/v5/pkg/core" ) func TestHandleNewMission_FailureIsReported(t *testing.T) { @@ -27,9 +28,33 @@ func TestHandleNewMission_FailureIsReported(t *testing.T) { require.Len(t, cb.data, 1) assert.Contains(t, cb.data[0], "not initialized") + rec.mu.Lock() rec.calls = nil + rec.mu.Unlock() parserService = parser.NewParser(Logger) _, err = handleNewMission(dispatcher.Event{Args: []string{"not json"}}) require.Error(t, err) rec.wait(t, ":MISSION:FAILED:") } + +// panickingParser fails every mission start with a panic. +type panickingParser struct{ parser.Service } + +func (panickingParser) ParseMission([]string) (core.Mission, core.World, error) { + panic("boom") +} + +func TestHandleNewMission_PanicIsReported(t *testing.T) { + rec := &callbackRecorder{} + a3interface.SetCallbackSinkForTest(rec.sink) + t.Cleanup(func() { a3interface.SetCallbackSinkForTest(nil) }) + previousLogger, previousParser := Logger, parserService + t.Cleanup(func() { Logger, parserService = previousLogger, previousParser }) + Logger = slog.Default() + parserService = panickingParser{} + + assert.Panics(t, func() { _, _ = handleNewMission(dispatcher.Event{}) }, "the dispatcher still sees the panic") + cb := rec.wait(t, ":MISSION:FAILED:") + require.Len(t, cb.data, 1) + assert.Contains(t, cb.data[0], "panic") +} diff --git a/internal/storage/websocket/connection.go b/internal/storage/websocket/connection.go index 65852c2..b4543f1 100644 --- a/internal/storage/websocket/connection.go +++ b/internal/storage/websocket/connection.go @@ -100,6 +100,10 @@ type connection struct { resumeData []byte // resume envelope for this mission resumable bool // the server acknowledged start with resume support + // protocol is the version the server serving the mission last announced: + // on the start acknowledgement, a resume reply, or a restarted start. + protocol atomic.Uint32 + acked atomic.Uint64 // highest seq the server acknowledged sent atomic.Uint64 // highest seq written on the current socket resumes atomic.Uint64 // replacement connections that resumed the mission @@ -238,6 +242,7 @@ func (c *connection) handshake(conn *ws.Conn) (uint64, uint64, error) { case streaming.ServerResumeOK: c.logger.Info("WebSocket mission resumed", "serverSeq", reply.Seq, "localSeq", c.log.lastSeq()) c.resumes.Add(1) + c.protocol.Store(uint32(reply.ProtocolVersion)) // the server may have changed meanwhile c.ackUpTo(reply.Seq) return reply.Seq, epoch, nil @@ -266,11 +271,13 @@ func (c *connection) handshake(conn *ws.Conn) (uint64, uint64, error) { if err := writeData(conn, start); err != nil { return 0, 0, err } - if _, err := readServerMessage(conn, func(m streaming.AckMessage) bool { + ack, err := readServerMessage(conn, func(m streaming.AckMessage) bool { return m.Type == streaming.ServerAck && m.For == streaming.TypeStartMission - }); err != nil { + }) + if err != nil { return 0, 0, fmt.Errorf("restart mission: %w", err) } + c.protocol.Store(uint32(ack.ProtocolVersion)) // Acknowledgements now count from the new session's start. c.acked.Store(1) return 1, epoch, nil diff --git a/internal/storage/websocket/discard_test.go b/internal/storage/websocket/discard_test.go index 92f7675..8f4d899 100644 --- a/internal/storage/websocket/discard_test.go +++ b/internal/storage/websocket/discard_test.go @@ -367,3 +367,81 @@ func TestFailedStartLeavesNoMission(t *testing.T) { assert.True(t, rs.mission(id).completed) assert.Equal(t, int32(2), rs.starts.Load(), "the next start does not end the failed one first") } + +func TestAbandonedEmptyMissionIsDiscarded(t *testing.T) { + rs := newResumeServer(t) + spoolDir := t.TempDir() + b := New(recoveryConfig(rs.url(), spoolDir)) + require.NoError(t, b.Init()) + defer b.Close() + + // Registered, but the game moved on before anything was recorded. + first := startResumable(t, b) + second := startResumable(t, b) + m := rs.mission(first) + assert.True(t, m.discarded, "an empty recording is not kept") + assert.NoDirExists(t, filepath.Join(spoolDir, first)) + require.NoError(t, b.EndMission("")) + assert.True(t, rs.mission(second).completed) +} + +func TestAbandonedMissionUnconfirmedKeepsSpool(t *testing.T) { + oldAbandon := abandonTimeout + abandonTimeout = 200 * time.Millisecond // below the ack stall, so no reconnect resends it + t.Cleanup(func() { abandonTimeout = oldAbandon }) + rs := newResumeServer(t) + spoolDir := t.TempDir() + cfg := recoveryConfig(rs.url(), spoolDir) + cfg.RecoveryInterval = time.Hour + b := New(cfg) + require.NoError(t, b.Init()) + defer b.Close() + + first := startResumable(t, b) + recordStates(t, b, 1, 3) + require.Eventually(t, func() bool { return rs.lastSeq(first) == 4 }, 2*time.Second, 5*time.Millisecond) + rs.loseEnd.Store(true) // the interrupted end never arrives + second := startResumable(t, b) + assert.NotEqual(t, first, second) + assert.False(t, rs.mission(first).completed) + assert.DirExists(t, filepath.Join(spoolDir, first), "kept for recovery") + rs.loseEnd.Store(false) + require.NoError(t, b.EndMission("")) + assert.True(t, rs.mission(second).completed) +} + +func TestDiscardAfterServerDowngradeIsNotSent(t *testing.T) { + rs := newResumeServer(t) + b := New(fastConfig(rs.url())) + require.NoError(t, b.Init()) + defer b.Close() + + id := startResumable(t, b) + require.True(t, b.discardable()) + recordStates(t, b, 1, 3) + require.Eventually(t, func() bool { return rs.lastSeq(id) == 4 }, 2*time.Second, 5*time.Millisecond) + + // The web server is replaced by one without discard; the mission resumes. + rs.noDrop.Store(true) + resumes := rs.resumes.Load() + rs.kill() + require.Eventually(t, func() bool { return rs.resumes.Load() > resumes }, 5*time.Second, 10*time.Millisecond) + require.Eventually(t, func() bool { return !b.discardable() }, 5*time.Second, 10*time.Millisecond) + + require.ErrorIs(t, b.DiscardMission(), storage.ErrMissionKept) + requireNoDiscard(t, rs.mission(id).received) +} + +func TestRecoveryOfDiscardAfterServerDowngrade(t *testing.T) { + fastRecovery(t) + rs := newResumeServer(t) + spoolDir := t.TempDir() + b, id := discardUnconfirmed(t, rs, spoolDir) + rs.noDrop.Store(true) // the server holds the session but cannot discard + + b.recoverSpools(b.done) + + assert.False(t, rs.mission(id).discarded) + requireNoDiscard(t, rs.mission(id).received) + assert.NoDirExists(t, filepath.Join(spoolDir, id), "an older server cannot do more") +} diff --git a/internal/storage/websocket/recover.go b/internal/storage/websocket/recover.go index c73882b..f615784 100644 --- a/internal/storage/websocket/recover.go +++ b/internal/storage/websocket/recover.go @@ -294,8 +294,10 @@ func (b *Backend) recoverMission(dir string, done <-chan struct{}) error { switch { case reply.Type == streaming.ServerResumeCompleted: return errDiscardFinished // the server confirmed the discard before - case reply.Type == streaming.ServerResumeUnknown && reply.ProtocolVersion < streaming.ProtocolVersionDiscard: - return errDiscardFinished // nothing more an older server can do + case reply.ProtocolVersion < streaming.ProtocolVersionDiscard: + // The server was replaced by an older one: nothing more it can do, + // and it saves what it holds as partial. + return errDiscardFinished } // resume_ok: the server still holds the session. resume_unknown: it // may still keep the mission, saved as partial after the producer diff --git a/internal/storage/websocket/resume_test.go b/internal/storage/websocket/resume_test.go index 6e05492..5cfe98f 100644 --- a/internal/storage/websocket/resume_test.go +++ b/internal/storage/websocket/resume_test.go @@ -39,6 +39,7 @@ type resumeServer struct { legacy atomic.Bool // behave like a web server without resume support noDrop atomic.Bool // support resume but predate discard_mission muteStart atomic.Bool // never acknowledge start_mission + loseEnd atomic.Bool // drop end_mission as if lost in transit resumes atomic.Int32 starts atomic.Int32 conns2ndP atomic.Int32 // connections that began with resume @@ -187,7 +188,10 @@ func (rs *resumeServer) serve(c *ws.Conn) { _ = c.WriteJSON(streaming.AckMessage{Type: streaming.ServerAck, For: env.Type}) return } - if m == nil { + if m == nil || (env.Type == streaming.TypeDiscardMission && rs.noDrop.Load()) { + continue // an older server ignores discard_mission + } + if env.Type == streaming.TypeEndMission && rs.loseEnd.Load() { continue } rs.mu.Lock() diff --git a/internal/storage/websocket/spool.go b/internal/storage/websocket/spool.go index 8c4b1e3..19a640e 100644 --- a/internal/storage/websocket/spool.go +++ b/internal/storage/websocket/spool.go @@ -292,6 +292,19 @@ func readSegment(path string, from, to uint64, maxBytes int, out *[]logEntry, by // close flushes and closes the spool, keeping its files. func (s *spool) close() error { + return s.shut(false) +} + +// remove closes the spool and deletes its files. The segments go before the +// lock is released, so recovery never takes up a spool being removed: it +// would resend a mission that was saved, discarded or never started. +func (s *spool) remove() error { + _ = s.shut(true) + return os.RemoveAll(s.dir) +} + +// shut closes the spool, deleting its segments first when drop is set. +func (s *spool) shut(drop bool) error { s.mu.Lock() defer s.mu.Unlock() if s.closed { @@ -301,13 +314,20 @@ func (s *spool) close() error { close(s.done) var err error if s.file != nil { - if s.ioErr == nil { + if s.ioErr == nil && !drop { err = s.w.Flush() } if cerr := s.file.Close(); err == nil { err = cerr } } + if drop { + for _, seg := range s.segments { + if rerr := os.Remove(seg.path); err == nil && rerr != nil && !os.IsNotExist(rerr) { + err = rerr + } + } + } // The mission is complete on disk; recovery may take it from here. if rerr := os.Remove(filepath.Join(s.dir, spoolLiveFile)); err == nil && rerr != nil && !os.IsNotExist(rerr) { err = rerr @@ -315,9 +335,3 @@ func (s *spool) close() error { s.lock.release() return err } - -// remove closes the spool and deletes its files. -func (s *spool) remove() error { - _ = s.close() - return os.RemoveAll(s.dir) -} diff --git a/internal/storage/websocket/websocket.go b/internal/storage/websocket/websocket.go index 7ce462c..b9b0616 100644 --- a/internal/storage/websocket/websocket.go +++ b/internal/storage/websocket/websocket.go @@ -62,7 +62,6 @@ type Backend struct { mu sync.Mutex restart bool active bool // a mission has started and not ended - discardable bool // the server acknowledged start with discard support conn *connection cfg Config producerID string // identifies this extension process to the server @@ -216,9 +215,19 @@ func (b *Backend) StartMission(mission *core.Mission, world *core.World) error { // The game moved on without saving or discarding the previous mission. // It ends as interrupted, so the server saves what it got as partial and // nothing is left to recover, and this mission gets a fresh connection. + // One that recorded nothing after its start is dropped instead, when the + // server can. if b.recording() { - slog.Default().Warn("WebSocket mission abandoned without save or discard; ending it as interrupted") - _ = b.endMission(streaming.EndMissionPayload{Interrupted: true}, abandonTimeout) + b.mu.Lock() + old, oldID := b.conn, b.missionID + b.mu.Unlock() + if old.log.lastSeq() <= 1 && b.discardable() { + slog.Default().Warn("WebSocket mission abandoned before it recorded anything; discarding it", "missionId", oldID) + _ = b.discardMission(abandonTimeout) + } else { + slog.Default().Warn("WebSocket mission abandoned without save or discard; ending it as interrupted", "missionId", oldID) + _ = b.endMission(streaming.EndMissionPayload{Interrupted: true}, abandonTimeout) + } } b.mu.Lock() @@ -239,8 +248,8 @@ func (b *Backend) StartMission(mission *core.Mission, world *core.World) error { conn := b.conn b.missionID = newID() missionID := b.missionID - b.discardable = false b.mu.Unlock() + conn.protocol.Store(0) base := uint8(1) raw, err := marshalPayload(streaming.TypeStartMission, streaming.StartMissionPayload{ @@ -291,12 +300,18 @@ func (b *Backend) StartMission(mission *core.Mission, world *core.World) error { conn.ackUpTo(max(ack.Seq, 1)) b.resumeRecovery() } - discardable := ack.ProtocolVersion >= streaming.ProtocolVersionDiscard + conn.protocol.Store(uint32(ack.ProtocolVersion)) + slog.Default().Info("WebSocket mission started", "missionId", missionID, "resumable", resumable, "discardable", b.discardable()) + return nil +} + +// discardable reports whether the server serving the mission accepts +// discard_mission, as it last announced. +func (b *Backend) discardable() bool { b.mu.Lock() - b.discardable = discardable + conn := b.conn b.mu.Unlock() - slog.Default().Info("WebSocket mission started", "missionId", missionID, "resumable", resumable, "discardable", discardable) - return nil + return conn.protocol.Load() >= streaming.ProtocolVersionDiscard } // EndMission sends end_mission and waits for server ack. @@ -315,8 +330,9 @@ func (b *Backend) EndMission(tag string) error { } // abandonTimeout bounds the wait for the server to save an abandoned mission -// before the next one starts; past it, the spool is kept for recovery. -const abandonTimeout = 30 * time.Second +// before the next one starts; past it, the spool is kept for recovery. Tests +// lower it. +var abandonTimeout = 30 * time.Second // endMission sends end_mission, waits for its ack and forgets the mission. // Caller holds missionMu. @@ -339,8 +355,8 @@ func (b *Backend) endMission(payload any, timeout time.Duration) error { return err } -// errDiscardUnsupported means the server predates discard_mission, or never -// acknowledged the mission start that would tell. +// errDiscardUnsupported means the server serving the mission predates +// discard_mission. var errDiscardUnsupported = fmt.Errorf("websocket server cannot discard missions: %w", storage.ErrMissionKept) // DiscardMission ends the mission without saving it: it sends @@ -352,18 +368,24 @@ var errDiscardUnsupported = fmt.Errorf("websocket server cannot discard missions func (b *Backend) DiscardMission() error { b.missionMu.Lock() defer b.missionMu.Unlock() + return b.discardMission(b.saveTimeout()) +} +// discardMission sends discard_mission, waits for its ack and forgets the +// mission. Caller holds missionMu. +func (b *Backend) discardMission(timeout time.Duration) error { b.mu.Lock() - active, discardable, missionID := b.active, b.discardable, b.missionID + active, missionID := b.active, b.missionID b.mu.Unlock() if !active { return nil } + discardable := b.discardable() err := errDiscardUnsupported if discardable { payload := streaming.DiscardMissionPayload{MissionID: missionID, ProducerID: b.producerID} - err = b.sendEnvelopeAndWait(streaming.TypeDiscardMission, payload, b.saveTimeout()) + err = b.sendEnvelopeAndWait(streaming.TypeDiscardMission, payload, timeout) } // A spool that holds discard_mission lets recovery finish the discard. @@ -403,7 +425,6 @@ func (b *Backend) clearMission() *spool { conn := b.conn b.restart = true b.active = false - b.discardable = false b.missionID = "" b.mu.Unlock() sp := conn.log.currentSpool() From 3d996d0dd0fb5ae54049bda80108b347c47a15e2 Mon Sep 17 00:00:00 2001 From: Smith <13200576+smitt14ua@users.noreply.github.com> Date: Tue, 27 Oct 2026 01:00:20 +0200 Subject: [PATCH 05/10] fix: keep one storage backend per process, refuse discards after a downgrade, echo a registration token Fourth review round: - :STORAGE:INIT:, which the addon sends on every mission load, built a new backend each time and leaked the old one; a mission the game moved on from stayed open in it. Storage is now initialized once and later calls are answered with :STORAGE:OK: again, so the next StartMission ends the abandoned mission. - :MISSION:START: echoes an optional registration token (third argument) in :MISSION:OK: and :MISSION:FAILED:. - A reconnect that finds the server replaced by one below protocol 3 refuses a discard still waiting for its ack instead of waiting out the save timeout. - A spool being removed is marked dropped and its segments deleted under its lock; recovery deletes a dropped spool and marks the spools it finishes before releasing them. - Against a server below protocol 3, own missions are resent in full only while no other records. A failed start closes its connection. The start ack's protocol is stored before reconnects can record a newer one. - Tests for each; the fake server exempts recovered missions from supersede, like the web server. --- README.md | 8 ++- cmd/ocap_recorder/main.go | 31 ++++----- cmd/ocap_recorder/main_test.go | 33 +++++++-- cmd/ocap_recorder/storage.go | 33 ++++++++- cmd/ocap_recorder/storage_test.go | 55 +++++++++++++++ internal/storage/websocket/connection.go | 26 +++++-- internal/storage/websocket/discard_test.go | 80 ++++++++++++++++++++-- internal/storage/websocket/recover.go | 21 ++++++ internal/storage/websocket/resume_test.go | 13 ++-- internal/storage/websocket/spool.go | 35 ++++++++-- internal/storage/websocket/websocket.go | 27 ++++++-- 11 files changed, 311 insertions(+), 51 deletions(-) diff --git a/README.md b/README.md index edc1fae..f5c1ec9 100644 --- a/README.md +++ b/README.md @@ -159,7 +159,9 @@ extension checks `spoolDir` at startup and then every minute, resends such missions to the web service, and deletes each spool once the web service has saved its mission. A mission resent in full is marked as recovered, so a web service at protocol version 3 does not take it for the producer's next -mission and save the mission it records now as `partial`: +mission and save the mission it records now as `partial`. An older web service +does, so with one the running process resends its own missions in full only +while it records none: - A mission cut short by a crash is ended on the crashed process' behalf and saved as `partial`, with everything recorded up to the last flush (at most about a second is lost). @@ -283,8 +285,8 @@ All commands follow a `:RESOURCE:ACTION:` naming convention. New commands must u | Command | Purpose | |---------|---------| | `:SYS:INIT:` | Initialize extension, send `:SYS:READY:` callback | -| `:STORAGE:INIT:` | Initialize storage backend, ungate buffered handlers; answers with a `:STORAGE:OK:` callback, or `:STORAGE:ERROR:` with the error | -| `:MISSION:START:` | Start recording mission; answers with a `:MISSION:OK:` callback, or `:MISSION:FAILED:` with the error | +| `:STORAGE:INIT:` | Initialize storage backend once per process (later calls reuse it), ungate buffered handlers; answers with a `:STORAGE:OK:` callback, or `:STORAGE:ERROR:` with the error | +| `:MISSION:START:` | Start recording mission; answers with a `:MISSION:OK:` callback, or `:MISSION:FAILED:` with the error. An optional third argument, a registration token, is echoed as the callback's last value | | `:MISSION:SAVE:` | End recording, flush data, upload if configured | | `:MISSION:DISCARD:` | End recording without saving it (`memory`, `websocket`); answers with a `:MISSION:DISCARDED:` callback | | `:MISSION:FOCUS_START:` | Set playback focus start frame | diff --git a/cmd/ocap_recorder/main.go b/cmd/ocap_recorder/main.go index 7689d61..eb4c048 100644 --- a/cmd/ocap_recorder/main.go +++ b/cmd/ocap_recorder/main.go @@ -314,9 +314,17 @@ func initAPIClient() { // delegates DB persistence to the storage backend, and sets runtime state. // It answers with a :MISSION:OK: callback, or :MISSION:FAILED: with the error, // so the addon never waits for a mission that did not start. +// +// An optional third argument is the addon's registration token, echoed in +// both callbacks, so a late answer to an earlier mission's start is not taken +// for this one's. func handleNewMission(e dispatcher.Event) (result any, err error) { + token := "" + if len(e.Args) > 2 { + token = util.TrimQuotes(e.Args[2]) + } reportFailure := func(msg string) { - if cbErr := a3interface.WriteArmaCallback(ExtensionName, ":MISSION:FAILED:", msg); cbErr != nil { + if cbErr := a3interface.WriteArmaCallback(ExtensionName, ":MISSION:FAILED:", msg, token); cbErr != nil { Logger.Warn("Failed to send MISSION:FAILED callback", "error", cbErr) } } @@ -329,8 +337,12 @@ func handleNewMission(e dispatcher.Event) (result any, err error) { result, err = startNewMission(e) if err != nil { reportFailure(err.Error()) + return result, err + } + if err := a3interface.WriteArmaCallback(ExtensionName, ":MISSION:OK:", "OK", token); err != nil { + Logger.Warn("Failed to send MISSION:OK callback", "error", err) } - return result, err + return result, nil } func startNewMission(e dispatcher.Event) (any, error) { @@ -364,11 +376,6 @@ func startNewMission(e dispatcher.Event) (any, error) { } Logger.Info("New mission logged", "missionName", coreMission.MissionName) - - // 4. ArmA callback - if err := a3interface.WriteArmaCallback(ExtensionName, ":MISSION:OK:", "OK"); err != nil { - Logger.Warn("Failed to send MISSION:OK callback", "error", err) - } return nil, nil } @@ -381,15 +388,7 @@ func registerLifecycleHandlers(d *dispatcher.Dispatcher) { }) d.Register(":STORAGE:INIT:", func(e dispatcher.Event) (any, error) { - go func() { - if err := initStorage(); err != nil { - Logger.Error("Storage initialization failed", "error", err) - // The addon stops waiting for storage to record. - if cbErr := a3interface.WriteArmaCallback(ExtensionName, ":STORAGE:ERROR:", err.Error()); cbErr != nil { - Logger.Warn("Failed to send STORAGE:ERROR callback", "error", cbErr) - } - } - }() + go startStorage() return nil, nil }) diff --git a/cmd/ocap_recorder/main_test.go b/cmd/ocap_recorder/main_test.go index 6ca9e63..25972a8 100644 --- a/cmd/ocap_recorder/main_test.go +++ b/cmd/ocap_recorder/main_test.go @@ -22,11 +22,12 @@ func TestHandleNewMission_FailureIsReported(t *testing.T) { Logger = slog.Default() parserService = nil - _, err := handleNewMission(dispatcher.Event{}) + _, err := handleNewMission(dispatcher.Event{Args: []string{"{}", "{}", `"tok-1"`}}) require.Error(t, err) cb := rec.wait(t, ":MISSION:FAILED:") - require.Len(t, cb.data, 1) + require.Len(t, cb.data, 2) assert.Contains(t, cb.data[0], "not initialized") + assert.Equal(t, `"tok-1"`, cb.data[1], "the addon's registration token is echoed") rec.mu.Lock() rec.calls = nil @@ -34,7 +35,31 @@ func TestHandleNewMission_FailureIsReported(t *testing.T) { parserService = parser.NewParser(Logger) _, err = handleNewMission(dispatcher.Event{Args: []string{"not json"}}) require.Error(t, err) - rec.wait(t, ":MISSION:FAILED:") + cb = rec.wait(t, ":MISSION:FAILED:") + assert.Equal(t, `""`, cb.data[1], "an addon without a token gets an empty one") +} + +// acceptingParser starts every mission. +type acceptingParser struct{ parser.Service } + +func (acceptingParser) ParseMission([]string) (core.Mission, core.World, error) { + return core.Mission{MissionName: "M"}, core.World{WorldName: "W"}, nil +} + +func TestHandleNewMission_OKEchoesToken(t *testing.T) { + rec := &callbackRecorder{} + a3interface.SetCallbackSinkForTest(rec.sink) + t.Cleanup(func() { a3interface.SetCallbackSinkForTest(nil) }) + previousLogger, previousParser, previousBackend := Logger, parserService, storageBackend + t.Cleanup(func() { Logger, parserService, storageBackend = previousLogger, previousParser, previousBackend }) + Logger = slog.Default() + parserService = acceptingParser{} + storageBackend = nil + + _, err := handleNewMission(dispatcher.Event{Args: []string{"{}", "{}", `"tok-2"`}}) + require.NoError(t, err) + cb := rec.wait(t, ":MISSION:OK:") + assert.Equal(t, []string{`"OK"`, `"tok-2"`}, cb.data) } // panickingParser fails every mission start with a panic. @@ -55,6 +80,6 @@ func TestHandleNewMission_PanicIsReported(t *testing.T) { assert.Panics(t, func() { _, _ = handleNewMission(dispatcher.Event{}) }, "the dispatcher still sees the panic") cb := rec.wait(t, ":MISSION:FAILED:") - require.Len(t, cb.data, 1) + require.Len(t, cb.data, 2) assert.Contains(t, cb.data[0], "panic") } diff --git a/cmd/ocap_recorder/storage.go b/cmd/ocap_recorder/storage.go index 8a66614..5c98d50 100644 --- a/cmd/ocap_recorder/storage.go +++ b/cmd/ocap_recorder/storage.go @@ -3,6 +3,7 @@ package main import ( "fmt" "strings" + "sync" "github.com/OCAP2/extension/v5/internal/config" "github.com/OCAP2/extension/v5/internal/storage" @@ -12,21 +13,49 @@ import ( "github.com/OCAP2/extension/v5/pkg/a3interface" ) +// startStorage handles :STORAGE:INIT:. It answers with a :STORAGE:OK: +// callback, or :STORAGE:ERROR: so the addon stops waiting for storage. +func startStorage() { + if err := initStorage(); err != nil { + Logger.Error("Storage initialization failed", "error", err) + if cbErr := a3interface.WriteArmaCallback(ExtensionName, ":STORAGE:ERROR:", err.Error()); cbErr != nil { + Logger.Warn("Failed to send STORAGE:ERROR callback", "error", cbErr) + } + } +} + +// storageInitMu serializes :STORAGE:INIT:, which the addon sends on every +// mission load. +var storageInitMu sync.Mutex + func initStorage() error { Logger.Debug("Received :STORAGE:INIT: call") + storageInitMu.Lock() + defer storageInitMu.Unlock() storageCfg := config.GetStorageConfig() + // The backend lives as long as the process: the next mission reuses it, + // so a mission the game moved on from is ended by the next StartMission + // instead of staying open in a backend nothing uses any more. + if storageBackend != nil { + Logger.Info("Storage already initialized; reusing it", "type", storageCfg.Type) + if err := a3interface.WriteArmaCallback(ExtensionName, ":STORAGE:OK:", storageCfg.Type); err != nil { + Logger.Warn("Failed to send STORAGE:OK callback", "error", err) + } + return nil + } + backend, err := createStorageBackend(storageCfg) if err != nil { Logger.Error("Failed to create storage backend", "error", err) return err } - storageBackend = backend - if err := storageBackend.Init(); err != nil { + if err := backend.Init(); err != nil { Logger.Error("Failed to initialize storage backend", "error", err) return err } + storageBackend = backend // Initialize worker manager workerManager = worker.NewManager(worker.Dependencies{ diff --git a/cmd/ocap_recorder/storage_test.go b/cmd/ocap_recorder/storage_test.go index b7dd3cc..766ab33 100644 --- a/cmd/ocap_recorder/storage_test.go +++ b/cmd/ocap_recorder/storage_test.go @@ -5,8 +5,11 @@ import ( "testing" "github.com/OCAP2/extension/v5/internal/config" + "github.com/OCAP2/extension/v5/internal/dispatcher" + "github.com/OCAP2/extension/v5/internal/logging" "github.com/OCAP2/extension/v5/internal/storage" "github.com/OCAP2/extension/v5/internal/storage/websocket" + "github.com/OCAP2/extension/v5/pkg/a3interface" "github.com/spf13/viper" "github.com/stretchr/testify/require" ) @@ -41,6 +44,58 @@ func TestWebSocketBackendRequiresSecret(t *testing.T) { require.Error(t, err) } +// isolateStorage gives a test its own storage globals. +func isolateStorage(t *testing.T) *callbackRecorder { + t.Helper() + rec := &callbackRecorder{} + a3interface.SetCallbackSinkForTest(rec.sink) + previousLogger, previousBackend, previousDispatcher, previousWorkers := Logger, storageBackend, eventDispatcher, workerManager + t.Cleanup(func() { + a3interface.SetCallbackSinkForTest(nil) + Logger, storageBackend, eventDispatcher, workerManager = previousLogger, previousBackend, previousDispatcher, previousWorkers + viper.Reset() + }) + Logger = slog.Default() + storageBackend = nil + d, err := dispatcher.New(logging.NewDispatcherLogger(Logger)) + require.NoError(t, err) + eventDispatcher = d + return rec +} + +func TestInitStorageReusesBackend(t *testing.T) { + rec := isolateStorage(t) + viper.Set("storage.type", "memory") + viper.Set("storage.memory.outputDir", t.TempDir()) + + require.NoError(t, initStorage()) + first := storageBackend + require.NotNil(t, first) + // The addon initializes storage again on the next mission load. + require.NoError(t, initStorage()) + require.Same(t, first, storageBackend, "a mission left open is ended by the next start, in the same backend") + + rec.mu.Lock() + defer rec.mu.Unlock() + oks := 0 + for _, c := range rec.calls { + if c.function == ":STORAGE:OK:" { + oks++ + } + } + require.Equal(t, 2, oks, "every :STORAGE:INIT: is answered") +} + +func TestStartStorageReportsFailure(t *testing.T) { + rec := isolateStorage(t) + viper.Set("storage.type", "not-real") + + startStorage() + cb := rec.wait(t, ":STORAGE:ERROR:") + require.Len(t, cb.data, 1) + require.Nil(t, storageBackend) +} + func TestWebSocketURLPreservesPrefix(t *testing.T) { require.Equal(t, "wss://example.test/ocap", httpToWS("https://example.test/ocap/")) } diff --git a/internal/storage/websocket/connection.go b/internal/storage/websocket/connection.go index b4543f1..91aeb04 100644 --- a/internal/storage/websocket/connection.go +++ b/internal/storage/websocket/connection.go @@ -242,7 +242,7 @@ func (c *connection) handshake(conn *ws.Conn) (uint64, uint64, error) { case streaming.ServerResumeOK: c.logger.Info("WebSocket mission resumed", "serverSeq", reply.Seq, "localSeq", c.log.lastSeq()) c.resumes.Add(1) - c.protocol.Store(uint32(reply.ProtocolVersion)) // the server may have changed meanwhile + c.setProtocol(reply.ProtocolVersion) // the server may have changed meanwhile c.ackUpTo(reply.Seq) return reply.Seq, epoch, nil @@ -277,7 +277,7 @@ func (c *connection) handshake(conn *ws.Conn) (uint64, uint64, error) { if err != nil { return 0, 0, fmt.Errorf("restart mission: %w", err) } - c.protocol.Store(uint32(ack.ProtocolVersion)) + c.setProtocol(ack.ProtocolVersion) // Acknowledgements now count from the new session's start. c.acked.Store(1) return 1, epoch, nil @@ -449,6 +449,20 @@ func (c *connection) ackUpTo(seq uint64) { c.progressAt.Store(time.Now().UnixNano()) // progress restarts the stall clock } +// ackRefused stands in for the acknowledgement of a discard_mission that a +// server too old to take it will never send. It never comes from a server. +const ackRefused = "refused" + +// setProtocol records the version the server serving the mission announced +// on a reconnect. A server replaced by one older than ProtocolVersionDiscard +// refuses a discard_mission still waiting for its acknowledgement. +func (c *connection) setProtocol(v uint8) { + old := c.protocol.Swap(uint32(v)) + if old >= streaming.ProtocolVersionDiscard && v < streaming.ProtocolVersionDiscard { + c.deliverAck(streaming.AckMessage{Type: ackRefused, For: streaming.TypeDiscardMission}) + } +} + func (c *connection) deliverAck(m streaming.AckMessage) { select { case c.ackCh <- m: @@ -626,9 +640,13 @@ func (c *connection) waitAck(ackFor string, timeout time.Duration) (streaming.Ac for { select { case ack := <-c.ackCh: - if ack.For == ackFor { - return ack, nil + if ack.For != ackFor { + continue + } + if ack.Type == ackRefused { + return streaming.AckMessage{}, errDiscardUnsupported } + return ack, nil case <-timer.C: return streaming.AckMessage{}, fmt.Errorf("timeout waiting for ack of %q", ackFor) case <-c.done: diff --git a/internal/storage/websocket/discard_test.go b/internal/storage/websocket/discard_test.go index 8f4d899..9291b41 100644 --- a/internal/storage/websocket/discard_test.go +++ b/internal/storage/websocket/discard_test.go @@ -50,7 +50,7 @@ func TestDiscardMissionDropsRecording(t *testing.T) { var p streaming.DiscardMissionPayload require.NoError(t, json.Unmarshal(m.received[6].Payload, &p)) assert.Equal(t, streaming.DiscardMissionPayload{MissionID: id, ProducerID: b.producerID}, p, "the discard names the mission") - assert.NoDirExists(t, filepath.Join(spoolDir, id), "a confirmed discard leaves nothing to recover") + requireGone(t, filepath.Join(spoolDir, id)) // a confirmed discard leaves nothing to recover // The next mission records as usual. next := startResumable(t, b) @@ -115,7 +115,7 @@ func TestDiscardMissionOnServerWithoutDiscard(t *testing.T) { m := rs.mission(id) assert.False(t, m.completed, "the server keeps the mission until its grace expires") requireNoDiscard(t, m.received) - assert.NoDirExists(t, filepath.Join(spoolDir, id), "recovery must not resend a discarded mission") + requireGone(t, filepath.Join(spoolDir, id)) // recovery must not resend a discarded mission require.Eventually(t, func() bool { rs.mu.Lock() defer rs.mu.Unlock() @@ -307,7 +307,7 @@ func TestAbandonedMissionEndsAsInterrupted(t *testing.T) { assert.True(t, endPayload(t, m).Interrupted) requireContiguous(t, m, 5) // start, 3 states, end assert.False(t, rs.partial(first)) - assert.NoDirExists(t, filepath.Join(spoolDir, first), "nothing is left to recover") + requireGone(t, filepath.Join(spoolDir, first)) // nothing is left to recover recordStates(t, b, 1, 2) require.NoError(t, b.EndMission("")) @@ -357,9 +357,10 @@ func TestFailedStartLeavesNoMission(t *testing.T) { rs.muteStart.Store(true) require.Error(t, b.StartMission(&core.Mission{MissionName: "M"}, &core.World{WorldName: "W"})) assert.False(t, b.recording(), "a mission that did not start is not abandoned later") - entries, err := os.ReadDir(spoolDir) - require.NoError(t, err) - assert.Empty(t, entries, "its spool is removed") + require.Eventually(t, func() bool { + entries, err := os.ReadDir(spoolDir) + return err == nil && len(entries) == 0 + }, 5*time.Second, 10*time.Millisecond, "its spool is removed") rs.muteStart.Store(false) id := startResumable(t, b) @@ -380,11 +381,76 @@ func TestAbandonedEmptyMissionIsDiscarded(t *testing.T) { second := startResumable(t, b) m := rs.mission(first) assert.True(t, m.discarded, "an empty recording is not kept") - assert.NoDirExists(t, filepath.Join(spoolDir, first)) + requireGone(t, filepath.Join(spoolDir, first)) require.NoError(t, b.EndMission("")) assert.True(t, rs.mission(second).completed) } +func TestAbandonedEmptyMissionOnServerWithoutDiscard(t *testing.T) { + rs := newResumeServer(t) + rs.noDrop.Store(true) + b := New(fastConfig(rs.url())) + require.NoError(t, b.Init()) + defer b.Close() + + first := startResumable(t, b) + startResumable(t, b) + m := rs.mission(first) + assert.True(t, m.completed, "ended, since this server cannot drop it") + assert.False(t, m.discarded) + requireNoDiscard(t, m.received) + assert.True(t, endPayload(t, m).Interrupted) +} + +func TestPendingDiscardRefusedAfterServerDowngrade(t *testing.T) { + rs := newResumeServer(t) + b := New(fastConfig(rs.url())) // SaveTimeout 5 s + require.NoError(t, b.Init()) + defer b.Close() + + id := startResumable(t, b) + recordStates(t, b, 1, 3) + require.Eventually(t, func() bool { return rs.lastSeq(id) == 4 }, 2*time.Second, 5*time.Millisecond) + + // The link dies with the discard in flight, and the web server comes back + // without discard support. + rs.discard.Store(true) + done := make(chan error, 1) + start := time.Now() + go func() { done <- b.DiscardMission() }() + require.Eventually(t, func() bool { return rs.lastSeq(id) == 4 }, time.Second, 5*time.Millisecond) + time.Sleep(50 * time.Millisecond) // the discard is logged and written into the dead link + rs.noDrop.Store(true) + rs.discard.Store(false) + rs.kill() + + select { + case err := <-done: + require.ErrorIs(t, err, storage.ErrMissionKept, "the older server refuses the waiting discard") + assert.Less(t, time.Since(start), 4*time.Second, "without waiting for the save timeout") + case <-time.After(10 * time.Second): + t.Fatal("DiscardMission did not complete") + } + assert.False(t, rs.mission(id).discarded) +} + +func TestRecoveryDeletesDroppedSpool(t *testing.T) { + fastRecovery(t) + rs := newResumeServer(t) + spoolDir := t.TempDir() + dir := filepath.Join(spoolDir, "m-dropped") + writeTestSpool(t, dir, "m-dropped", "p-old", 3) + // Its writer was removing it when recovery took it. + require.NoError(t, markSpoolDropped(dir)) + + b := New(recoveryConfig(rs.url(), spoolDir)) + require.NoError(t, b.Init()) + defer b.Close() + + requireGone(t, dir) + assert.Zero(t, rs.starts.Load(), "a dropped spool is never resent") +} + func TestAbandonedMissionUnconfirmedKeepsSpool(t *testing.T) { oldAbandon := abandonTimeout abandonTimeout = 200 * time.Millisecond // below the ack stall, so no reconnect resends it diff --git a/internal/storage/websocket/recover.go b/internal/storage/websocket/recover.go index f615784..d77f2d4 100644 --- a/internal/storage/websocket/recover.go +++ b/internal/storage/websocket/recover.go @@ -41,6 +41,9 @@ var ( // errDiscardFinished means the spool held a mission the addon discarded // and the server no longer keeps it; the spool is deleted. errDiscardFinished = errors.New("mission discard finished") + // errRecoveryDeferred means the spooled mission is this process' own and + // waits until it records no other one; the spool is kept. + errRecoveryDeferred = errors.New("mission recovery deferred") ) // recoveryLoop resends missions left in the spool directory: by a game @@ -105,6 +108,12 @@ func (b *Backend) recoverSpools(done <-chan struct{}) { continue } err = b.recoverMission(dir, done) + finished := err == nil || errors.Is(err, errAlreadySaved) || errors.Is(err, errDiscardFinished) || errors.Is(err, errSpoolDiscarded) + if finished { + // Marked while locked: a process taking the spool before it is + // gone deletes it instead of resending it. + _ = markSpoolDropped(dir) + } lock.release() // before removal: Windows cannot delete an open file switch { case err == nil: @@ -119,6 +128,8 @@ func (b *Backend) recoverSpools(done <-chan struct{}) { _ = os.RemoveAll(dir) case errors.Is(err, errSpoolDiscarded): _ = os.RemoveAll(dir) + case errors.Is(err, errRecoveryDeferred): + b.logger().Debug("WebSocket mission recovery waits until the current mission ends", "dir", dir) case errors.Is(err, errRecoveryUnsupported): // Asking an older server every minute only fills both logs. b.recoveryPausedUntil.Store(time.Now().Add(recoveryUnsupportedPause).UnixNano()) @@ -206,6 +217,9 @@ func scanSegment(path string, fn func(seq uint64, data []byte) error) error { // continues a session that is still waiting, or takes the whole mission when // it no longer has one. func (b *Backend) recoverMission(dir string, done <-chan struct{}) error { + if spoolDropped(dir) { + return errSpoolDiscarded // its writer or a recovery was removing it + } paths, newest, err := spoolSegments(dir) if err != nil { return err @@ -310,6 +324,13 @@ func (b *Backend) recoverMission(dir string, done <-chan struct{}) error { return errAlreadySaved case streaming.ServerResumeOK: from = reply.Seq + 1 + case streaming.ServerResumeUnknown: + // A server older than ProtocolVersionDiscard takes any start as the + // producer's next mission and saves its live one as partial. This + // process' own mission waits until it records none. + if !discarding && payload.ProducerID == b.producerID && b.recording() && !b.discardable() { + return errRecoveryDeferred + } } // Read acknowledgements concurrently: the server acknowledges as it goes, diff --git a/internal/storage/websocket/resume_test.go b/internal/storage/websocket/resume_test.go index 5cfe98f..6156e5b 100644 --- a/internal/storage/websocket/resume_test.go +++ b/internal/storage/websocket/resume_test.go @@ -53,6 +53,7 @@ type fakeMission struct { gaps int completed bool discarded bool // finished by discard_mission instead of end_mission + recovered bool // started by a resend from the spool } func newResumeServer(t *testing.T) *resumeServer { @@ -116,20 +117,22 @@ func (rs *resumeServer) serve(c *ws.Conn) { } rs.mu.Lock() // A producer records one mission at a time: its other - // unfinished missions are saved as partial, unless the start - // resends a mission from the spool. + // unfinished missions are saved as partial. A server that + // knows recovered starts neither supersedes with one nor + // supersedes one. + knowsRecovered := !rs.noDrop.Load() for id, other := range rs.missions { - if p.Recovered && !rs.noDrop.Load() { + if p.Recovered && knowsRecovered { break } - if id != p.MissionID && other.producer == p.ProducerID && !other.completed { + if id != p.MissionID && other.producer == p.ProducerID && !other.completed && !(other.recovered && knowsRecovered) { rs.partials[id] = other delete(rs.missions, id) } } m = rs.missions[p.MissionID] if m == nil { - m = &fakeMission{producer: p.ProducerID, lastSeq: env.Seq} + m = &fakeMission{producer: p.ProducerID, lastSeq: env.Seq, recovered: p.Recovered} rs.missions[p.MissionID] = m } else if env.Seq > m.lastSeq { m.lastSeq = env.Seq diff --git a/internal/storage/websocket/spool.go b/internal/storage/websocket/spool.go index 19a640e..e8af213 100644 --- a/internal/storage/websocket/spool.go +++ b/internal/storage/websocket/spool.go @@ -295,12 +295,36 @@ func (s *spool) close() error { return s.shut(false) } -// remove closes the spool and deletes its files. The segments go before the -// lock is released, so recovery never takes up a spool being removed: it -// would resend a mission that was saved, discarded or never started. +// remove closes the spool and deletes its files. It is marked dropped and its +// segments go before the lock is released, so recovery never takes up a +// spool being removed: it would resend a mission that was saved, discarded or +// never started. Recovery deletes a dropped spool it finds, so a directory +// another process holds open for a moment is not an error. func (s *spool) remove() error { _ = s.shut(true) - return os.RemoveAll(s.dir) + if err := os.RemoveAll(s.dir); err != nil { + if spoolDropped(s.dir) { + s.logger.Debug("WebSocket spool left for recovery to delete", "dir", s.dir, "error", err) + return nil + } + return err + } + return nil +} + +// spoolDroppedFile marks a spool whose mission needs nothing more; recovery +// deletes it. +const spoolDroppedFile = "dropped" + +// markSpoolDropped marks the spool in dir dropped. Call it while holding the +// spool's lock. +func markSpoolDropped(dir string) error { + return os.WriteFile(filepath.Join(dir, spoolDroppedFile), nil, 0o644) +} + +func spoolDropped(dir string) bool { + _, err := os.Stat(filepath.Join(dir, spoolDroppedFile)) + return err == nil } // shut closes the spool, deleting its segments first when drop is set. @@ -322,6 +346,9 @@ func (s *spool) shut(drop bool) error { } } if drop { + if merr := markSpoolDropped(s.dir); err == nil && merr != nil { + err = merr + } for _, seg := range s.segments { if rerr := os.Remove(seg.path); err == nil && rerr != nil && !os.IsNotExist(rerr) { err = rerr diff --git a/internal/storage/websocket/websocket.go b/internal/storage/websocket/websocket.go index b9b0616..14a5637 100644 --- a/internal/storage/websocket/websocket.go +++ b/internal/storage/websocket/websocket.go @@ -3,6 +3,7 @@ package websocket import ( "crypto/rand" "encoding/json" + "errors" "fmt" "log/slog" "path/filepath" @@ -288,19 +289,22 @@ func (b *Backend) StartMission(mission *core.Mission, world *core.World) error { ack, err := conn.waitAck(streaming.TypeStartMission, ackTimeout) if err != nil { // Nothing records a mission that did not start: forget it, so the - // next start does not take it for an abandoned one. + // next start does not take it for an abandoned one. A server that + // started it late loses its producer and drops the empty session. if sp := b.clearMission(); sp != nil { removeSpool(sp) } + _ = conn.close() return err } + // Stored first: a reconnect from here on records what it finds. + conn.protocol.Store(uint32(ack.ProtocolVersion)) resumable := ack.ProtocolVersion >= streaming.ProtocolVersionResume conn.setResumable(resumable) if resumable { conn.ackUpTo(max(ack.Seq, 1)) b.resumeRecovery() } - conn.protocol.Store(uint32(ack.ProtocolVersion)) slog.Default().Info("WebSocket mission started", "missionId", missionID, "resumable", resumable, "discardable", b.discardable()) return nil } @@ -381,11 +385,25 @@ func (b *Backend) discardMission(timeout time.Duration) error { return nil } + b.mu.Lock() + conn := b.conn + b.mu.Unlock() + conn.drainAcks() + // A reconnect that finds the server replaced by an older one lowers the + // version, and refuses a discard already waiting for its ack. discardable := b.discardable() err := errDiscardUnsupported if discardable { payload := streaming.DiscardMissionPayload{MissionID: missionID, ProducerID: b.producerID} - err = b.sendEnvelopeAndWait(streaming.TypeDiscardMission, payload, timeout) + err = conn.send(func(seq uint64) ([]byte, error) { + return marshalEnvelope(streaming.TypeDiscardMission, payload, seq) + }) + if err == nil { + _, err = conn.waitAck(streaming.TypeDiscardMission, timeout) + } + if errors.Is(err, errDiscardUnsupported) { + discardable = false + } } // A spool that holds discard_mission lets recovery finish the discard. @@ -404,9 +422,6 @@ func (b *Backend) discardMission(timeout time.Duration) error { case err == nil: slog.Default().Info("WebSocket mission discarded", "missionId", missionID) case !discardable: - b.mu.Lock() - conn := b.conn - b.mu.Unlock() _ = conn.close() // the server stops waiting for the mission; StartMission dials anew slog.Default().Warn("WebSocket server cannot discard missions; it saves what it received as partial", "missionId", missionID) case keep: From bc87c27c412411e901a333c302a67e5538284090 Mon Sep 17 00:00:00 2001 From: Smith <13200576+smitt14ua@users.noreply.github.com> Date: Tue, 27 Oct 2026 01:22:26 +0200 Subject: [PATCH 06/10] fix: end abandoned missions when the next mission loads, order a start after a dispatched discard Fifth review round: - A later :STORAGE:INIT: (the next mission loaded) ends a mission still open through the new optional storage.Abandonable. Waiting for the next recording to start, which the addon now defers, left it open. - An abandoned mission on a legacy server is disconnected instead of ended: such a server ignores "interrupted" and would save it as complete. - :MISSION:START: waits for a save or discard already dispatched, so a discard cannot reach the backend after the next start and drop it. - A discard whose envelope never made it into the log does not keep the spool. Removing a spool deletes the dropped marker last, and recovery drops the spools it finishes under their lock too. - Tests for these and for the own-mission deferral on old servers; cmd tests no longer write Logger, which the package init reads. --- README.md | 10 ++-- cmd/ocap_recorder/main.go | 5 +- cmd/ocap_recorder/main_test.go | 62 ++++++++++++++-------- cmd/ocap_recorder/mission_save.go | 9 ++++ cmd/ocap_recorder/storage.go | 8 +++ cmd/ocap_recorder/storage_test.go | 34 ++++++++++-- internal/storage/storage.go | 10 ++++ internal/storage/websocket/discard_test.go | 58 ++++++++++++++++++++ internal/storage/websocket/recover.go | 13 +++-- internal/storage/websocket/resume_test.go | 4 +- internal/storage/websocket/spool.go | 49 ++++++++++++++++- internal/storage/websocket/websocket.go | 61 ++++++++++++++------- 12 files changed, 266 insertions(+), 57 deletions(-) diff --git a/README.md b/README.md index f5c1ec9..09067cb 100644 --- a/README.md +++ b/README.md @@ -172,10 +172,12 @@ while it records none: When the game moves on to the next mission without saving or discarding the current one, for example after an admin `#missions`, the proxy ends it as -interrupted before the next one starts, and the web service saves it as -`partial`. Only if that is not confirmed within 30 seconds does its spool stay -behind for recovery. One that recorded nothing after its start is discarded -instead, when the web service supports it. +interrupted as soon as the next mission loads (`:STORAGE:INIT:`), and the web +service saves it as `partial`. Only if that is not confirmed within 30 seconds +does its spool stay behind for recovery. One that recorded nothing after its +start is discarded instead, when the web service supports it. A web service +without resume support is disconnected instead, which saves the mission as +`partial` there. When the addon ends a recording that does not meet its save limits (for example one shorter than `OCAP_settings_minMissionTime`), it sends diff --git a/cmd/ocap_recorder/main.go b/cmd/ocap_recorder/main.go index eb4c048..18b3f1c 100644 --- a/cmd/ocap_recorder/main.go +++ b/cmd/ocap_recorder/main.go @@ -367,7 +367,10 @@ func startNewMission(e dispatcher.Event) (any, error) { MarkerCache.Reset() EntityCache.Reset() - // 3. Start backend (handles DB ops for GORM/SQLite backends, stores missionID internally) + // 3. Start backend (handles DB ops for GORM/SQLite backends, stores missionID internally), + // once the previous mission's save or discard, dispatched before this + // start, has run. + saveState.waitIdle() if storageBackend != nil { if err := storageBackend.StartMission(&coreMission, &coreWorld); err != nil { Logger.Error("Failed to start mission in storage backend", "error", err) diff --git a/cmd/ocap_recorder/main_test.go b/cmd/ocap_recorder/main_test.go index 25972a8..417bd22 100644 --- a/cmd/ocap_recorder/main_test.go +++ b/cmd/ocap_recorder/main_test.go @@ -1,8 +1,8 @@ package main import ( - "log/slog" "testing" + "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -13,15 +13,25 @@ import ( "github.com/OCAP2/extension/v5/pkg/core" ) -func TestHandleNewMission_FailureIsReported(t *testing.T) { +// isolateMissionStart records callbacks and gives a test its own parser and +// no storage. Logger is left alone: the package init still reads it. +func isolateMissionStart(t *testing.T, p parser.Service) *callbackRecorder { + t.Helper() rec := &callbackRecorder{} a3interface.SetCallbackSinkForTest(rec.sink) - t.Cleanup(func() { a3interface.SetCallbackSinkForTest(nil) }) - previousLogger, previousParser := Logger, parserService - t.Cleanup(func() { Logger, parserService = previousLogger, previousParser }) - Logger = slog.Default() + previousParser, previousBackend := parserService, storageBackend + t.Cleanup(func() { + a3interface.SetCallbackSinkForTest(nil) + parserService, storageBackend = previousParser, previousBackend + }) + parserService = p + storageBackend = nil + return rec +} + +func TestHandleNewMission_FailureIsReported(t *testing.T) { + rec := isolateMissionStart(t, nil) - parserService = nil _, err := handleNewMission(dispatcher.Event{Args: []string{"{}", "{}", `"tok-1"`}}) require.Error(t, err) cb := rec.wait(t, ":MISSION:FAILED:") @@ -46,15 +56,29 @@ func (acceptingParser) ParseMission([]string) (core.Mission, core.World, error) return core.Mission{MissionName: "M"}, core.World{WorldName: "W"}, nil } +func TestHandleNewMission_WaitsForSaveOrDiscard(t *testing.T) { + rec := isolateMissionStart(t, acceptingParser{}) + + // The previous mission's discard was dispatched just before this start. + saveState.inFlight.Store(true) + t.Cleanup(func() { saveState.inFlight.Store(false) }) + done := make(chan error, 1) + go func() { + _, err := handleNewMission(dispatcher.Event{Args: []string{"{}", "{}", `"tok-3"`}}) + done <- err + }() + select { + case <-done: + t.Fatal("the start ran before the discard finished") + case <-time.After(100 * time.Millisecond): + } + saveState.inFlight.Store(false) + require.NoError(t, <-done) + rec.wait(t, ":MISSION:OK:") +} + func TestHandleNewMission_OKEchoesToken(t *testing.T) { - rec := &callbackRecorder{} - a3interface.SetCallbackSinkForTest(rec.sink) - t.Cleanup(func() { a3interface.SetCallbackSinkForTest(nil) }) - previousLogger, previousParser, previousBackend := Logger, parserService, storageBackend - t.Cleanup(func() { Logger, parserService, storageBackend = previousLogger, previousParser, previousBackend }) - Logger = slog.Default() - parserService = acceptingParser{} - storageBackend = nil + rec := isolateMissionStart(t, acceptingParser{}) _, err := handleNewMission(dispatcher.Event{Args: []string{"{}", "{}", `"tok-2"`}}) require.NoError(t, err) @@ -70,13 +94,7 @@ func (panickingParser) ParseMission([]string) (core.Mission, core.World, error) } func TestHandleNewMission_PanicIsReported(t *testing.T) { - rec := &callbackRecorder{} - a3interface.SetCallbackSinkForTest(rec.sink) - t.Cleanup(func() { a3interface.SetCallbackSinkForTest(nil) }) - previousLogger, previousParser := Logger, parserService - t.Cleanup(func() { Logger, parserService = previousLogger, previousParser }) - Logger = slog.Default() - parserService = panickingParser{} + rec := isolateMissionStart(t, panickingParser{}) assert.Panics(t, func() { _, _ = handleNewMission(dispatcher.Event{}) }, "the dispatcher still sees the panic") cb := rec.wait(t, ":MISSION:FAILED:") diff --git a/cmd/ocap_recorder/mission_save.go b/cmd/ocap_recorder/mission_save.go index 1c7561b..7048382 100644 --- a/cmd/ocap_recorder/mission_save.go +++ b/cmd/ocap_recorder/mission_save.go @@ -22,6 +22,15 @@ type missionSaveState struct { // saveState is the package-level state used by the :MISSION:SAVE: handler. var saveState = &missionSaveState{} +// waitIdle waits until no save or discard is in flight. A mission start runs +// on its own handler, so without it, a discard dispatched just before the +// next start could reach the backend after it, and drop the new mission. +func (s *missionSaveState) waitIdle() { + for s.inFlight.Load() { + time.Sleep(10 * time.Millisecond) + } +} + // triggerMissionSave is the logic behind the :MISSION:SAVE: dispatcher handler, // extracted so it can be exercised in tests with a fake backend and a fake // api client. diff --git a/cmd/ocap_recorder/storage.go b/cmd/ocap_recorder/storage.go index 5c98d50..a652e7d 100644 --- a/cmd/ocap_recorder/storage.go +++ b/cmd/ocap_recorder/storage.go @@ -40,6 +40,14 @@ func initStorage() error { // instead of staying open in a backend nothing uses any more. if storageBackend != nil { Logger.Info("Storage already initialized; reusing it", "type", storageCfg.Type) + // A new mission loaded: one still recording was abandoned. + if a, ok := storageBackend.(storage.Abandonable); ok { + go func() { + if err := a.AbandonMission(); err != nil { + Logger.Warn("Failed to end the abandoned mission", "error", err) + } + }() + } if err := a3interface.WriteArmaCallback(ExtensionName, ":STORAGE:OK:", storageCfg.Type); err != nil { Logger.Warn("Failed to send STORAGE:OK callback", "error", err) } diff --git a/cmd/ocap_recorder/storage_test.go b/cmd/ocap_recorder/storage_test.go index 766ab33..c99d0bd 100644 --- a/cmd/ocap_recorder/storage_test.go +++ b/cmd/ocap_recorder/storage_test.go @@ -3,6 +3,7 @@ package main import ( "log/slog" "testing" + "time" "github.com/OCAP2/extension/v5/internal/config" "github.com/OCAP2/extension/v5/internal/dispatcher" @@ -49,13 +50,13 @@ func isolateStorage(t *testing.T) *callbackRecorder { t.Helper() rec := &callbackRecorder{} a3interface.SetCallbackSinkForTest(rec.sink) - previousLogger, previousBackend, previousDispatcher, previousWorkers := Logger, storageBackend, eventDispatcher, workerManager + // Logger is left alone: the package init still reads it. + previousBackend, previousDispatcher, previousWorkers := storageBackend, eventDispatcher, workerManager t.Cleanup(func() { a3interface.SetCallbackSinkForTest(nil) - Logger, storageBackend, eventDispatcher, workerManager = previousLogger, previousBackend, previousDispatcher, previousWorkers + storageBackend, eventDispatcher, workerManager = previousBackend, previousDispatcher, previousWorkers viper.Reset() }) - Logger = slog.Default() storageBackend = nil d, err := dispatcher.New(logging.NewDispatcherLogger(Logger)) require.NoError(t, err) @@ -86,6 +87,33 @@ func TestInitStorageReusesBackend(t *testing.T) { require.Equal(t, 2, oks, "every :STORAGE:INIT: is answered") } +// abandoningBackend is a fakeBackend that records AbandonMission calls. +type abandoningBackend struct { + fakeBackend + abandoned chan struct{} +} + +func (a *abandoningBackend) AbandonMission() error { + close(a.abandoned) + return nil +} + +func TestInitStorageAbandonsOpenMission(t *testing.T) { + isolateStorage(t) + viper.Set("storage.type", "websocket") + backend := &abandoningBackend{abandoned: make(chan struct{})} + storageBackend = backend + + // The next mission loads. + require.NoError(t, initStorage()) + select { + case <-backend.abandoned: + case <-time.After(2 * time.Second): + t.Fatal("the mission left open was not abandoned") + } + require.Same(t, storage.Backend(backend), storageBackend) +} + func TestStartStorageReportsFailure(t *testing.T) { rec := isolateStorage(t) viper.Set("storage.type", "not-real") diff --git a/internal/storage/storage.go b/internal/storage/storage.go index 98011e0..a754762 100644 --- a/internal/storage/storage.go +++ b/internal/storage/storage.go @@ -67,6 +67,16 @@ type Discardable interface { DiscardMission() error } +// Abandonable is an optional interface for storage backends that keep a +// mission open outside the process, such as the websocket stream. The next +// mission load ends a mission the game moved on from without saving or +// discarding it. +type Abandonable interface { + // AbandonMission ends the current mission, as interrupted. It is a no-op + // when no mission is being recorded. + AbandonMission() error +} + // ErrMissionKept means a discarded mission ended, but its storage keeps what // was recorded, as a backend without Discardable does. var ErrMissionKept = errors.New("storage keeps the discarded mission") diff --git a/internal/storage/websocket/discard_test.go b/internal/storage/websocket/discard_test.go index 9291b41..c631d9a 100644 --- a/internal/storage/websocket/discard_test.go +++ b/internal/storage/websocket/discard_test.go @@ -386,6 +386,64 @@ func TestAbandonedEmptyMissionIsDiscarded(t *testing.T) { assert.True(t, rs.mission(second).completed) } +func TestAbandonMissionWhenNextMissionLoads(t *testing.T) { + rs := newResumeServer(t) + spoolDir := t.TempDir() + b := New(recoveryConfig(rs.url(), spoolDir)) + require.NoError(t, b.Init()) + defer b.Close() + + require.NoError(t, b.AbandonMission(), "no-op without a mission") + id := startResumable(t, b) + recordStates(t, b, 1, 3) + require.NoError(t, b.AbandonMission()) + m := rs.mission(id) + require.True(t, m.completed) + assert.True(t, endPayload(t, m).Interrupted) + requireGone(t, filepath.Join(spoolDir, id)) + assert.False(t, b.recording()) + + next := startResumable(t, b) + require.NoError(t, b.EndMission("")) + assert.True(t, rs.mission(next).completed) +} + +func TestAbandonMissionOnLegacyServerDisconnects(t *testing.T) { + srv, ml := testServer(t) + defer srv.Close() + b := New(Config{URL: wsURL(srv), Secret: "s", SpoolDir: t.TempDir()}) + require.NoError(t, b.Init()) + defer b.Close() + + require.NoError(t, b.StartMission(&core.Mission{MissionName: "M"}, &core.World{WorldName: "W"})) + require.NoError(t, b.AddSoldier(&core.Soldier{ID: 1})) + require.NoError(t, b.AbandonMission()) + for _, env := range ml.all() { + require.NotEqual(t, streaming.TypeEndMission, env.Type, "a legacy server ignores interrupted and would save it whole") + } + assert.False(t, b.recording()) + require.NoError(t, b.StartMission(&core.Mission{MissionName: "N"}, &core.World{WorldName: "W"}), "the next mission dials anew") +} + +func TestRecoveryDefersOwnMissionOnServerWithoutRecovered(t *testing.T) { + fastRecovery(t) + rs := newResumeServer(t) + rs.noDrop.Store(true) // predates recovered starts: it would supersede the live mission + spoolDir := t.TempDir() + b := New(recoveryConfig(rs.url(), spoolDir)) + require.NoError(t, b.Init()) + defer b.Close() + + live := startResumable(t, b) + writeTestSpool(t, filepath.Join(spoolDir, "m-own"), "m-own", b.producerID, 3) + time.Sleep(spoolStaleAfter + 10*recoveryConfig("", "").RecoveryInterval) // several recovery passes + assert.Empty(t, rs.mission("m-own").received, "not resent while this process records") + assert.False(t, rs.partial(live)) + + require.NoError(t, b.EndMission("")) + require.Eventually(t, func() bool { return rs.mission("m-own").completed }, 5*time.Second, 10*time.Millisecond, "resent once it records none") +} + func TestAbandonedEmptyMissionOnServerWithoutDiscard(t *testing.T) { rs := newResumeServer(t) rs.noDrop.Store(true) diff --git a/internal/storage/websocket/recover.go b/internal/storage/websocket/recover.go index d77f2d4..1395a24 100644 --- a/internal/storage/websocket/recover.go +++ b/internal/storage/websocket/recover.go @@ -110,24 +110,23 @@ func (b *Backend) recoverSpools(done <-chan struct{}) { err = b.recoverMission(dir, done) finished := err == nil || errors.Is(err, errAlreadySaved) || errors.Is(err, errDiscardFinished) || errors.Is(err, errSpoolDiscarded) if finished { - // Marked while locked: a process taking the spool before it is - // gone deletes it instead of resending it. - _ = markSpoolDropped(dir) + // Marked and emptied while locked: a process taking the spool + // before it is gone deletes it instead of resending it. + _ = dropSpoolSegments(dir) } lock.release() // before removal: Windows cannot delete an open file + if finished { + _ = removeSpoolDir(dir) // or the next pass does + } switch { case err == nil: b.recovered.Add(1) b.logger().Info("WebSocket recovered mission from spool", "dir", dir) - _ = os.RemoveAll(dir) case errors.Is(err, errAlreadySaved): b.logger().Info("WebSocket spooled mission was already saved by the server; spool removed", "dir", dir) - _ = os.RemoveAll(dir) case errors.Is(err, errDiscardFinished): b.logger().Info("WebSocket discarded mission finished from spool; spool removed", "dir", dir) - _ = os.RemoveAll(dir) case errors.Is(err, errSpoolDiscarded): - _ = os.RemoveAll(dir) case errors.Is(err, errRecoveryDeferred): b.logger().Debug("WebSocket mission recovery waits until the current mission ends", "dir", dir) case errors.Is(err, errRecoveryUnsupported): diff --git a/internal/storage/websocket/resume_test.go b/internal/storage/websocket/resume_test.go index 6156e5b..c9b9651 100644 --- a/internal/storage/websocket/resume_test.go +++ b/internal/storage/websocket/resume_test.go @@ -126,7 +126,9 @@ func (rs *resumeServer) serve(c *ws.Conn) { break } if id != p.MissionID && other.producer == p.ProducerID && !other.completed && !(other.recovered && knowsRecovered) { - rs.partials[id] = other + if other.lastSeq > 1 || !knowsRecovered { + rs.partials[id] = other // a newer server drops one with nothing after its start + } delete(rs.missions, id) } } diff --git a/internal/storage/websocket/spool.go b/internal/storage/websocket/spool.go index e8af213..9f3c48c 100644 --- a/internal/storage/websocket/spool.go +++ b/internal/storage/websocket/spool.go @@ -302,7 +302,7 @@ func (s *spool) close() error { // another process holds open for a moment is not an error. func (s *spool) remove() error { _ = s.shut(true) - if err := os.RemoveAll(s.dir); err != nil { + if err := removeSpoolDir(s.dir); err != nil { if spoolDropped(s.dir) { s.logger.Debug("WebSocket spool left for recovery to delete", "dir", s.dir, "error", err) return nil @@ -312,6 +312,53 @@ func (s *spool) remove() error { return nil } +// removeSpoolDir deletes a dropped spool's directory. The dropped marker goes +// last: while anything else is left, recovery still knows not to resend it. +func removeSpoolDir(dir string) error { + entries, err := os.ReadDir(dir) + if err != nil { + if os.IsNotExist(err) { + return nil + } + return err + } + var first error + for _, e := range entries { + if e.Name() == spoolDroppedFile { + continue + } + if err := os.RemoveAll(filepath.Join(dir, e.Name())); err != nil && first == nil { + first = err + } + } + if first != nil { + return first + } + if err := os.Remove(filepath.Join(dir, spoolDroppedFile)); err != nil && !os.IsNotExist(err) { + return err + } + return os.Remove(dir) +} + +// dropSpoolSegments marks the spool in dir dropped and deletes its segments. +// Call it while holding the spool's lock. +func dropSpoolSegments(dir string) error { + if err := markSpoolDropped(dir); err != nil { + return err + } + paths, _, err := spoolSegments(dir) + if err != nil { + return err + } + var first error + for _, p := range paths { + if err := os.Remove(p); err != nil && first == nil { + first = err + } + } + return first +} + // spoolDroppedFile marks a spool whose mission needs nothing more; recovery // deletes it. const spoolDroppedFile = "dropped" diff --git a/internal/storage/websocket/websocket.go b/internal/storage/websocket/websocket.go index 14a5637..1fbc3b9 100644 --- a/internal/storage/websocket/websocket.go +++ b/internal/storage/websocket/websocket.go @@ -213,23 +213,9 @@ func (b *Backend) StartMission(mission *core.Mission, world *core.World) error { b.missionMu.Lock() defer b.missionMu.Unlock() - // The game moved on without saving or discarding the previous mission. - // It ends as interrupted, so the server saves what it got as partial and - // nothing is left to recover, and this mission gets a fresh connection. - // One that recorded nothing after its start is dropped instead, when the - // server can. - if b.recording() { - b.mu.Lock() - old, oldID := b.conn, b.missionID - b.mu.Unlock() - if old.log.lastSeq() <= 1 && b.discardable() { - slog.Default().Warn("WebSocket mission abandoned before it recorded anything; discarding it", "missionId", oldID) - _ = b.discardMission(abandonTimeout) - } else { - slog.Default().Warn("WebSocket mission abandoned without save or discard; ending it as interrupted", "missionId", oldID) - _ = b.endMission(streaming.EndMissionPayload{Interrupted: true}, abandonTimeout) - } - } + // Normally done when the next mission loads (AbandonMission); this + // covers a start without one. + b.abandonMission() b.mu.Lock() // A completed mission's server closes its socket. Dial a fresh connection @@ -309,6 +295,43 @@ func (b *Backend) StartMission(mission *core.Mission, world *core.World) error { return nil } +// AbandonMission ends a mission the game moved on from without saving or +// discarding it, for example after an admin changed the mission. It ends as +// interrupted, so the server saves what it got as partial and nothing is left +// to recover. One that recorded nothing after its start is dropped instead, +// when the server can. A server without resume support saves a mission as +// partial when its producer disconnects, so the connection is closed instead. +// It is a no-op when no mission is recording. +func (b *Backend) AbandonMission() error { + b.missionMu.Lock() + defer b.missionMu.Unlock() + return b.abandonMission() +} + +// abandonMission is AbandonMission with missionMu held. +func (b *Backend) abandonMission() error { + if !b.recording() { + return nil + } + b.mu.Lock() + conn, missionID := b.conn, b.missionID + b.mu.Unlock() + switch { + case !conn.isResumable(): + slog.Default().Warn("WebSocket mission abandoned without save or discard; disconnecting, the server saves it as partial", "missionId", missionID) + if sp := b.clearMission(); sp != nil { + removeSpool(sp) + } + return conn.close() + case conn.log.lastSeq() <= 1 && b.discardable(): + slog.Default().Warn("WebSocket mission abandoned before it recorded anything; discarding it", "missionId", missionID) + return b.discardMission(abandonTimeout) + default: + slog.Default().Warn("WebSocket mission abandoned without save or discard; ending it as interrupted", "missionId", missionID) + return b.endMission(streaming.EndMissionPayload{Interrupted: true}, abandonTimeout) + } +} + // discardable reports whether the server serving the mission accepts // discard_mission, as it last announced. func (b *Backend) discardable() bool { @@ -393,12 +416,14 @@ func (b *Backend) discardMission(timeout time.Duration) error { // version, and refuses a discard already waiting for its ack. discardable := b.discardable() err := errDiscardUnsupported + logged := false // discard_mission is in the log, and so in a healthy spool if discardable { payload := streaming.DiscardMissionPayload{MissionID: missionID, ProducerID: b.producerID} err = conn.send(func(seq uint64) ([]byte, error) { return marshalEnvelope(streaming.TypeDiscardMission, payload, seq) }) if err == nil { + logged = true _, err = conn.waitAck(streaming.TypeDiscardMission, timeout) } if errors.Is(err, errDiscardUnsupported) { @@ -409,7 +434,7 @@ func (b *Backend) discardMission(timeout time.Duration) error { // A spool that holds discard_mission lets recovery finish the discard. // Without it, recovery would resend the mission and the server save it. sp := b.clearMission() - keep := err != nil && discardable && sp != nil && sp.healthy() + keep := err != nil && discardable && logged && sp != nil && sp.healthy() if sp != nil { if keep { _ = sp.close() From 3c4fd911374a6d70415944986c4f3afa49f64207 Mon Sep 17 00:00:00 2001 From: Smith <13200576+smitt14ua@users.noreply.github.com> Date: Tue, 27 Oct 2026 01:59:03 +0200 Subject: [PATCH 07/10] fix: abandon before answering storage init, let a start wait only for the backend Sixth review round: - A repeated :STORAGE:INIT: ends the abandoned mission before it answers :STORAGE:OK:, so the new mission's start cannot reach the backend first and be abandoned in its place. - :MISSION:START: waits until a dispatched save or discard has ended the mission in the backend, not for the upload after a save. - The downgrade test waits for the discard to be logged instead of sleeping; Abandonable's doc mentions the empty-mission discard. --- cmd/ocap_recorder/main_test.go | 6 +++--- cmd/ocap_recorder/mission_save.go | 23 +++++++++++++++++----- cmd/ocap_recorder/storage.go | 12 +++++------ internal/storage/storage.go | 5 +++-- internal/storage/websocket/discard_test.go | 6 ++++-- 5 files changed, 34 insertions(+), 18 deletions(-) diff --git a/cmd/ocap_recorder/main_test.go b/cmd/ocap_recorder/main_test.go index 417bd22..c80054b 100644 --- a/cmd/ocap_recorder/main_test.go +++ b/cmd/ocap_recorder/main_test.go @@ -60,8 +60,8 @@ func TestHandleNewMission_WaitsForSaveOrDiscard(t *testing.T) { rec := isolateMissionStart(t, acceptingParser{}) // The previous mission's discard was dispatched just before this start. - saveState.inFlight.Store(true) - t.Cleanup(func() { saveState.inFlight.Store(false) }) + saveState.backendBusy.Store(true) + t.Cleanup(func() { saveState.backendBusy.Store(false) }) done := make(chan error, 1) go func() { _, err := handleNewMission(dispatcher.Event{Args: []string{"{}", "{}", `"tok-3"`}}) @@ -72,7 +72,7 @@ func TestHandleNewMission_WaitsForSaveOrDiscard(t *testing.T) { t.Fatal("the start ran before the discard finished") case <-time.After(100 * time.Millisecond): } - saveState.inFlight.Store(false) + saveState.backendBusy.Store(false) require.NoError(t, <-done) rec.wait(t, ":MISSION:OK:") } diff --git a/cmd/ocap_recorder/mission_save.go b/cmd/ocap_recorder/mission_save.go index 7048382..95c51a5 100644 --- a/cmd/ocap_recorder/mission_save.go +++ b/cmd/ocap_recorder/mission_save.go @@ -17,16 +17,20 @@ import ( // There is exactly one instance, owned by the package-level saveState variable. type missionSaveState struct { inFlight atomic.Bool + // backendBusy is set while a dispatched save or discard has not yet + // ended the mission in the backend; an upload after it does not count. + backendBusy atomic.Bool } // saveState is the package-level state used by the :MISSION:SAVE: handler. var saveState = &missionSaveState{} -// waitIdle waits until no save or discard is in flight. A mission start runs -// on its own handler, so without it, a discard dispatched just before the -// next start could reach the backend after it, and drop the new mission. +// waitIdle waits until a dispatched save or discard has ended the mission in +// the backend. A mission start runs on its own handler, so without it, a +// discard dispatched just before the next start could reach the backend +// after it, and drop the new mission. func (s *missionSaveState) waitIdle() { - for s.inFlight.Load() { + for s.backendBusy.Load() { time.Sleep(10 * time.Millisecond) } } @@ -51,6 +55,7 @@ func triggerMissionSave(state *missionSaveState, backend storage.Backend, client if !state.inFlight.CompareAndSwap(false, true) { return nil, fmt.Errorf("save already in progress") } + state.backendBusy.Store(true) go runMissionSaveWorker(state, backend, client, tag) return "queued", nil @@ -72,6 +77,7 @@ func triggerMissionDiscard(state *missionSaveState, backend storage.Backend) (an if !state.inFlight.CompareAndSwap(false, true) { return nil, fmt.Errorf("mission save or discard already in progress") } + state.backendBusy.Store(true) go runMissionDiscardWorker(state, backend) return "queued", nil @@ -91,9 +97,11 @@ func runMissionDiscardWorker(state *missionSaveState, backend storage.Backend) { } } }() + defer state.backendBusy.Store(false) return d.DiscardMission() }() } + state.backendBusy.Store(false) // a backend that keeps the mission did nothing flushOTel() // Cleared before the callback: the addon may start the next mission's @@ -156,7 +164,12 @@ func runMissionSaveWorker(state *missionSaveState, backend storage.Backend, clie } }() - if err := backend.EndMission(tag); err != nil { + // The next mission may start once the backend ended this one; the + // upload below does not hold it up. + if err := func() error { + defer state.backendBusy.Store(false) + return backend.EndMission(tag) + }(); err != nil { endErr = err if Logger != nil { Logger.Error("Failed to end mission in storage backend", "error", err) diff --git a/cmd/ocap_recorder/storage.go b/cmd/ocap_recorder/storage.go index a652e7d..c0c2847 100644 --- a/cmd/ocap_recorder/storage.go +++ b/cmd/ocap_recorder/storage.go @@ -40,13 +40,13 @@ func initStorage() error { // instead of staying open in a backend nothing uses any more. if storageBackend != nil { Logger.Info("Storage already initialized; reusing it", "type", storageCfg.Type) - // A new mission loaded: one still recording was abandoned. + // A new mission loaded: one still recording was abandoned. It ends + // before :STORAGE:OK:, so the new mission's start cannot come first + // and be taken for it. This runs on startStorage's goroutine. if a, ok := storageBackend.(storage.Abandonable); ok { - go func() { - if err := a.AbandonMission(); err != nil { - Logger.Warn("Failed to end the abandoned mission", "error", err) - } - }() + if err := a.AbandonMission(); err != nil { + Logger.Warn("Failed to end the abandoned mission", "error", err) + } } if err := a3interface.WriteArmaCallback(ExtensionName, ":STORAGE:OK:", storageCfg.Type); err != nil { Logger.Warn("Failed to send STORAGE:OK callback", "error", err) diff --git a/internal/storage/storage.go b/internal/storage/storage.go index a754762..91ac78b 100644 --- a/internal/storage/storage.go +++ b/internal/storage/storage.go @@ -72,8 +72,9 @@ type Discardable interface { // mission load ends a mission the game moved on from without saving or // discarding it. type Abandonable interface { - // AbandonMission ends the current mission, as interrupted. It is a no-op - // when no mission is being recorded. + // AbandonMission ends the current mission, as interrupted, or drops it + // when it recorded nothing. It is a no-op when no mission is being + // recorded. AbandonMission() error } diff --git a/internal/storage/websocket/discard_test.go b/internal/storage/websocket/discard_test.go index c631d9a..e120575 100644 --- a/internal/storage/websocket/discard_test.go +++ b/internal/storage/websocket/discard_test.go @@ -476,8 +476,10 @@ func TestPendingDiscardRefusedAfterServerDowngrade(t *testing.T) { done := make(chan error, 1) start := time.Now() go func() { done <- b.DiscardMission() }() - require.Eventually(t, func() bool { return rs.lastSeq(id) == 4 }, time.Second, 5*time.Millisecond) - time.Sleep(50 * time.Millisecond) // the discard is logged and written into the dead link + b.mu.Lock() + conn := b.conn + b.mu.Unlock() + require.Eventually(t, func() bool { return conn.log.lastSeq() == 5 }, time.Second, time.Millisecond, "the discard is logged") rs.noDrop.Store(true) rs.discard.Store(false) rs.kill() From 4d419f2ad4fd9461763d0ec6edd0a796146ada6b Mon Sep 17 00:00:00 2001 From: Smith <13200576+smitt14ua@users.noreply.github.com> Date: Tue, 27 Oct 2026 02:24:26 +0200 Subject: [PATCH 08/10] fix: refuse a start of an unloaded mission, do not wait on a mission being ended at load Seventh review round: - :STORAGE:INIT: records when the mission loaded; a :MISSION:START: dispatched before it, which waited behind the previous mission's save, is answered with :MISSION:FAILED: instead of opening a mission nothing records. - AbandonMission leaves a mission a save or discard is still ending to it, instead of holding up the next mission's :STORAGE:OK: until that server answers. - Recovery's resume carries "recovered" (new ResumePayload.Recovered), and the own-mission check against old servers is repeated right before a full resend's start. - The save worker reads the export before the next start can reset it. - Comments and README: the abandon happens at mission load; an unconfirmed discard needs a healthy spool. --- README.md | 4 +++- cmd/ocap_recorder/main.go | 12 ++++++++++ cmd/ocap_recorder/main_test.go | 19 ++++++++++++++++ cmd/ocap_recorder/mission_save.go | 26 +++++++++++++++------- cmd/ocap_recorder/storage.go | 4 ++-- cmd/ocap_recorder/storage_test.go | 2 +- internal/storage/websocket/discard_test.go | 21 +++++++++++++++++ internal/storage/websocket/recover.go | 9 ++++++-- internal/storage/websocket/websocket.go | 13 ++++++++++- pkg/streaming/messages.go | 5 +++++ 10 files changed, 100 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 09067cb..b3b8caa 100644 --- a/README.md +++ b/README.md @@ -168,7 +168,9 @@ while it records none: - A mission whose save was not confirmed is saved as recorded. - A mission whose discard was not confirmed is not resent; the web service is only told again to drop it, including any `partial` recording it saved of - the mission in the meantime. + the mission in the meantime. This needs the spool: without `spoolDir`, or + with a spool that hit its limit or failed, nothing is kept, and the web + service saves the mission as `partial` once its resume grace expires. When the game moves on to the next mission without saving or discarding the current one, for example after an admin `#missions`, the proxy ends it as diff --git a/cmd/ocap_recorder/main.go b/cmd/ocap_recorder/main.go index 18b3f1c..b40c115 100644 --- a/cmd/ocap_recorder/main.go +++ b/cmd/ocap_recorder/main.go @@ -14,6 +14,7 @@ import ( "path/filepath" "runtime" "sync" + "sync/atomic" "time" "github.com/OCAP2/extension/v5/internal/api" @@ -91,6 +92,9 @@ var ( // storageReady is closed when storage (DB or memory) is initialized and ready storageReady = make(chan struct{}) storageReadyOnce sync.Once + // missionLoadedAt (unix nanos) is when the last :STORAGE:INIT:, sent + // on every mission load, was dispatched. + missionLoadedAt atomic.Int64 // Services parserService parser.Service @@ -371,6 +375,11 @@ func startNewMission(e dispatcher.Event) (any, error) { // once the previous mission's save or discard, dispatched before this // start, has run. saveState.waitIdle() + // A start that waited behind a save while the game loaded the next + // mission belongs to the mission that is gone: nothing would record it. + if !e.Timestamp.IsZero() && e.Timestamp.UnixNano() < missionLoadedAt.Load() { + return nil, fmt.Errorf("the mission was unloaded before it started") + } if storageBackend != nil { if err := storageBackend.StartMission(&coreMission, &coreWorld); err != nil { Logger.Error("Failed to start mission in storage backend", "error", err) @@ -391,6 +400,9 @@ func registerLifecycleHandlers(d *dispatcher.Dispatcher) { }) d.Register(":STORAGE:INIT:", func(e dispatcher.Event) (any, error) { + // A mission loaded: a start dispatched before it belongs to the + // previous one (see startNewMission). + missionLoadedAt.Store(e.Timestamp.UnixNano()) go startStorage() return nil, nil }) diff --git a/cmd/ocap_recorder/main_test.go b/cmd/ocap_recorder/main_test.go index c80054b..db774a5 100644 --- a/cmd/ocap_recorder/main_test.go +++ b/cmd/ocap_recorder/main_test.go @@ -77,6 +77,25 @@ func TestHandleNewMission_WaitsForSaveOrDiscard(t *testing.T) { rec.wait(t, ":MISSION:OK:") } +func TestHandleNewMission_RejectsStartOfUnloadedMission(t *testing.T) { + rec := isolateMissionStart(t, acceptingParser{}) + previous := missionLoadedAt.Load() + t.Cleanup(func() { missionLoadedAt.Store(previous) }) + + // The start was dispatched, then the next mission loaded while it waited. + dispatched := time.Now() + missionLoadedAt.Store(dispatched.Add(time.Second).UnixNano()) + _, err := handleNewMission(dispatcher.Event{Args: []string{"{}", "{}", `"tok-4"`}, Timestamp: dispatched}) + require.Error(t, err) + cb := rec.wait(t, ":MISSION:FAILED:") + assert.Equal(t, `"tok-4"`, cb.data[1]) + + // A start dispatched after the load goes through. + _, err = handleNewMission(dispatcher.Event{Args: []string{"{}", "{}", `"tok-5"`}, Timestamp: dispatched.Add(2 * time.Second)}) + require.NoError(t, err) + rec.wait(t, ":MISSION:OK:") +} + func TestHandleNewMission_OKEchoesToken(t *testing.T) { rec := isolateMissionStart(t, acceptingParser{}) diff --git a/cmd/ocap_recorder/mission_save.go b/cmd/ocap_recorder/mission_save.go index 95c51a5..2e429d8 100644 --- a/cmd/ocap_recorder/mission_save.go +++ b/cmd/ocap_recorder/mission_save.go @@ -11,6 +11,7 @@ import ( "github.com/OCAP2/extension/v5/internal/api" "github.com/OCAP2/extension/v5/internal/storage" "github.com/OCAP2/extension/v5/pkg/a3interface" + "github.com/OCAP2/extension/v5/pkg/core" ) // missionSaveState holds the cross-call state for the async save worker. @@ -165,10 +166,20 @@ func runMissionSaveWorker(state *missionSaveState, backend storage.Backend, clie }() // The next mission may start once the backend ended this one; the - // upload below does not hold it up. + // upload below does not hold it up. The export is read first: the + // next start resets it. + u, uploadable := backend.(storage.Uploadable) + var exportPath string + var exportMeta core.UploadMetadata if err := func() error { defer state.backendBusy.Store(false) - return backend.EndMission(tag) + if err := backend.EndMission(tag); err != nil { + return err + } + if uploadable { + exportPath, exportMeta = u.GetExportedFilePath(), u.GetExportMetadata() + } + return nil }(); err != nil { endErr = err if Logger != nil { @@ -182,11 +193,10 @@ func runMissionSaveWorker(state *missionSaveState, backend storage.Backend, clie ) } - if u, ok := backend.(storage.Uploadable); ok && client != nil { - if path := u.GetExportedFilePath(); path != "" { + if uploadable && client != nil { + if path := exportPath; path != "" { savedPath = path - meta := u.GetExportMetadata() - if err := client.Upload(path, meta); err != nil { + if err := client.Upload(path, exportMeta); err != nil { uploadErr = err if Logger != nil { Logger.Error("Failed to upload to OCAP web", @@ -201,9 +211,9 @@ func runMissionSaveWorker(state *missionSaveState, backend storage.Backend, clie ) } } - } else if u, ok := backend.(storage.Uploadable); ok { + } else if uploadable { // Client not configured — still report the path so the addon can inform players. - savedPath = u.GetExportedFilePath() + savedPath = exportPath } }() diff --git a/cmd/ocap_recorder/storage.go b/cmd/ocap_recorder/storage.go index c0c2847..209e1db 100644 --- a/cmd/ocap_recorder/storage.go +++ b/cmd/ocap_recorder/storage.go @@ -36,8 +36,8 @@ func initStorage() error { storageCfg := config.GetStorageConfig() // The backend lives as long as the process: the next mission reuses it, - // so a mission the game moved on from is ended by the next StartMission - // instead of staying open in a backend nothing uses any more. + // so a mission the game moved on from is ended here instead of staying + // open in a backend nothing uses any more. if storageBackend != nil { Logger.Info("Storage already initialized; reusing it", "type", storageCfg.Type) // A new mission loaded: one still recording was abandoned. It ends diff --git a/cmd/ocap_recorder/storage_test.go b/cmd/ocap_recorder/storage_test.go index c99d0bd..3999a0a 100644 --- a/cmd/ocap_recorder/storage_test.go +++ b/cmd/ocap_recorder/storage_test.go @@ -74,7 +74,7 @@ func TestInitStorageReusesBackend(t *testing.T) { require.NotNil(t, first) // The addon initializes storage again on the next mission load. require.NoError(t, initStorage()) - require.Same(t, first, storageBackend, "a mission left open is ended by the next start, in the same backend") + require.Same(t, first, storageBackend, "the next mission reuses the backend that ends the one left open") rec.mu.Lock() defer rec.mu.Unlock() diff --git a/internal/storage/websocket/discard_test.go b/internal/storage/websocket/discard_test.go index e120575..0e53e22 100644 --- a/internal/storage/websocket/discard_test.go +++ b/internal/storage/websocket/discard_test.go @@ -408,6 +408,27 @@ func TestAbandonMissionWhenNextMissionLoads(t *testing.T) { assert.True(t, rs.mission(next).completed) } +func TestAbandonMissionLeavesMissionBeingEnded(t *testing.T) { + oldAck := ackTimeout + ackTimeout = 100 * time.Millisecond + t.Cleanup(func() { ackTimeout = oldAck }) + rs := newResumeServer(t) + b := New(fastConfig(rs.url())) + require.NoError(t, b.Init()) + defer b.Close() + id := startResumable(t, b) + + // A save holds the mission while it waits for its server. + b.missionMu.Lock() + start := time.Now() + require.NoError(t, b.AbandonMission()) + assert.Less(t, time.Since(start), 2*time.Second, "the next mission load is not held up") + b.missionMu.Unlock() + assert.True(t, b.recording(), "left to the save that ends it") + assert.False(t, rs.mission(id).completed) + require.NoError(t, b.EndMission("")) +} + func TestAbandonMissionOnLegacyServerDisconnects(t *testing.T) { srv, ml := testServer(t) defer srv.Close() diff --git a/internal/storage/websocket/recover.go b/internal/storage/websocket/recover.go index 1395a24..766a878 100644 --- a/internal/storage/websocket/recover.go +++ b/internal/storage/websocket/recover.go @@ -278,7 +278,7 @@ func (b *Backend) recoverMission(dir string, done <-chan struct{}) error { } }() - resume, err := marshalEnvelope(streaming.TypeResume, streaming.ResumePayload{MissionID: payload.MissionID, ProducerID: payload.ProducerID}, 0) + resume, err := marshalEnvelope(streaming.TypeResume, streaming.ResumePayload{MissionID: payload.MissionID, ProducerID: payload.ProducerID, Recovered: true}, 0) if err != nil { return err } @@ -389,8 +389,13 @@ func (b *Backend) recoverMission(dir string, done <-chan struct{}) error { } if first { // A resent start must not take the producer's live mission - // for one it left behind. + // for one it left behind. An older server ignores the mark: + // checked again right before the start, as a mission may have + // started during the handshake. first = false + if payload.ProducerID == b.producerID && b.recording() && !b.discardable() { + return errRecoveryDeferred + } var err error if data, err = markRecovered(data); err != nil { return fmt.Errorf("mark start recovered: %w", err) diff --git a/internal/storage/websocket/websocket.go b/internal/storage/websocket/websocket.go index 1fbc3b9..0e361d8 100644 --- a/internal/storage/websocket/websocket.go +++ b/internal/storage/websocket/websocket.go @@ -302,8 +302,19 @@ func (b *Backend) StartMission(mission *core.Mission, world *core.World) error { // when the server can. A server without resume support saves a mission as // partial when its producer disconnects, so the connection is closed instead. // It is a no-op when no mission is recording. +// +// A save or discard that holds the mission for longer than a start does is +// ending it anyway, and the next mission load is not held up waiting for its +// server: AbandonMission then leaves the mission to it. func (b *Backend) AbandonMission() error { - b.missionMu.Lock() + deadline := time.Now().Add(ackTimeout + time.Second) + for !b.missionMu.TryLock() { + if time.Now().After(deadline) { + slog.Default().Info("WebSocket mission is being ended; not abandoning it") + return nil + } + time.Sleep(10 * time.Millisecond) + } defer b.missionMu.Unlock() return b.abandonMission() } diff --git a/pkg/streaming/messages.go b/pkg/streaming/messages.go index 17139ba..ed9852a 100644 --- a/pkg/streaming/messages.go +++ b/pkg/streaming/messages.go @@ -145,6 +145,11 @@ type StartMissionPayload struct { type ResumePayload struct { MissionID string `json:"missionId"` ProducerID string `json:"producerId"` + // Recovered marks a resume from the spool, for a mission the producer + // no longer records; like StartMissionPayload.Recovered, it keeps the + // session from being taken for one the producer's next start + // supersedes. + Recovered bool `json:"recovered,omitempty"` } // EndMissionPayload carries finalization hints for end_mission. From 9cac5c36cc3d56464c60298761552d82dc948937 Mon Sep 17 00:00:00 2001 From: Smith <13200576+smitt14ua@users.noreply.github.com> Date: Tue, 27 Oct 2026 03:01:48 +0200 Subject: [PATCH 09/10] fix: never block the game thread on a queued start, compare load times monotonically Eighth review round: - :MISSION:START: is no longer a blocking queue. A start can wait behind the previous mission's save, and with the addon's re-registration a third start filled the queue and froze the game's thread; a full queue now refuses the start, which the addon reports. - The unloaded-mission check compares event times with Before, on the monotonic clock, so a wall clock stepping back cannot refuse a valid start. - Storage init skips the abandon only while a dispatched save or discard is ending the mission, and otherwise waits for the mission lock; a slow start could make the old time-bounded wait give up and leave a mission open. - Tests: :STORAGE:OK: follows the abandon; a dispatched save is left to end the mission. ErrMissionKept's doc. --- cmd/ocap_recorder/main.go | 17 ++++++--- cmd/ocap_recorder/main_test.go | 3 +- cmd/ocap_recorder/storage.go | 6 ++- cmd/ocap_recorder/storage_test.go | 43 ++++++++++++++++------ internal/storage/storage.go | 4 +- internal/storage/websocket/discard_test.go | 21 ----------- internal/storage/websocket/websocket.go | 13 +------ 7 files changed, 52 insertions(+), 55 deletions(-) diff --git a/cmd/ocap_recorder/main.go b/cmd/ocap_recorder/main.go index b40c115..2af40aa 100644 --- a/cmd/ocap_recorder/main.go +++ b/cmd/ocap_recorder/main.go @@ -92,9 +92,10 @@ var ( // storageReady is closed when storage (DB or memory) is initialized and ready storageReady = make(chan struct{}) storageReadyOnce sync.Once - // missionLoadedAt (unix nanos) is when the last :STORAGE:INIT:, sent - // on every mission load, was dispatched. - missionLoadedAt atomic.Int64 + // missionLoadedAt is when the last :STORAGE:INIT:, sent on every + // mission load, was dispatched. Compared with Before, which uses the + // monotonic clock, so a wall clock stepping back does not matter. + missionLoadedAt atomic.Pointer[time.Time] // Services parserService parser.Service @@ -377,7 +378,7 @@ func startNewMission(e dispatcher.Event) (any, error) { saveState.waitIdle() // A start that waited behind a save while the game loaded the next // mission belongs to the mission that is gone: nothing would record it. - if !e.Timestamp.IsZero() && e.Timestamp.UnixNano() < missionLoadedAt.Load() { + if loaded := missionLoadedAt.Load(); loaded != nil && !e.Timestamp.IsZero() && e.Timestamp.Before(*loaded) { return nil, fmt.Errorf("the mission was unloaded before it started") } if storageBackend != nil { @@ -402,7 +403,8 @@ func registerLifecycleHandlers(d *dispatcher.Dispatcher) { d.Register(":STORAGE:INIT:", func(e dispatcher.Event) (any, error) { // A mission loaded: a start dispatched before it belongs to the // previous one (see startNewMission). - missionLoadedAt.Store(e.Timestamp.UnixNano()) + loaded := e.Timestamp + missionLoadedAt.Store(&loaded) go startStorage() return nil, nil }) @@ -442,7 +444,10 @@ func registerLifecycleHandlers(d *dispatcher.Dispatcher) { return nil, nil }) - d.Register(":MISSION:START:", handleNewMission, dispatcher.Buffered(1), dispatcher.Blocking(), dispatcher.Gated(storageReady)) + // Not blocking: a start waits for a save or discard still ending the + // previous mission, and a queue it filled must not hold up the game's + // thread. A refused start is reported to the addon by callExtension. + d.Register(":MISSION:START:", handleNewMission, dispatcher.Buffered(1), dispatcher.Gated(storageReady)) d.Register(":MISSION:SAVE:", func(e dispatcher.Event) (any, error) { // Optional save-time tag override (addon exportData _tag). Sent as a diff --git a/cmd/ocap_recorder/main_test.go b/cmd/ocap_recorder/main_test.go index db774a5..a574d96 100644 --- a/cmd/ocap_recorder/main_test.go +++ b/cmd/ocap_recorder/main_test.go @@ -84,7 +84,8 @@ func TestHandleNewMission_RejectsStartOfUnloadedMission(t *testing.T) { // The start was dispatched, then the next mission loaded while it waited. dispatched := time.Now() - missionLoadedAt.Store(dispatched.Add(time.Second).UnixNano()) + loaded := dispatched.Add(time.Second) + missionLoadedAt.Store(&loaded) _, err := handleNewMission(dispatcher.Event{Args: []string{"{}", "{}", `"tok-4"`}, Timestamp: dispatched}) require.Error(t, err) cb := rec.wait(t, ":MISSION:FAILED:") diff --git a/cmd/ocap_recorder/storage.go b/cmd/ocap_recorder/storage.go index 209e1db..264b2eb 100644 --- a/cmd/ocap_recorder/storage.go +++ b/cmd/ocap_recorder/storage.go @@ -42,8 +42,10 @@ func initStorage() error { Logger.Info("Storage already initialized; reusing it", "type", storageCfg.Type) // A new mission loaded: one still recording was abandoned. It ends // before :STORAGE:OK:, so the new mission's start cannot come first - // and be taken for it. This runs on startStorage's goroutine. - if a, ok := storageBackend.(storage.Abandonable); ok { + // and be taken for it. This runs on startStorage's goroutine. A save + // or discard dispatched before is ending it already; waiting for its + // server would only hold up the next mission. + if a, ok := storageBackend.(storage.Abandonable); ok && !saveState.backendBusy.Load() { if err := a.AbandonMission(); err != nil { Logger.Warn("Failed to end the abandoned mission", "error", err) } diff --git a/cmd/ocap_recorder/storage_test.go b/cmd/ocap_recorder/storage_test.go index 3999a0a..2fdd788 100644 --- a/cmd/ocap_recorder/storage_test.go +++ b/cmd/ocap_recorder/storage_test.go @@ -3,7 +3,6 @@ package main import ( "log/slog" "testing" - "time" "github.com/OCAP2/extension/v5/internal/config" "github.com/OCAP2/extension/v5/internal/dispatcher" @@ -12,6 +11,7 @@ import ( "github.com/OCAP2/extension/v5/internal/storage/websocket" "github.com/OCAP2/extension/v5/pkg/a3interface" "github.com/spf13/viper" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -87,33 +87,54 @@ func TestInitStorageReusesBackend(t *testing.T) { require.Equal(t, 2, oks, "every :STORAGE:INIT: is answered") } -// abandoningBackend is a fakeBackend that records AbandonMission calls. +// abandoningBackend is a fakeBackend that records AbandonMission calls and +// whether :STORAGE:OK: had gone out by then. type abandoningBackend struct { fakeBackend - abandoned chan struct{} + rec *callbackRecorder + abandoned int + okBefore bool } func (a *abandoningBackend) AbandonMission() error { - close(a.abandoned) + a.abandoned++ + a.rec.mu.Lock() + defer a.rec.mu.Unlock() + for _, c := range a.rec.calls { + if c.function == ":STORAGE:OK:" { + a.okBefore = true + } + } return nil } func TestInitStorageAbandonsOpenMission(t *testing.T) { - isolateStorage(t) + rec := isolateStorage(t) viper.Set("storage.type", "websocket") - backend := &abandoningBackend{abandoned: make(chan struct{})} + backend := &abandoningBackend{rec: rec} storageBackend = backend // The next mission loads. require.NoError(t, initStorage()) - select { - case <-backend.abandoned: - case <-time.After(2 * time.Second): - t.Fatal("the mission left open was not abandoned") - } + assert.Equal(t, 1, backend.abandoned, "the mission left open is abandoned") + assert.False(t, backend.okBefore, "before :STORAGE:OK:, so the next start cannot come first") + rec.wait(t, ":STORAGE:OK:") require.Same(t, storage.Backend(backend), storageBackend) } +func TestInitStorageLeavesMissionToDispatchedSave(t *testing.T) { + rec := isolateStorage(t) + viper.Set("storage.type", "websocket") + backend := &abandoningBackend{rec: rec} + storageBackend = backend + saveState.backendBusy.Store(true) // a save dispatched before is ending it + t.Cleanup(func() { saveState.backendBusy.Store(false) }) + + require.NoError(t, initStorage()) + assert.Zero(t, backend.abandoned, "the next mission is not held up waiting for that save") + rec.wait(t, ":STORAGE:OK:") +} + func TestStartStorageReportsFailure(t *testing.T) { rec := isolateStorage(t) viper.Set("storage.type", "not-real") diff --git a/internal/storage/storage.go b/internal/storage/storage.go index 91ac78b..47385fd 100644 --- a/internal/storage/storage.go +++ b/internal/storage/storage.go @@ -78,8 +78,8 @@ type Abandonable interface { AbandonMission() error } -// ErrMissionKept means a discarded mission ended, but its storage keeps what -// was recorded, as a backend without Discardable does. +// ErrMissionKept means the storage keeps what was recorded of a discarded +// mission, as a backend without Discardable does. var ErrMissionKept = errors.New("storage keeps the discarded mission") // Uploadable is an optional interface for storage backends that produce diff --git a/internal/storage/websocket/discard_test.go b/internal/storage/websocket/discard_test.go index 0e53e22..e120575 100644 --- a/internal/storage/websocket/discard_test.go +++ b/internal/storage/websocket/discard_test.go @@ -408,27 +408,6 @@ func TestAbandonMissionWhenNextMissionLoads(t *testing.T) { assert.True(t, rs.mission(next).completed) } -func TestAbandonMissionLeavesMissionBeingEnded(t *testing.T) { - oldAck := ackTimeout - ackTimeout = 100 * time.Millisecond - t.Cleanup(func() { ackTimeout = oldAck }) - rs := newResumeServer(t) - b := New(fastConfig(rs.url())) - require.NoError(t, b.Init()) - defer b.Close() - id := startResumable(t, b) - - // A save holds the mission while it waits for its server. - b.missionMu.Lock() - start := time.Now() - require.NoError(t, b.AbandonMission()) - assert.Less(t, time.Since(start), 2*time.Second, "the next mission load is not held up") - b.missionMu.Unlock() - assert.True(t, b.recording(), "left to the save that ends it") - assert.False(t, rs.mission(id).completed) - require.NoError(t, b.EndMission("")) -} - func TestAbandonMissionOnLegacyServerDisconnects(t *testing.T) { srv, ml := testServer(t) defer srv.Close() diff --git a/internal/storage/websocket/websocket.go b/internal/storage/websocket/websocket.go index 0e361d8..1fbc3b9 100644 --- a/internal/storage/websocket/websocket.go +++ b/internal/storage/websocket/websocket.go @@ -302,19 +302,8 @@ func (b *Backend) StartMission(mission *core.Mission, world *core.World) error { // when the server can. A server without resume support saves a mission as // partial when its producer disconnects, so the connection is closed instead. // It is a no-op when no mission is recording. -// -// A save or discard that holds the mission for longer than a start does is -// ending it anyway, and the next mission load is not held up waiting for its -// server: AbandonMission then leaves the mission to it. func (b *Backend) AbandonMission() error { - deadline := time.Now().Add(ackTimeout + time.Second) - for !b.missionMu.TryLock() { - if time.Now().After(deadline) { - slog.Default().Info("WebSocket mission is being ended; not abandoning it") - return nil - } - time.Sleep(10 * time.Millisecond) - } + b.missionMu.Lock() defer b.missionMu.Unlock() return b.abandonMission() } From 6e18784428921bf20ebcc5bc22d6f899997c28c4 Mon Sep 17 00:00:00 2001 From: Smith <13200576+smitt14ua@users.noreply.github.com> Date: Tue, 27 Oct 2026 03:31:59 +0200 Subject: [PATCH 10/10] feat: drop an abandoned recording below the addon's save limits, never keep a spool whose discard missed the disk :MISSION:START: takes an optional fourth argument, "discard": if the game moves on from the mission (#missions, #restart) before :MISSION:KEEP:, the next mission load discards it instead of ending it as interrupted, so the web keeps no partial of it. The addon sends :MISSION:KEEP: once the recording reaches its minimum mission time. A spool kept to finish an unconfirmed discard is synced on close; if its last records do not reach the disk, it is removed under its lock instead, so recovery cannot resend the mission without its discard. --- CLAUDE.md | 1 + README.md | 9 ++- cmd/ocap_recorder/main.go | 18 ++++- cmd/ocap_recorder/main_test.go | 23 ++++++ cmd/ocap_recorder/storage_test.go | 10 ++- internal/storage/storage.go | 10 ++- internal/storage/websocket/discard_test.go | 92 ++++++++++++++++++++++ internal/storage/websocket/resume_test.go | 4 +- internal/storage/websocket/spool.go | 45 +++++++++-- internal/storage/websocket/websocket.go | 58 +++++++++----- 10 files changed, 238 insertions(+), 32 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index a91dffe..b16bb77 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -79,6 +79,7 @@ All commands follow a `:RESOURCE:ACTION:` naming convention. New commands must u | `:ACE3:DEATH:`, `:ACE3:UNCONSCIOUS:` | ACE3 integration events | | `:SYS:INIT:`, `:STORAGE:INIT:` | Initialize extension and storage | | `:MISSION:START:`, `:MISSION:SAVE:`, `:MISSION:DISCARD:` | Begin/end recording, or end it unsaved | +| `:MISSION:KEEP:` | Keep a recording the game moves on from (it met the addon's save limits) | ### Data Flow diff --git a/README.md b/README.md index b3b8caa..b5b2500 100644 --- a/README.md +++ b/README.md @@ -177,7 +177,11 @@ current one, for example after an admin `#missions`, the proxy ends it as interrupted as soon as the next mission loads (`:STORAGE:INIT:`), and the web service saves it as `partial`. Only if that is not confirmed within 30 seconds does its spool stay behind for recovery. One that recorded nothing after its -start is discarded instead, when the web service supports it. A web service +start is discarded instead, when the web service supports it, and so is one +the addon would not save yet: it starts such a recording with `"discard"` and +sends `:MISSION:KEEP:` once it meets `OCAP_settings_minMissionTime`, so a +mission an admin only loaded and left with `#missions` or `#restart` leaves +no `partial` behind. A web service without resume support is disconnected instead, which saves the mission as `partial` there. @@ -290,7 +294,8 @@ All commands follow a `:RESOURCE:ACTION:` naming convention. New commands must u |---------|---------| | `:SYS:INIT:` | Initialize extension, send `:SYS:READY:` callback | | `:STORAGE:INIT:` | Initialize storage backend once per process (later calls reuse it), ungate buffered handlers; answers with a `:STORAGE:OK:` callback, or `:STORAGE:ERROR:` with the error | -| `:MISSION:START:` | Start recording mission; answers with a `:MISSION:OK:` callback, or `:MISSION:FAILED:` with the error. An optional third argument, a registration token, is echoed as the callback's last value | +| `:MISSION:START:` | Start recording mission; answers with a `:MISSION:OK:` callback, or `:MISSION:FAILED:` with the error. An optional third argument, a registration token, is echoed as the callback's last value. An optional fourth, `discard`, drops the recording if the game moves on from it before `:MISSION:KEEP:` (`websocket`) | +| `:MISSION:KEEP:` | The recording met the addon's save limits: if the game moves on from it now, it is kept as `partial` (`websocket`) | | `:MISSION:SAVE:` | End recording, flush data, upload if configured | | `:MISSION:DISCARD:` | End recording without saving it (`memory`, `websocket`); answers with a `:MISSION:DISCARDED:` callback | | `:MISSION:FOCUS_START:` | Set playback focus start frame | diff --git a/cmd/ocap_recorder/main.go b/cmd/ocap_recorder/main.go index 2af40aa..fa2eca6 100644 --- a/cmd/ocap_recorder/main.go +++ b/cmd/ocap_recorder/main.go @@ -322,7 +322,8 @@ func initAPIClient() { // // An optional third argument is the addon's registration token, echoed in // both callbacks, so a late answer to an earlier mission's start is not taken -// for this one's. +// for this one's. An optional fourth, "discard", drops the recording if the +// game moves on from it before :MISSION:KEEP: (see storage.Abandonable). func handleNewMission(e dispatcher.Event) (result any, err error) { token := "" if len(e.Args) > 2 { @@ -386,6 +387,10 @@ func startNewMission(e dispatcher.Event) (any, error) { Logger.Error("Failed to start mission in storage backend", "error", err) return nil, err } + // Before :MISSION:OK:, so before the addon records anything. + if a, ok := storageBackend.(storage.Abandonable); ok && len(e.Args) > 3 && util.TrimQuotes(e.Args[3]) == "discard" { + a.SetDiscardAbandoned(true) + } } Logger.Info("New mission logged", "missionName", coreMission.MissionName) @@ -467,4 +472,15 @@ func registerLifecycleHandlers(d *dispatcher.Dispatcher) { Logger.Info("Received :MISSION:DISCARD: command, queuing async discard") return triggerMissionDiscard(saveState, storageBackend) }) + + // The recording met the addon's save limits: if the game moves on from + // it now, it is kept (as partial) instead of dropped. Synchronous, so it + // is in place before a mission load that follows it. + d.Register(":MISSION:KEEP:", func(e dispatcher.Event) (any, error) { + Logger.Info("Received :MISSION:KEEP: command; an abandoned recording is kept") + if a, ok := storageBackend.(storage.Abandonable); ok { + a.SetDiscardAbandoned(false) + } + return nil, nil + }) } diff --git a/cmd/ocap_recorder/main_test.go b/cmd/ocap_recorder/main_test.go index a574d96..ddb2bc0 100644 --- a/cmd/ocap_recorder/main_test.go +++ b/cmd/ocap_recorder/main_test.go @@ -8,6 +8,7 @@ import ( "github.com/stretchr/testify/require" "github.com/OCAP2/extension/v5/internal/dispatcher" + "github.com/OCAP2/extension/v5/internal/logging" "github.com/OCAP2/extension/v5/internal/parser" "github.com/OCAP2/extension/v5/pkg/a3interface" "github.com/OCAP2/extension/v5/pkg/core" @@ -106,6 +107,28 @@ func TestHandleNewMission_OKEchoesToken(t *testing.T) { assert.Equal(t, []string{`"OK"`, `"tok-2"`}, cb.data) } +func TestHandleNewMission_DiscardAbandonedUntilKept(t *testing.T) { + rec := isolateMissionStart(t, acceptingParser{}) + backend := &abandoningBackend{rec: rec} + storageBackend = backend + + _, err := handleNewMission(dispatcher.Event{Args: []string{"{}", "{}", `"tok-6"`}}) + require.NoError(t, err) + assert.Empty(t, backend.discard, "an addon that does not ask keeps an abandoned recording") + + _, err = handleNewMission(dispatcher.Event{Args: []string{"{}", "{}", `"tok-7"`, `"discard"`}}) + require.NoError(t, err) + assert.Equal(t, []bool{true}, backend.discard) + + // The recording met the addon's save limits. + d, err := dispatcher.New(logging.NewDispatcherLogger(Logger)) + require.NoError(t, err) + registerLifecycleHandlers(d) + _, err = d.Dispatch(dispatcher.Event{Command: ":MISSION:KEEP:"}) + require.NoError(t, err) + assert.Equal(t, []bool{true, false}, backend.discard) +} + // panickingParser fails every mission start with a panic. type panickingParser struct{ parser.Service } diff --git a/cmd/ocap_recorder/storage_test.go b/cmd/ocap_recorder/storage_test.go index 2fdd788..45d2b69 100644 --- a/cmd/ocap_recorder/storage_test.go +++ b/cmd/ocap_recorder/storage_test.go @@ -10,6 +10,7 @@ import ( "github.com/OCAP2/extension/v5/internal/storage" "github.com/OCAP2/extension/v5/internal/storage/websocket" "github.com/OCAP2/extension/v5/pkg/a3interface" + "github.com/OCAP2/extension/v5/pkg/core" "github.com/spf13/viper" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -88,12 +89,19 @@ func TestInitStorageReusesBackend(t *testing.T) { } // abandoningBackend is a fakeBackend that records AbandonMission calls and -// whether :STORAGE:OK: had gone out by then. +// whether :STORAGE:OK: had gone out by then, and SetDiscardAbandoned calls. type abandoningBackend struct { fakeBackend rec *callbackRecorder abandoned int okBefore bool + discard []bool +} + +func (a *abandoningBackend) StartMission(*core.Mission, *core.World) error { return nil } + +func (a *abandoningBackend) SetDiscardAbandoned(discard bool) { + a.discard = append(a.discard, discard) } func (a *abandoningBackend) AbandonMission() error { diff --git a/internal/storage/storage.go b/internal/storage/storage.go index 47385fd..84fe968 100644 --- a/internal/storage/storage.go +++ b/internal/storage/storage.go @@ -73,9 +73,15 @@ type Discardable interface { // discarding it. type Abandonable interface { // AbandonMission ends the current mission, as interrupted, or drops it - // when it recorded nothing. It is a no-op when no mission is being - // recorded. + // when it recorded nothing or SetDiscardAbandoned asked for it. It is a + // no-op when no mission is being recorded. AbandonMission() error + // SetDiscardAbandoned sets whether AbandonMission drops the current + // mission instead of ending it, for a recording the addon would not save + // yet (:MISSION:START: with "discard", until :MISSION:KEEP:). Each + // mission starts without it. It is a no-op when no mission is being + // recorded, and never waits for the backend. + SetDiscardAbandoned(discard bool) } // ErrMissionKept means the storage keeps what was recorded of a discarded diff --git a/internal/storage/websocket/discard_test.go b/internal/storage/websocket/discard_test.go index e120575..8230699 100644 --- a/internal/storage/websocket/discard_test.go +++ b/internal/storage/websocket/discard_test.go @@ -1,7 +1,9 @@ package websocket import ( + "bufio" "encoding/json" + "errors" "log/slog" "os" "path/filepath" @@ -408,6 +410,60 @@ func TestAbandonMissionWhenNextMissionLoads(t *testing.T) { assert.True(t, rs.mission(next).completed) } +func TestAbandonedMissionBelowSaveLimitsIsDiscarded(t *testing.T) { + rs := newResumeServer(t) + spoolDir := t.TempDir() + b := New(recoveryConfig(rs.url(), spoolDir)) + require.NoError(t, b.Init()) + defer b.Close() + + // An admin loads the mission, records a little and changes the mission. + short := startResumable(t, b) + b.SetDiscardAbandoned(true) + recordStates(t, b, 1, 3) + require.NoError(t, b.AbandonMission()) + m := rs.mission(short) + assert.True(t, m.discarded, "a recording the addon would not save is dropped") + requireGone(t, filepath.Join(spoolDir, short)) + + // One that met the save limits is kept as partial. + kept := startResumable(t, b) + b.SetDiscardAbandoned(true) + recordStates(t, b, 1, 3) + b.SetDiscardAbandoned(false) + require.NoError(t, b.AbandonMission()) + m = rs.mission(kept) + require.True(t, m.completed) + assert.True(t, endPayload(t, m).Interrupted) + requireNoDiscard(t, m.received) + + // Each mission starts keeping what it records. + b.SetDiscardAbandoned(true) // no mission: no-op + next := startResumable(t, b) + recordStates(t, b, 1, 3) + require.NoError(t, b.AbandonMission()) + m = rs.mission(next) + require.True(t, m.completed) + assert.True(t, endPayload(t, m).Interrupted) +} + +func TestAbandonedMissionBelowSaveLimitsOnServerWithoutDiscard(t *testing.T) { + rs := newResumeServer(t) + rs.noDrop.Store(true) + b := New(fastConfig(rs.url())) + require.NoError(t, b.Init()) + defer b.Close() + + id := startResumable(t, b) + b.SetDiscardAbandoned(true) + recordStates(t, b, 1, 3) + require.NoError(t, b.AbandonMission()) + m := rs.mission(id) + assert.True(t, m.completed, "ended, since this server cannot drop it") + requireNoDiscard(t, m.received) + assert.True(t, endPayload(t, m).Interrupted) +} + func TestAbandonMissionOnLegacyServerDisconnects(t *testing.T) { srv, ml := testServer(t) defer srv.Close() @@ -536,6 +592,42 @@ func TestAbandonedMissionUnconfirmedKeepsSpool(t *testing.T) { assert.True(t, rs.mission(second).completed) } +// failingWriter fails every write, as a full disk would. +type failingWriter struct{} + +func (failingWriter) Write([]byte) (int, error) { return 0, errors.New("no space left on device") } + +func TestUnconfirmedDiscardWithFailedFinalFlushRemovesSpool(t *testing.T) { + fastRecovery(t) + spoolFlushInterval = time.Hour // only the final flush writes the discard + rs := newResumeServer(t) + spoolDir := t.TempDir() + cfg := recoveryConfig(rs.url(), spoolDir) + cfg.SaveTimeout = 200 * time.Millisecond // below the ack stall, so no reconnect reads the spool + b := New(cfg) + require.NoError(t, b.Init()) + defer b.Close() + + id := startResumable(t, b) + recordStates(t, b, 1, 3) + require.Eventually(t, func() bool { return rs.lastSeq(id) == 4 }, 2*time.Second, 5*time.Millisecond) + b.mu.Lock() + sp := b.conn.log.currentSpool() + b.mu.Unlock() + sp.mu.Lock() + sp.flushLocked() + sp.w = bufio.NewWriter(failingWriter{}) // the next flush, the final one, fails + sp.mu.Unlock() + + rs.loseEnd.Store(true) // the discard never arrives + require.Error(t, b.DiscardMission()) + requireGone(t, filepath.Join(spoolDir, id)) + rs.loseEnd.Store(false) + time.Sleep(spoolStaleAfter + 5*cfg.RecoveryInterval) // several recovery passes + assert.Equal(t, int32(1), rs.starts.Load(), "recovery never resends the mission without its discard") + assert.False(t, rs.mission(id).completed) +} + func TestDiscardAfterServerDowngradeIsNotSent(t *testing.T) { rs := newResumeServer(t) b := New(fastConfig(rs.url())) diff --git a/internal/storage/websocket/resume_test.go b/internal/storage/websocket/resume_test.go index c9b9651..5cf1ddf 100644 --- a/internal/storage/websocket/resume_test.go +++ b/internal/storage/websocket/resume_test.go @@ -39,7 +39,7 @@ type resumeServer struct { legacy atomic.Bool // behave like a web server without resume support noDrop atomic.Bool // support resume but predate discard_mission muteStart atomic.Bool // never acknowledge start_mission - loseEnd atomic.Bool // drop end_mission as if lost in transit + loseEnd atomic.Bool // drop end_mission and discard_mission as if lost in transit resumes atomic.Int32 starts atomic.Int32 conns2ndP atomic.Int32 // connections that began with resume @@ -196,7 +196,7 @@ func (rs *resumeServer) serve(c *ws.Conn) { if m == nil || (env.Type == streaming.TypeDiscardMission && rs.noDrop.Load()) { continue // an older server ignores discard_mission } - if env.Type == streaming.TypeEndMission && rs.loseEnd.Load() { + if (env.Type == streaming.TypeEndMission || env.Type == streaming.TypeDiscardMission) && rs.loseEnd.Load() { continue } rs.mu.Lock() diff --git a/internal/storage/websocket/spool.go b/internal/storage/websocket/spool.go index 9f3c48c..a03d07b 100644 --- a/internal/storage/websocket/spool.go +++ b/internal/storage/websocket/spool.go @@ -292,7 +292,28 @@ func readSegment(path string, from, to uint64, maxBytes int, out *[]logEntry, by // close flushes and closes the spool, keeping its files. func (s *spool) close() error { - return s.shut(false) + _, err := s.shut(false, false) + return err +} + +// keepIntact closes a spool kept for recovery to finish the mission with the +// record it ends with, such as discard_mission. If any record, the last one +// included, did not reach the disk, the spool is removed instead, as remove +// does: recovery would resend the mission without it, and the server save +// it. It reports whether the spool was kept. +func (s *spool) keepIntact() bool { + dropped, err := s.shut(false, true) + if !dropped { + if err != nil { + s.logger.Warn("WebSocket spool closed with an error", "dir", s.dir, "error", err) + } + return true + } + s.logger.Warn("WebSocket spool incomplete on disk; removing it so recovery does not resend the mission", "dir", s.dir, "error", err) + if rerr := removeSpoolDir(s.dir); rerr != nil && !spoolDropped(s.dir) { + s.logger.Warn("WebSocket spool not removed", "dir", s.dir, "error", rerr) + } + return false } // remove closes the spool and deletes its files. It is marked dropped and its @@ -301,7 +322,7 @@ func (s *spool) close() error { // never started. Recovery deletes a dropped spool it finds, so a directory // another process holds open for a moment is not an error. func (s *spool) remove() error { - _ = s.shut(true) + _, _ = s.shut(true, false) if err := removeSpoolDir(s.dir); err != nil { if spoolDropped(s.dir) { s.logger.Debug("WebSocket spool left for recovery to delete", "dir", s.dir, "error", err) @@ -374,12 +395,15 @@ func spoolDropped(dir string) bool { return err == nil } -// shut closes the spool, deleting its segments first when drop is set. -func (s *spool) shut(drop bool) error { +// shut closes the spool, deleting its segments first when drop is set, or +// when intact is set and a record did not reach the disk; all under the +// spool's lock, so recovery never takes up an incomplete one. It reports +// whether it deleted them. +func (s *spool) shut(drop, intact bool) (bool, error) { s.mu.Lock() defer s.mu.Unlock() if s.closed { - return nil + return false, nil } s.closed = true close(s.done) @@ -387,11 +411,20 @@ func (s *spool) shut(drop bool) error { if s.file != nil { if s.ioErr == nil && !drop { err = s.w.Flush() + if intact && err == nil { + err = s.file.Sync() + } } if cerr := s.file.Close(); err == nil { err = cerr } } + if intact && (err != nil || s.failed != nil) { + if err == nil { + err = s.failed + } + drop = true + } if drop { if merr := markSpoolDropped(s.dir); err == nil && merr != nil { err = merr @@ -407,5 +440,5 @@ func (s *spool) shut(drop bool) error { err = rerr } s.lock.release() - return err + return drop, err } diff --git a/internal/storage/websocket/websocket.go b/internal/storage/websocket/websocket.go index 1fbc3b9..977c4cc 100644 --- a/internal/storage/websocket/websocket.go +++ b/internal/storage/websocket/websocket.go @@ -59,18 +59,21 @@ func (cfg Config) connOptions() connOptions { // Backend streams mission data over WebSocket to the OCAP2 web server. // It implements storage.Backend but not storage.Uploadable. type Backend struct { - missionMu sync.Mutex - mu sync.Mutex - restart bool - active bool // a mission has started and not ended - conn *connection - cfg Config - producerID string // identifies this extension process to the server - missionID string // identifies the current recording - nextMarkerID atomic.Uint64 - done chan struct{} // closed by Close; stops spool recovery - closeOnce sync.Once - loops sync.WaitGroup // recovery and stats loops + missionMu sync.Mutex + mu sync.Mutex + restart bool + active bool // a mission has started and not ended + // discardAbandoned makes abandonMission drop the mission; see + // SetDiscardAbandoned. + discardAbandoned bool + conn *connection + cfg Config + producerID string // identifies this extension process to the server + missionID string // identifies the current recording + nextMarkerID atomic.Uint64 + done chan struct{} // closed by Close; stops spool recovery + closeOnce sync.Once + loops sync.WaitGroup // recovery and stats loops // Totals of connections replaced between missions; see Stats. reconnects, resumes, restarts atomic.Uint64 @@ -234,6 +237,7 @@ func (b *Backend) StartMission(mission *core.Mission, world *core.World) error { } conn := b.conn b.missionID = newID() + b.discardAbandoned = false missionID := b.missionID b.mu.Unlock() conn.protocol.Store(0) @@ -298,9 +302,10 @@ func (b *Backend) StartMission(mission *core.Mission, world *core.World) error { // AbandonMission ends a mission the game moved on from without saving or // discarding it, for example after an admin changed the mission. It ends as // interrupted, so the server saves what it got as partial and nothing is left -// to recover. One that recorded nothing after its start is dropped instead, -// when the server can. A server without resume support saves a mission as -// partial when its producer disconnects, so the connection is closed instead. +// to recover. One that recorded nothing after its start, or one the addon +// would not save yet (SetDiscardAbandoned), is dropped instead, when the +// server can. A server without resume support saves a mission as partial +// when its producer disconnects, so the connection is closed instead. // It is a no-op when no mission is recording. func (b *Backend) AbandonMission() error { b.missionMu.Lock() @@ -314,7 +319,7 @@ func (b *Backend) abandonMission() error { return nil } b.mu.Lock() - conn, missionID := b.conn, b.missionID + conn, missionID, discard := b.conn, b.missionID, b.discardAbandoned b.mu.Unlock() switch { case !conn.isResumable(): @@ -326,12 +331,27 @@ func (b *Backend) abandonMission() error { case conn.log.lastSeq() <= 1 && b.discardable(): slog.Default().Warn("WebSocket mission abandoned before it recorded anything; discarding it", "missionId", missionID) return b.discardMission(abandonTimeout) + case discard && b.discardable(): + slog.Default().Warn("WebSocket mission abandoned before it met the addon's save limits; discarding it", "missionId", missionID) + return b.discardMission(abandonTimeout) default: slog.Default().Warn("WebSocket mission abandoned without save or discard; ending it as interrupted", "missionId", missionID) return b.endMission(streaming.EndMissionPayload{Interrupted: true}, abandonTimeout) } } +// SetDiscardAbandoned sets whether AbandonMission drops the current mission, +// for a recording the addon would not save yet. Each mission starts without +// it. It does not take missionMu, so a save holding it never blocks the +// game's thread here. +func (b *Backend) SetDiscardAbandoned(discard bool) { + b.mu.Lock() + defer b.mu.Unlock() + if b.active { + b.discardAbandoned = discard + } +} + // discardable reports whether the server serving the mission accepts // discard_mission, as it last announced. func (b *Backend) discardable() bool { @@ -432,12 +452,13 @@ func (b *Backend) discardMission(timeout time.Duration) error { } // A spool that holds discard_mission lets recovery finish the discard. - // Without it, recovery would resend the mission and the server save it. + // Without it, recovery would resend the mission and the server save it, + // so one whose last records do not reach the disk is removed instead. sp := b.clearMission() keep := err != nil && discardable && logged && sp != nil && sp.healthy() if sp != nil { if keep { - _ = sp.close() + keep = sp.keepIntact() } else { removeSpool(sp) } @@ -465,6 +486,7 @@ func (b *Backend) clearMission() *spool { conn := b.conn b.restart = true b.active = false + b.discardAbandoned = false b.missionID = "" b.mu.Unlock() sp := conn.log.currentSpool()