From 4707e2990a44e181a009a57630da30e8858bd976 Mon Sep 17 00:00:00 2001 From: Rob Zolkos Date: Fri, 2 Oct 2026 15:29:58 -0400 Subject: [PATCH] Remove what the connector's worker side left behind #815 removed the worker side but left the code it used, and no command reaches it: Ledger.Redispatch and its helpers, the ledger's task and attempt writers (LaunchTask, EndAttempt, CreateTask, SupersedeTask, Dispatch and TaskDispatch, MarkRunning, RecordTaker, ExposeEvent, JoinConversation, StartableRecords, adoption, refusals, still-running), the outbox readers and resolver, the never-installed ledger hooks, OpenExistingLedger, ResolveStateDir, Held, UnfinishedFromOthers, setup.TryLock, setup.Duration, and almost all of the driver package. The process-identity code status and guided setup use moves from internal/connector/driver to internal/connector/procid. The schema, migrations and triggers are unchanged; status, doctor, discard, import, shadow promote and the hold still read and decide about what an older build left. Tests for those build worker-era state with worker_era_test.go, which writes the rows the removed dispatcher wrote. Also moves two Unix-only start-wait tests behind a build tag, so the commands tests compile for Windows again. --- internal/commands/connect_operator_test.go | 2 +- internal/commands/connect_setup_guided.go | 4 +- internal/commands/connect_start_wait_test.go | 94 -- .../commands/connect_start_wait_unix_test.go | 113 ++ internal/commands/connect_worker.go | 26 +- internal/commands/connect_worker_other.go | 2 +- internal/commands/connect_worker_unix_test.go | 6 +- internal/connector/blocked_retry_test.go | 35 +- internal/connector/doc.go | 2 +- internal/connector/driver/announce_test.go | 50 - internal/connector/driver/driver.go | 618 -------- internal/connector/driver/driver_test.go | 341 ----- internal/connector/driver/env.go | 59 - internal/connector/driver/preflight.go | 345 ----- internal/connector/driver/preflight_test.go | 216 --- .../connector/driver/probe_reuse_unix_test.go | 69 - internal/connector/driver/redact.go | 366 ----- internal/connector/driver/redact_test.go | 140 -- internal/connector/driver/worker.go | 668 --------- internal/connector/driver/worker_other.go | 63 - internal/connector/driver/worker_unix.go | 120 -- internal/connector/handoff.go | 10 +- internal/connector/handoff_test.go | 10 +- internal/connector/invariants_test.go | 10 +- internal/connector/ledger.go | 65 +- internal/connector/ledger_admission.go | 22 +- internal/connector/ledger_admission_test.go | 52 +- internal/connector/ledger_dangerous_test.go | 62 - internal/connector/ledger_decisions.go | 406 +---- internal/connector/ledger_dispatch.go | 1098 +------------- internal/connector/ledger_dispatch_test.go | 1311 +---------------- internal/connector/ledger_events.go | 11 +- internal/connector/ledger_hold.go | 84 +- internal/connector/ledger_import.go | 5 - internal/connector/ledger_migrations_test.go | 2 +- internal/connector/ledger_owner_unix_test.go | 4 +- internal/connector/ledger_status.go | 43 +- internal/connector/ledger_tasks.go | 1303 +--------------- internal/connector/ledger_tasks_test.go | 601 -------- internal/connector/ledger_test.go | 2 +- .../connector/operator_invariants_test.go | 661 ++------- internal/connector/operator_migration_test.go | 10 +- internal/connector/operator_status_test.go | 36 +- internal/connector/outbox.go | 276 +--- internal/connector/procid/procid.go | 50 + internal/connector/procid/procid_other.go | 14 + internal/connector/procid/procid_unix.go | 157 ++ internal/connector/procid/procid_unix_test.go | 153 ++ .../{driver => procid}/proctime_darwin.go | 2 +- .../{driver => procid}/proctime_linux.go | 12 +- .../{driver => procid}/proctime_other.go | 6 +- .../{driver => procid}/zombie_darwin_test.go | 14 +- .../{driver => procid}/zombie_linux_test.go | 17 +- internal/connector/round7_test.go | 2 +- internal/connector/setup/file.go | 23 - internal/connector/setup/private.go | 18 +- internal/connector/setup/private_unix_test.go | 37 +- internal/connector/worker_era_test.go | 288 ++++ 58 files changed, 1190 insertions(+), 9026 deletions(-) create mode 100644 internal/commands/connect_start_wait_unix_test.go delete mode 100644 internal/connector/driver/announce_test.go delete mode 100644 internal/connector/driver/driver.go delete mode 100644 internal/connector/driver/driver_test.go delete mode 100644 internal/connector/driver/env.go delete mode 100644 internal/connector/driver/preflight.go delete mode 100644 internal/connector/driver/preflight_test.go delete mode 100644 internal/connector/driver/probe_reuse_unix_test.go delete mode 100644 internal/connector/driver/redact.go delete mode 100644 internal/connector/driver/redact_test.go delete mode 100644 internal/connector/driver/worker.go delete mode 100644 internal/connector/driver/worker_other.go delete mode 100644 internal/connector/driver/worker_unix.go delete mode 100644 internal/connector/ledger_dangerous_test.go delete mode 100644 internal/connector/ledger_tasks_test.go create mode 100644 internal/connector/procid/procid.go create mode 100644 internal/connector/procid/procid_other.go create mode 100644 internal/connector/procid/procid_unix.go create mode 100644 internal/connector/procid/procid_unix_test.go rename internal/connector/{driver => procid}/proctime_darwin.go (98%) rename internal/connector/{driver => procid}/proctime_linux.go (91%) rename internal/connector/{driver => procid}/proctime_other.go (76%) rename internal/connector/{driver => procid}/zombie_darwin_test.go (80%) rename internal/connector/{driver => procid}/zombie_linux_test.go (80%) create mode 100644 internal/connector/worker_era_test.go diff --git a/internal/commands/connect_operator_test.go b/internal/commands/connect_operator_test.go index 8719715c3..f2fdf788b 100644 --- a/internal/commands/connect_operator_test.go +++ b/internal/commands/connect_operator_test.go @@ -160,7 +160,7 @@ func TestConnectDiscardAndRelease(t *testing.T) { l, err = connector.OpenLedgerReadOnly(context.Background(), filepath.Join(dir, connector.LedgerFile)) require.NoError(t, err) defer func() { _ = l.Close() }() - held, err := l.Held(ctx) + _, held, err := l.HoldMarker(ctx) require.NoError(t, err) assert.False(t, held) two, _, err := l.Get(ctx, 2) diff --git a/internal/commands/connect_setup_guided.go b/internal/commands/connect_setup_guided.go index 6a000a5c3..7b302002c 100644 --- a/internal/commands/connect_setup_guided.go +++ b/internal/commands/connect_setup_guided.go @@ -20,7 +20,7 @@ import ( "github.com/basecamp/basecamp-cli/internal/auth" "github.com/basecamp/basecamp-cli/internal/config" "github.com/basecamp/basecamp-cli/internal/connector" - "github.com/basecamp/basecamp-cli/internal/connector/driver" + "github.com/basecamp/basecamp-cli/internal/connector/procid" "github.com/basecamp/basecamp-cli/internal/connector/setup" "github.com/basecamp/basecamp-cli/internal/hostutil" "github.com/basecamp/basecamp-cli/internal/output" @@ -565,7 +565,7 @@ func holderStillRuns(holder connector.InstanceHolderInfo) bool { if err != nil { return false } - p, err := driver.LookupProcess(holder.PID) + p, err := procid.LookupProcess(holder.PID) if err != nil { return false } diff --git a/internal/commands/connect_start_wait_test.go b/internal/commands/connect_start_wait_test.go index f1b7d7ebd..355009cb6 100644 --- a/internal/commands/connect_start_wait_test.go +++ b/internal/commands/connect_start_wait_test.go @@ -25,7 +25,6 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/basecamp/basecamp-cli/internal/appctx" "github.com/basecamp/basecamp-cli/internal/auth" "github.com/basecamp/basecamp-cli/internal/config" "github.com/basecamp/basecamp-cli/internal/connector" @@ -483,55 +482,6 @@ func TestAnOversizedServerFaultIsWaitedOut(t *testing.T) { assert.Equal(t, 2, mints()) } -// A second connector for an agent that is already running says so at once, -// even while the token endpoint is rate-limiting it: the lock is checked -// before the start waits for a token, not after. -func TestASecondConnectorIsRefusedBeforeItWaitsForAToken(t *testing.T) { - f := newOperatorFixture(t) - dir, err := connectStateDir(f.file, false) - require.NoError(t, err) - lock, err := connector.AcquireInstanceLock(dir, f.file.AccountID, f.file.Agent.PersonID, time.Now()) - require.NoError(t, err) - t.Cleanup(func() { _ = lock.Release() }) - - app := newConnectSetupApp(t, f.s, "agent") - creds, err := app.Auth.GetStore().Load(app.Auth.CredentialKey()) - require.NoError(t, err) - creds.ExpiresAt = time.Now().Add(-time.Minute).Unix() - require.NoError(t, app.Auth.GetStore().Save(app.Auth.CredentialKey(), creds)) - f.s.mu.Lock() - f.s.rateLimitMints, f.s.mints = true, 0 - f.s.mu.Unlock() - - ctx, cancel := context.WithCancel(t.Context()) - defer cancel() - var buf bytes.Buffer - app.Output = output.New(output.Options{Format: output.FormatJSON, Writer: &buf}) - cmd := NewConnectCmd() - cmd.SetArgs(nil) - cmd.SetContext(appctx.WithApp(ctx, app)) - cmd.SetOut(&buf) - cmd.SetErr(&buf) - cmd.SilenceErrors, cmd.SilenceUsage = true, true - done := make(chan error, 1) - go func() { done <- cmd.Execute() }() - - select { - case err := <-done: - refusal := usageError(t, err) - assert.Equal(t, output.CodeLockUnavailable, refusal.Code, buf.String()) - assert.Contains(t, refusal.Message, fmt.Sprintf("held by pid %d", os.Getpid()), "the refusal names the holder") - assert.Contains(t, refusal.Hint, "One connector serves every repo") - case <-time.After(10 * time.Second): - cancel() - <-done - t.Fatalf("a second connector waited for a token instead of refusing: %s", buf.String()) - } - f.s.mu.Lock() - defer f.s.mu.Unlock() - assert.Zero(t, f.s.mints, "nothing was minted for a connector that cannot run") -} - // A 503 that names its Retry-After is waited out for that long, by the // start and the feed alike, not on backoff. func TestAServerFaultsRetryAfterIsHonored(t *testing.T) { @@ -567,50 +517,6 @@ func (b *lockedBuffer) String() string { return b.buf.String() } -// A start waiting for its token does not hold the agent's lock: until the -// credential is proven to be that agent, the lock is only looked at, so a -// profile whose credential is some other agent's cannot keep the real -// connector out while it waits. -func TestAStartWaitingForItsTokenHoldsNoLock(t *testing.T) { - f := newOperatorFixture(t) - app := newConnectSetupApp(t, f.s, "agent") - creds, err := app.Auth.GetStore().Load(app.Auth.CredentialKey()) - require.NoError(t, err) - creds.ExpiresAt = time.Now().Add(-time.Minute).Unix() - require.NoError(t, app.Auth.GetStore().Save(app.Auth.CredentialKey(), creds)) - f.s.mu.Lock() - f.s.rateLimitMints = true - f.s.mu.Unlock() - - ctx, cancel := context.WithCancel(t.Context()) - defer cancel() - var out lockedBuffer - app.Output = output.New(output.Options{Format: output.FormatJSON, Writer: &out}) - cmd := NewConnectCmd() - cmd.SetArgs(nil) - cmd.SetContext(appctx.WithApp(ctx, app)) - cmd.SetOut(&out) - cmd.SetErr(&out) - cmd.SilenceErrors, cmd.SilenceUsage = true, true - done := make(chan error, 1) - go func() { done <- cmd.Execute() }() - - require.Eventually(t, func() bool { return strings.Contains(out.String(), "could not get the agent's token yet") }, - 10*time.Second, 20*time.Millisecond, "the start never waited: %s", out.String()) - dir, err := connectStateDir(f.file, false) - require.NoError(t, err) - lock, err := connector.AcquireInstanceLock(dir, f.file.AccountID, f.file.Agent.PersonID, time.Now()) - require.NoError(t, err, "a waiting start held the agent's lock") - require.NoError(t, lock.Release()) - - cancel() - select { - case <-done: - case <-time.After(10 * time.Second): - t.Fatal("the waiting start did not stop when canceled") - } -} - // okTokens is a renewal that succeeds at once. type okTokens struct{} diff --git a/internal/commands/connect_start_wait_unix_test.go b/internal/commands/connect_start_wait_unix_test.go new file mode 100644 index 000000000..d8bcc7a2f --- /dev/null +++ b/internal/commands/connect_start_wait_unix_test.go @@ -0,0 +1,113 @@ +//go:build unix + +package commands + +import ( + "bytes" + "context" + "fmt" + "os" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/basecamp/basecamp-cli/internal/appctx" + "github.com/basecamp/basecamp-cli/internal/connector" + "github.com/basecamp/basecamp-cli/internal/output" +) + +// A second connector for an agent that is already running says so at once, +// even while the token endpoint is rate-limiting it: the lock is checked +// before the start waits for a token, not after. +func TestASecondConnectorIsRefusedBeforeItWaitsForAToken(t *testing.T) { + f := newOperatorFixture(t) + dir, err := connectStateDir(f.file, false) + require.NoError(t, err) + lock, err := connector.AcquireInstanceLock(dir, f.file.AccountID, f.file.Agent.PersonID, time.Now()) + require.NoError(t, err) + t.Cleanup(func() { _ = lock.Release() }) + + app := newConnectSetupApp(t, f.s, "agent") + creds, err := app.Auth.GetStore().Load(app.Auth.CredentialKey()) + require.NoError(t, err) + creds.ExpiresAt = time.Now().Add(-time.Minute).Unix() + require.NoError(t, app.Auth.GetStore().Save(app.Auth.CredentialKey(), creds)) + f.s.mu.Lock() + f.s.rateLimitMints, f.s.mints = true, 0 + f.s.mu.Unlock() + + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + var buf bytes.Buffer + app.Output = output.New(output.Options{Format: output.FormatJSON, Writer: &buf}) + cmd := NewConnectCmd() + cmd.SetArgs(nil) + cmd.SetContext(appctx.WithApp(ctx, app)) + cmd.SetOut(&buf) + cmd.SetErr(&buf) + cmd.SilenceErrors, cmd.SilenceUsage = true, true + done := make(chan error, 1) + go func() { done <- cmd.Execute() }() + + select { + case err := <-done: + refusal := usageError(t, err) + assert.Equal(t, output.CodeLockUnavailable, refusal.Code, buf.String()) + assert.Contains(t, refusal.Message, fmt.Sprintf("held by pid %d", os.Getpid()), "the refusal names the holder") + assert.Contains(t, refusal.Hint, "One connector serves every repo") + case <-time.After(10 * time.Second): + cancel() + <-done + t.Fatalf("a second connector waited for a token instead of refusing: %s", buf.String()) + } + f.s.mu.Lock() + defer f.s.mu.Unlock() + assert.Zero(t, f.s.mints, "nothing was minted for a connector that cannot run") +} + +// A start waiting for its token does not hold the agent's lock: until the +// credential is proven to be that agent, the lock is only looked at, so a +// profile whose credential is some other agent's cannot keep the real +// connector out while it waits. +func TestAStartWaitingForItsTokenHoldsNoLock(t *testing.T) { + f := newOperatorFixture(t) + app := newConnectSetupApp(t, f.s, "agent") + creds, err := app.Auth.GetStore().Load(app.Auth.CredentialKey()) + require.NoError(t, err) + creds.ExpiresAt = time.Now().Add(-time.Minute).Unix() + require.NoError(t, app.Auth.GetStore().Save(app.Auth.CredentialKey(), creds)) + f.s.mu.Lock() + f.s.rateLimitMints = true + f.s.mu.Unlock() + + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + var out lockedBuffer + app.Output = output.New(output.Options{Format: output.FormatJSON, Writer: &out}) + cmd := NewConnectCmd() + cmd.SetArgs(nil) + cmd.SetContext(appctx.WithApp(ctx, app)) + cmd.SetOut(&out) + cmd.SetErr(&out) + cmd.SilenceErrors, cmd.SilenceUsage = true, true + done := make(chan error, 1) + go func() { done <- cmd.Execute() }() + + require.Eventually(t, func() bool { return strings.Contains(out.String(), "could not get the agent's token yet") }, + 10*time.Second, 20*time.Millisecond, "the start never waited: %s", out.String()) + dir, err := connectStateDir(f.file, false) + require.NoError(t, err) + lock, err := connector.AcquireInstanceLock(dir, f.file.AccountID, f.file.Agent.PersonID, time.Now()) + require.NoError(t, err, "a waiting start held the agent's lock") + require.NoError(t, lock.Release()) + + cancel() + select { + case <-done: + case <-time.After(10 * time.Second): + t.Fatal("the waiting start did not stop when canceled") + } +} diff --git a/internal/commands/connect_worker.go b/internal/commands/connect_worker.go index bd234a122..b71084ba8 100644 --- a/internal/commands/connect_worker.go +++ b/internal/commands/connect_worker.go @@ -6,13 +6,13 @@ import ( "errors" "github.com/basecamp/basecamp-cli/internal/connector" - "github.com/basecamp/basecamp-cli/internal/connector/driver" + "github.com/basecamp/basecamp-cli/internal/connector/procid" ) -// The operator commands act on a recorded worker only through the driver's -// one-owner rule (driver/worker.go): a pid is not an identity, so every -// question about a recorded worker is driver.OwnsWorker's, and nothing here -// tests a pid of its own. +// Status asks about a worker an older build recorded only through the +// one-owner rule (procid): a pid is not an identity, so every question about +// a recorded worker is procid.OwnsWorker's, and nothing here tests a pid of +// its own. // Worker states the operator commands report. const ( @@ -22,7 +22,7 @@ const ( workerUnverified = "unverified" workerNotRecorded = "not_recorded" // workerUnaccounted is the task token's holder alone: it was delivered - // and the connector cannot name who has it (connector.TokenHolder). + // and the connector that delivered it could not name who has it. workerUnaccounted = "unaccounted" ) @@ -39,9 +39,9 @@ func recordedWorkerState(t connector.TaskStatus) string { // A holder the connector could not account for is answered before the kernel // is asked anything: there is no pid to ask about, and "not recorded" would // read as nothing having taken the token, which is the opposite of what -// happened. It is the same distinction the release point acts on -// (connector.TokenHolder), and this is where the person who must settle the -// held attempt reads it. +// happened. The connector that delivered it held the attempt on that +// distinction, and this is where the person who must settle the held +// attempt reads it. func recordedTakerState(t connector.TaskStatus) string { if t.TakerUnaccounted { return workerUnaccounted @@ -52,15 +52,15 @@ func recordedTakerState(t connector.TaskStatus) string { // recordedProcessState asks the one-owner rule about a process the ledger // recorded. It takes the identity the ledger builds (TaskStatus's own // WorkerIdentity and TakerIdentity) rather than assembling one from parts: -// a driver.Process put together here would have no StartedExact, and the +// a procid.Process put together here would have no StartedExact, and the // rule answers ErrIdentityUnknown to that — every live worker would read as // "unverified". -func recordedProcessState(p driver.Process) string { +func recordedProcessState(p procid.Process) string { if p.PID <= 0 || p.PGID <= 0 || p.StartedAt.IsZero() { return workerNotRecorded } - switch owns, err := driver.OwnsWorker(p); { - case errors.Is(err, driver.ErrGroupOutlivedLeader): + switch owns, err := procid.OwnsWorker(p); { + case errors.Is(err, procid.ErrGroupOutlivedLeader): return workerHeld case err != nil: return workerUnverified diff --git a/internal/commands/connect_worker_other.go b/internal/commands/connect_worker_other.go index 75cf53377..7a77ce6ae 100644 --- a/internal/commands/connect_worker_other.go +++ b/internal/commands/connect_worker_other.go @@ -9,7 +9,7 @@ import ( // The one-owner rule is Unix's: process groups, and a start time that makes a // pid an identity, are what it rests on. Elsewhere the connector does not run // (the run command refuses), and nothing here can say whether a recorded -// worker is still itself — so nothing is signaled and nothing is claimed. +// worker is still itself, so nothing is claimed. const ( workerUnverified = "unverified" diff --git a/internal/commands/connect_worker_unix_test.go b/internal/commands/connect_worker_unix_test.go index b883fa755..8c53bd8d2 100644 --- a/internal/commands/connect_worker_unix_test.go +++ b/internal/commands/connect_worker_unix_test.go @@ -13,9 +13,9 @@ import ( // A token delivered to a process the connector could not name is the one // case the zero taker cannot express, and status is where the person who // must settle the held attempt reads it. "Nothing took the token" and "the -// token is out and nobody can say who has it" are opposite facts; the same -// conflation the dispatcher stopped making (TokenHolder.Unaccounted) must -// not survive on the operator's side of it. +// token is out and nobody can say who has it" are opposite facts, and the +// removed dispatcher held the attempt on the difference; it must not be lost +// on the operator's side of it. func TestStatusSaysWhenATokenHolderCannotBeAccountedFor(t *testing.T) { assert.Equal(t, workerUnaccounted, recordedTakerState(connector.TaskStatus{TakerUnaccounted: true}), diff --git a/internal/connector/blocked_retry_test.go b/internal/connector/blocked_retry_test.go index dcba83973..28ec3d908 100644 --- a/internal/connector/blocked_retry_test.go +++ b/internal/connector/blocked_retry_test.go @@ -507,30 +507,6 @@ func TestAStaleRetryHandoffIsNotDecidedInsideTheInterval(t *testing.T) { assert.True(t, ok, "and at the moment the schedule names, it loads") } -// A person's redispatch is not a timer, and does not wait for one. It marks -// the record due, so the rerun it asks for runs at once — and so a rerun that -// never happened (the command died between the authorization and the -// admission run) is picked up by the next sweep rather than stranded. -func TestARedispatchMakesABlockedRecordDueAtOnce(t *testing.T) { - ledger, clock := retryLedger(t) - ctx := context.Background() - blockRecord(t, ledger, 1, adapterBucketID, admission.ReasonReadFailed) - - _, ok, err := ledger.Admission().LoadUndecided(ctx, 1) - require.NoError(t, err) - require.False(t, ok, "nine minutes early, on the timer's account") - - out, err := ledger.Redispatch(ctx, 1, "jorge", []int64{adapterBucketID}) - require.NoError(t, err) - require.True(t, out.Rerun) - - _, ok, err = ledger.Admission().LoadUndecided(ctx, 1) - require.NoError(t, err) - assert.True(t, ok, "a person asked for it now") - assert.Equal(t, []int64{1}, dueIDs(t, ledger, BlockedRetryScope{Now: clock.Now(), Limit: 10}), - "and if the rerun never happens, the sweep picks it up") -} - // Copilot on #770, and it is the starvation the card named, reintroduced // through the back door. The claim stops a row being offered twice; it does // nothing if the claimed rows still spend the sweep's budget. A limit filled @@ -573,15 +549,8 @@ func TestABlockedRecordWithNoScheduledAttemptIsNotRunnable(t *testing.T) { _, ok, err := ledger.Admission().LoadUndecided(ctx, 1) require.NoError(t, err) assert.False(t, ok, "nothing has given this record an attempt to make") - - // A person is what gives an untimed record one. - _, err = ledger.Redispatch(ctx, 1, "jorge", []int64{adapterBucketID}) - require.NoError(t, err) - _, ok, err = ledger.Admission().LoadUndecided(ctx, 1) - require.NoError(t, err) - assert.True(t, ok, "the redispatch is the attempt") - assert.Equal(t, []int64{1}, dueIDs(t, ledger, BlockedRetryScope{Now: clock.Now(), Limit: 10}), - "and if the rerun never happens, the sweep picks it up rather than stranding it") + assert.Empty(t, dueIDs(t, ledger, BlockedRetryScope{Now: clock.Now(), Limit: 10}), + "and the sweep does not offer it either") } // queryPlan is what SQLite says it will actually do, which is the only diff --git a/internal/connector/doc.go b/internal/connector/doc.go index 127aa1927..a2e4355fe 100644 --- a/internal/connector/doc.go +++ b/internal/connector/doc.go @@ -7,7 +7,7 @@ // Feed rows are pointers — id, type, bucket, creator, recording — and nothing // else. No title, no body, no URL, no names. Intake writes that pointer to the // ledger if the id is new and hands the id to a queue. That is the whole of -// the work on the feed's delivery path: a slow admission or a busy dispatcher +// the work on the feed's delivery path: a slow admission or a slow hand-off // is absorbed by the backlog rather than felt by the socket. // // Absorbed, not unbounded. The backlog is a bounded queue, and at its pause diff --git a/internal/connector/driver/announce_test.go b/internal/connector/driver/announce_test.go deleted file mode 100644 index f6d2c9c2a..000000000 --- a/internal/connector/driver/announce_test.go +++ /dev/null @@ -1,50 +0,0 @@ -//go:build unix - -package driver - -import ( - "context" - "testing" - "time" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -// Invariant 7: the worker is announced as soon as it exists, not when a -// session is ready on top of it. StartWorker is where every driver's worker -// is started, so it is where the announcement is made — a driver cannot -// forget it, and the connector cannot be left arming a token socket for a -// process group it will not be told about until a handshake that is waiting -// on that socket has returned. -func TestAWorkerIsAnnouncedAsSoonAsItsProcessExists(t *testing.T) { - var announced []Process - cfg := SessionConfig{ - Scope: Scope{WorkDir: t.TempDir()}, - Started: func(p Process) { announced = append(announced, p) }, - } - w, err := StartWorker(context.Background(), cfg, Command{Path: "/bin/sleep", Args: []string{"30"}, Env: []string{}}) - require.NoError(t, err) - t.Cleanup(func() { w.Terminate(time.Second) }) - - require.Len(t, announced, 1, "announced once, by the time StartWorker returns") - assert.Equal(t, w.Process(), announced[0], "and it is the worker StartWorker is handing back") - assert.Positive(t, announced[0].PID) - assert.Equal(t, announced[0].PID, announced[0].PGID, "which leads its own group, so the token socket can be told one number") -} - -// A start that never made a process has nothing to announce: the connector -// would otherwise arm a socket for a group that does not exist. -func TestAStartThatLaunchedNothingAnnouncesNothing(t *testing.T) { - announced := 0 - cfg := SessionConfig{ - Scope: Scope{WorkDir: t.TempDir()}, - Started: func(Process) { announced++ }, - } - _, err := StartWorker(context.Background(), cfg, Command{Path: "/nonexistent/agent-not-here"}) - require.ErrorIs(t, err, ErrNotStarted) - _, err = StartWorker(context.Background(), SessionConfig{Launcher: refusingLauncher{}, Scope: cfg.Scope, Started: cfg.Started}, - Command{Path: "/bin/true"}) - require.ErrorIs(t, err, ErrNotStarted) - assert.Zero(t, announced) -} diff --git a/internal/connector/driver/driver.go b/internal/connector/driver/driver.go deleted file mode 100644 index dae675a35..000000000 --- a/internal/connector/driver/driver.go +++ /dev/null @@ -1,618 +0,0 @@ -// Package driver is the connector's agent boundary: how a dispatched task -// becomes a working coding agent, and how the connector hears what it does. -// -// # The shape is ACP's -// -// The interface is Agent Client Protocol v1's session model, whatever speaks -// underneath. A Driver opens a session (session/new) or reloads one -// (session/load) in a working directory with an explicit set of MCP servers; -// a Session takes prompts, each returning a stop reason (session/prompt); -// progress arrives as a stream of updates (session/update); a turn is ended -// with Cancel (session/cancel); and a permission the agent asks for is -// answered by the connector's policy (session/request_permission). A spawn -// driver (claude -p, codex exec) is an adapter onto that shape: it maps its -// vendor stream onto the same updates and stop reasons, freezes the policy -// into flags it verifies, and cancels by ending the process group it started. -// So the ACP driver is one more driver, not a rewrite. -// -// # Invariants every driver holds -// -// Each is held by a test in the driver that implements it. -// -// 1. Nothing is inherited. A worker process gets exactly the environment in -// SessionConfig.Env and each MCP server exactly MCPServer.Env; the -// connector's own environment (which carries tokens of its host) never -// reaches either. No secret is ever put in a process's argv. -// 2. The permission mode is set explicitly and verified. A session whose -// agent did not confirm the mode the policy asked for is unsafe, and the -// driver refuses to go on with it (ErrUnsafeMode) rather than run under -// the host's own configuration. -// 3. A refusal is the driver's own record. A policy refusal is not -// distinguishable from a cancel by the agent's stop reason, so every -// refusal the driver made or observed is recorded once, through -// SessionConfig.Refusals, at the moment it is made or observed; it is -// reported as well as a Refusal on the prompt's result and as an update; -// and a stop the connector did not ask for is never reported as -// TurnCanceled. See "Refusals" below. -// 4. ErrNotStarted means no worker process ever existed. It is the only -// start error after which the connector retries on its own, so a driver -// returns it only when it can prove nothing ran; any doubt is some other -// error. A configuration no retry can fix wraps ErrUnusable as well, and -// is not retried. Whatever the error, a start that fails leaves no -// process behind: either none was started, or the driver ended the one it -// started — through Terminate, so the whole group goes — before -// returning. A driver that cannot promise that returns a Session the -// connector can Close instead of an error. -// 5. A worker is ended by the process group the driver started, never by -// name. Close is idempotent and leaves no process of the session behind. -// 6. Content stays in the stream. Updates carry kinds, ids, tool names and -// counts; they never carry the agent's text or a tool's input, so a sink -// that logs an update cannot log content. What a sink does log from an -// agent stream goes through the redaction rule (redact.go). -// 7. A worker is announced as soon as it exists, not when its session is -// ready. SessionConfig.Started is called with the worker's process the -// moment the fork returns, before any handshake the driver runs — which -// is what lets the connector arm the task token's socket for a worker -// whose handshake cannot finish until its MCP servers have connected. -// StartWorker calls it, so every driver that starts a process holds this -// by construction; a driver whose session runs somewhere the connector -// cannot signal announces nothing, and the connector arms on what -// Session.Process reports instead. -// -// # Refusals: where one is recorded, and when it counts as settled -// -// A refusal is a permission the agent asked for and did not get. It is -// recorded in the ledger, once, at the moment the driver answers the request -// — or, for an agent that answers its own requests under a mode the driver -// froze (claude -p), at the moment the driver first reads that it was -// refused. It is never held only in a session's memory, because a worker that -// exits before its result, a connector that crashes mid-turn, and a turn cut -// short by a deadline all end the session that memory lives in. -// -// 1. The driver calls SessionConfig.Refusals.RecordRefusal before it sends -// its answer to the agent, or before it emits the update for a refusal -// it observed. It calls it once per tool call id: a refusal the stream -// announced and the result repeats is one refusal. A refusal with NO -// tool call id — one read from a line of the agent's output rather than -// from a call — counts every time it happens, identical text included: -// two refusals of the same tool are two refusals, and nothing but an id -// can say they are one. -// The recorder itself deduplicates NOTHING: it records what it is told, -// once per call. Deciding what is one refusal is the driver's, which -// knows what it read — a tool call id where the agent gives one, and -// where a driver reads refusals from lines of output, the line AND its -// occurrence in that output, so two identical lines are two refusals and -// reading the same output twice records neither again (card 19). -// 2. The dispatcher's recorder writes it to the attempt's row at once -// (connector.Ledger.RecordRefusal: attempts.refusals, incremented while -// the attempt is live). A write the ledger refuses is carried by the -// recorder into the attempt's settlement instead, and logged. -// 3. The refusal is settled with its attempt: EndAttempt adds whatever the -// recorder could not write, and the ended attempt's count is final. The -// session's updates are drained before the attempt is released, and the -// recorder is called before an update is emitted, so a worker that exits -// between a refusal and its result has already recorded it. -// -// Once-ness is the driver's (a set of tool call ids per session), not a key in -// the ledger: it holds for as long as a session lives, which is as long as a -// refusal can be reported twice. A connector that restarts does not resume a -// session — its attempt is settled as lost and its task superseded — so a -// ledger key on (attempt, tool call) would buy nothing, and this is settled, -// not open. -// -// Where a refusal can be seen differs by agent: Claude Code announces it in -// its stream and repeats it in the turn's result, and an agent that writes -// refusals only to stderr is read through Worker.StderrLines, not -// StderrTail — the tail is the last line, and whatever the agent prints next -// would bury the refusal. -// -// Where this can still be broken: a refusal the agent never reports — a tool -// it declined to ask for, or a denial its stream does not carry — is not a -// refusal the driver can record. -package driver - -import ( - "context" - "errors" - "time" -) - -// Driver starts and reloads sessions for one kind of coding agent. -type Driver interface { - // Name is the driver's name as connect.json and the ledger spell it: - // "claude", "codex", "acp". - Name() string - // Capabilities says what the driver supports beyond NewSession and Prompt. - Capabilities() Capabilities - // NewSession starts a worker and opens a session in cfg.Cwd. The worker - // is announced through cfg.Started as soon as its process exists, before - // the session is opened on top of it (invariant 7). - // - // An error that wraps ErrNotStarted means no process ever existed, and - // the connector may retry the start once. Any other error from a start - // that launched a process wraps a *StartError carrying that process, whose - // group the driver has already asked to end: the connector confirms it - // gone (ConfirmGroupGone) before it settles anything, however long the - // driver's own handshake took to fail. - NewSession(ctx context.Context, cfg SessionConfig) (Session, error) - // LoadSession reopens a session by the id an earlier Session reported, - // where Capabilities().LoadSession is true. Its errors read as - // NewSession's, and leave no process behind either. - LoadSession(ctx context.Context, cfg SessionConfig, sessionID string) (Session, error) -} - -// Capabilities are what a driver advertises, as an ACP agent advertises its -// own at initialize. -type Capabilities struct { - // LoadSession: LoadSession works, so a follow-up after the worker ended - // can continue its conversation. - LoadSession bool - // FollowUpPrompts: a live session takes further prompts, so a follow-up - // is delivered into the same session rather than as a new attempt. - FollowUpPrompts bool - // PermissionCallback: the agent asks, and PermissionPolicy.Decide answers - // each request. False for a spawn driver, whose permissions are frozen - // into flags from PermissionPolicy.Rules before the process starts. - PermissionCallback bool -} - -// Session is one live conversation with a worker. -type Session interface { - // ID is the agent's session id (ACP sessionId, Claude Code's session_id). - // It is known when NewSession returns. - ID() string - // Process is the worker's process, or the zero Process when the session - // runs somewhere the connector cannot signal. - Process() Process - // Prompt sends one prompt and blocks until the turn ends. The first - // prompt of a session is its handshake: a driver that verifies the - // agent's mode on it returns ErrUnsafeMode and ends the session. A ctx - // that ends makes Prompt return ctx's error without ending the turn; use - // Cancel for that. - Prompt(ctx context.Context, prompt string) (PromptResult, error) - // Updates streams the session's progress. It is closed when the session - // ends. A consumer that stops reading does not stall the agent: a driver - // drops updates rather than block. - Updates() <-chan Update - // Cancel ends the turn in flight. Prompt then returns TurnCanceled. - // With no turn in flight it does nothing. - Cancel(ctx context.Context) error - // Close ends the session and its worker: the process group is signaled, - // given grace, and killed. Idempotent; safe concurrently with Prompt, - // which then returns an error. - Close() error - // Done is closed once the worker has exited, however it exited. - Done() <-chan struct{} - // Exit is how the worker exited; meaningful once Done is closed. - Exit() Exit -} - -// SessionConfig is everything a driver needs to start a session. The -// dispatcher builds it from the task's record; the driver adds nothing of its -// own beyond its binary and its flags. -type SessionConfig struct { - // Cwd is the directory the session runs in, absolute: the connector's - // own, the one it was started in. - Cwd string - // Env is the worker process's whole environment, as KEY=VALUE. Nothing - // else is inherited (invariant 1). BuildEnv makes one from an allowlist. - Env []string - // MCPServers are the only MCP servers the agent gets. A driver makes the - // agent ignore every other MCP configuration it would otherwise load. - MCPServers []MCPServer - // Policy answers permissions. - Policy PermissionPolicy - // Launcher wraps the worker command. Nil means DirectLauncher. - Launcher Launcher - // Scope is what the launcher is told the worker is for. - Scope Scope - // SocketDir is the directory holding the task token's unix socket, which - // the worker's MCP server dials. It is PrivateDir in the ordinary case - // and a short directory of the connector's own where a socket path under - // PrivateDir would be longer than a unix socket takes. It is in Scope - // too, which is what a launcher is given. - SocketDir string - // PrivateDir is an owner-only directory the driver may write session - // files into (an MCP config, say). The driver removes what it wrote when - // the session is closed; the dispatcher sweeps the directory on start. - PrivateDir string - // Refusals records every refusal at the moment it is made or observed. - // Nil records nothing; the dispatcher always sets it. - Refusals RefusalRecorder - // Started is called once with the worker's process as soon as that - // process exists — the fork has returned and the group is the worker's — - // and before whatever handshake the driver runs on top of it (invariant - // 7). It is how the connector learns the process group while NewSession - // is still running, which is what arms the task token's socket in time - // for an agent that starts its MCP servers during its own handshake: - // arming only once NewSession returned deadlocks such an agent against a - // token the connector cannot hand over until the handshake it is blocking - // has finished. It must not block: a driver calls it on the goroutine - // that is starting the session. Nil announces nothing. - Started func(Process) - // Redaction is what the driver takes out of every error it returns and - // every text an update or a stderr tail carries (redact.go). The driver - // adds the environment it builds, its MCP servers' environments and - // PrivateDir to it. - Redaction Redaction -} - -// MCPServer is one stdio MCP server handed to the agent, as ACP's -// mcpServers[] entry. -type MCPServer struct { - // Name is the server's name as the agent's tools will be prefixed. - Name string - // Command is the executable, absolute. - Command string - // Args are its arguments. Never a secret: argv is readable by every - // process on the machine. - Args []string - // Env is the server's whole environment, KEY -> VALUE. Declared - // explicitly, never counted on to be inherited: some agents pass their - // own environment down and some pass almost nothing. - Env map[string]string -} - -// Process is a worker process the connector started. -type Process struct { - // PID is the process's id; zero when there is none to signal. - PID int - // PGID is its process group, which Close signals. A driver starts every - // worker as the leader of a new group, so PGID == PID. - PGID int - // StartedAt is when the process started, to tell it from a later one - // that reused its id. - StartedAt time.Time - // StartedExact is true when StartedAt is the kernel's own start time for - // the pid, which is what an identity is compared by. False is a - // wall-clock stamp the driver took around the fork because the kernel - // could not be asked: readable, but not an identity, and the one-owner - // rule signals nothing and releases nothing on one (ErrIdentityUnknown). - StartedExact bool -} - -// Exit is how a worker ended. -type Exit struct { - // Code is the exit status, or -1 when a signal ended the process. - Code int - // Signaled is true when a signal ended it. - Signaled bool - // Err is a failure to wait on the process at all. - Err error -} - -// TurnStop is why a prompt turn ended: ACP v1's stop reasons. -type TurnStop string - -const ( - // TurnEndTurn is the agent finishing its turn. - TurnEndTurn TurnStop = "end_turn" - // TurnMaxTokens is the token limit. - TurnMaxTokens TurnStop = "max_tokens" - // TurnMaxTurnRequests is the agent's own request budget for the turn. - TurnMaxTurnRequests TurnStop = "max_turn_requests" - // TurnRefusal is the agent refusing to continue. - TurnRefusal TurnStop = "refusal" - // TurnCanceled is a cancel the connector asked for, and only that - // (invariant 3). The value is ACP's spelling. - TurnCanceled TurnStop = "cancelled" //nolint:misspell // ACP's wire value -) - -// PromptResult is a finished turn. -type PromptResult struct { - Stop TurnStop - // Refusals are the permissions refused during the turn (invariant 3). - Refusals []Refusal - // Usage is the turn's token use, where the agent reports it. - Usage Usage -} - -// Refusal is one permission the policy refused. -type Refusal struct { - // ToolCallID is the agent's id for the call. - ToolCallID string - // Tool is the tool's name or ACP kind; never its input. - Tool string -} - -// RefusalRecorder records a refusal at the moment a driver makes or observes -// it (see "Refusals" above). RecordRefusal must not block for long: a driver -// calls it on the goroutine that reads the agent's stream. -type RefusalRecorder interface { - RecordRefusal(ctx context.Context, r Refusal) error -} - -// Usage is token accounting. -type Usage struct { - InputTokens int64 - OutputTokens int64 - // ContextUsed and ContextSize are ACP usage_update's {used, size}, where - // known. - ContextUsed int64 - ContextSize int64 -} - -// UpdateKind names a session update, as ACP's sessionUpdate does. -type UpdateKind string - -const ( - UpdateToolCall UpdateKind = "tool_call" - UpdateToolCallUpdate UpdateKind = "tool_call_update" - UpdateUsage UpdateKind = "usage_update" - UpdateAgentMessageChunk UpdateKind = "agent_message_chunk" - // UpdatePlan is optional: no adapter the spike ran emitted one. - UpdatePlan UpdateKind = "plan" - // UpdatePermission is a permission decision the driver made or observed. - UpdatePermission UpdateKind = "permission" -) - -// ToolStatus is a tool call's status. -type ToolStatus string - -const ( - ToolPending ToolStatus = "pending" - ToolInProgress ToolStatus = "in_progress" - ToolCompleted ToolStatus = "completed" - ToolFailed ToolStatus = "failed" -) - -// ToolKind is ACP's tool kind. -type ToolKind string - -const ( - ToolRead ToolKind = "read" - ToolEdit ToolKind = "edit" - ToolDelete ToolKind = "delete" - ToolMove ToolKind = "move" - ToolSearch ToolKind = "search" - ToolExecute ToolKind = "execute" - ToolThink ToolKind = "think" - ToolFetch ToolKind = "fetch" - ToolOther ToolKind = "other" -) - -// Update is one piece of progress. It carries no content (invariant 6): -// progress is for liveness, budgets and the ledger, never for reading what -// the agent said. -type Update struct { - Kind UpdateKind - At time.Time - - // ToolCallID, Tool, ToolKind and Status describe a tool call. - ToolCallID string - // Tool is the tool's name ("Bash", "mcp__basecamp__basecamp_connect"). - Tool string - ToolKind ToolKind - Status ToolStatus - - // Usage is set on UpdateUsage. - Usage *Usage - // Chars is the length of an agent message chunk, whose text is not - // carried. - Chars int - // Allowed is set on UpdatePermission: whether the policy allowed it. - Allowed bool -} - -// PermissionPolicy is the connector's answer to what a worker may do. -// Permission answers are policy, not containment: the worker still runs with -// the operator's ambient authority, and nothing here is a sandbox. -type PermissionPolicy interface { - // Decide answers one request, for drivers that ask - // (Capabilities.PermissionCallback). - Decide(ctx context.Context, req PermissionRequest) PermissionDecision - // Rules is the same policy, pre-decided, for drivers whose permissions - // are fixed before the worker starts. - Rules() PermissionRules -} - -// PermissionRequest is ACP's session/request_permission, reduced to what a -// policy decides on. -type PermissionRequest struct { - ToolCallID string - Tool string - Kind ToolKind - // Locations are the paths the call touches, where the agent says. - Locations []string - // Options are the choices the agent offers. A driver selects by kind, - // never by id or label: ids are not portable across agents. - Options []PermissionOption -} - -// PermissionOption is one choice the agent offers. -type PermissionOption struct { - ID string - Kind PermissionOptionKind -} - -// PermissionOptionKind is ACP's option kind. -type PermissionOptionKind string - -const ( - AllowOnce PermissionOptionKind = "allow_once" - AllowAlways PermissionOptionKind = "allow_always" - RejectOnce PermissionOptionKind = "reject_once" - RejectAlways PermissionOptionKind = "reject_always" -) - -// PermissionDecision is the policy's answer. A driver answers with the offered -// option of kind AllowOnce or RejectOnce, and refuses when the kind it needs -// is not offered. -type PermissionDecision struct { - Allow bool -} - -// PermissionRules is a policy pre-decided. -// -// It names no directory. The connector's policy bounded edits to the working -// directory while a project was routed to one; with the routes gone the only -// directory left is wherever the operator started the connector, and a bound -// that moves with that is a guarantee in name and an accident in behavior. -// What confines a worker to a directory now is the agent's own sandbox -// (Codex's workspace-write) or the sandbox launcher being built separately — -// not this. -type PermissionRules struct { - // Mode is the asking mode the agent must run in and confirm. - Mode PermissionMode - // AllowKinds are the tool kinds allowed without asking, besides edits. - AllowKinds []ToolKind - // AllowMCPServers are the MCP servers whose every tool is allowed. - AllowMCPServers []string -} - -// PermissionMode is the connector's name for an agent's permission mode. A -// driver maps it to the agent's own mode id and verifies the agent reports -// that id back. -type PermissionMode string - -const ( - // ModeEdits allows edits, and refuses, without asking anyone, whatever - // the rules do not allow. It bounds where an edit may land no further - // than the agent's own sandbox does. - ModeEdits PermissionMode = "edits" - // ModeAnything allows every tool the agent has, commands included, - // without asking anyone: dangerous mode. The worker runs with the - // operator's own authority on this computer, and nothing but the agent's - // judgment stands between a request and a command. - ModeAnything PermissionMode = "anything" -) - -// Launcher wraps the worker command: the seam where a sandbox launcher -// (sandbox-run) takes the dispatch. Scopes in, working directory and receipts -// out. -type Launcher interface { - // Launch returns the command that actually runs and the directory it runs - // in. A launcher refuses a request whose scope it cannot honor. - Launch(ctx context.Context, req LaunchRequest) (Launched, error) - // Receipts are what the launcher confirms the worker did, for the attempt - // the scope named. The direct launcher confirms nothing. - Receipts(ctx context.Context, attemptID string) ([]Receipt, error) -} - -// Scope is what a worker is for, as the launcher is told. -type Scope struct { - TaskID int64 - AttemptID string - // EventIDs are the events the task may cover. Only the originating event - // has been handed to the worker when the session starts; the others are - // exposed as they are prompted. - EventIDs []int64 - // WorkDir is the directory the worker runs in: the connector's own, the - // one it was started in. It is where the process starts, not a bound on - // where it may write. - WorkDir string - // SocketDir holds the task token's unix socket, which the worker's MCP - // server dials. A launcher that confines a worker must let it reach this - // directory, or the worker's MCP server cannot be handed its token. It is - // SessionConfig.PrivateDir in the ordinary case, and a short directory of - // the connector's own where a socket path under PrivateDir would be - // longer than a unix socket takes. - SocketDir string - Class string -} - -// Command is a process to run: path, argv (without the path) and the whole -// environment. -type Command struct { - Path string - Args []string - Env []string - Dir string -} - -// LaunchRequest is a worker command and its scope. -type LaunchRequest struct { - Scope Scope - Command Command -} - -// Launched is what runs. -type Launched struct { - Command Command - // WorkDir is the directory the worker works in: Scope.WorkDir for the - // direct launcher, a broker-owned scope under a sandbox. - WorkDir string -} - -// Receipt is something a launcher confirms a worker posted. -type Receipt struct { - Kind string - ID int64 - URL string -} - -// DirectLauncher runs the worker as it is, in the scope's directory. -type DirectLauncher struct{} - -// Launch implements Launcher. -func (DirectLauncher) Launch(_ context.Context, req LaunchRequest) (Launched, error) { - if req.Scope.WorkDir == "" { - return Launched{}, errors.New("driver: a launch needs a working directory to start the process in") - } - cmd := req.Command - cmd.Dir = req.Scope.WorkDir - return Launched{Command: cmd, WorkDir: req.Scope.WorkDir}, nil -} - -// Receipts implements Launcher. -func (DirectLauncher) Receipts(context.Context, string) ([]Receipt, error) { return nil, nil } - -// StartError is a start that failed after it launched a process. The -// driver has asked the process's group to end; the connector owns confirming -// it gone before it settles the attempt. -type StartError struct { - Process Process - Err error -} - -func (e *StartError) Error() string { - return "driver: the worker started and then failed: " + e.Err.Error() -} -func (e *StartError) Unwrap() error { return e.Err } - -// StartedProcess is the process a failed start launched, if it launched one. -func StartedProcess(err error) Process { - var started *StartError - if errors.As(err, &started) { - return started.Process - } - return Process{} -} - -// DefaultGrace is how long a worker's process group has between SIGTERM and -// SIGKILL. -const DefaultGrace = 10 * time.Second - -// Errors a driver reports. -var ( - // ErrNotStarted wraps a start that failed before any worker process - // existed (invariant 4): the binary is missing, the launcher refused, the - // fork failed. Only this is retried automatically. - ErrNotStarted = errors.New("driver: the worker was not started") - // ErrUnusable wraps ErrNotStarted for a configuration no retry can fix: - // a mode the driver cannot express, a policy for another directory, an - // MCP server without a command. No process existed, and starting again - // would fail the same way, so the connector does not retry it. - ErrUnusable = errors.New("driver: the session's configuration cannot start a worker") - // ErrUnsafeMode is an agent that did not confirm the permission mode the - // policy asked for (invariant 2). The session is ended. - ErrUnsafeMode = errors.New("driver: the agent did not confirm the permission mode asked for") - // ErrSessionUnverified is a session that started but is not the one the - // connector asked for: an MCP server the agent did not connect, or a - // session id that is not the one requested. The driver ends such a - // session rather than let a worker run without the tools its dispatch - // needs — a worker with no Basecamp tools can neither read its dispatch - // nor report it, and would otherwise finish with the mention unanswered - // (card 23's finding). A driver's own sentinel for one of these wraps - // this one. - ErrSessionUnverified = errors.New("driver: the session is not the one the connector asked for") - // A server that stops working AFTER the handshake is not detectable from - // Claude Code's stream, which carries server status only in its init - // message: the connector's record is what catches it, since an event the - // worker could not report settles completed(unknown) and never succeeded, - // and the dispatcher logs connector.UnreportedFinishLine for a person to - // find. - // - // ErrSessionEnded is a call on a session whose worker is gone. - ErrSessionEnded = errors.New("driver: the session has ended") -) diff --git a/internal/connector/driver/driver_test.go b/internal/connector/driver/driver_test.go deleted file mode 100644 index 71740bede..000000000 --- a/internal/connector/driver/driver_test.go +++ /dev/null @@ -1,341 +0,0 @@ -//go:build unix - -package driver - -import ( - "context" - "errors" - "os" - "os/exec" - "path/filepath" - "strconv" - "strings" - "syscall" - "testing" - "time" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func lookupFrom(m map[string]string) func(string) (string, bool) { - return func(k string) (string, bool) { v, ok := m[k]; return v, ok } -} - -func TestBuildEnvTakesExactNamesOnly(t *testing.T) { - host := map[string]string{ - "HOME": "/home/x", "PATH": "/bin", "CLAUDE_CODE_MESSAGING_TOKEN": "test-token-not-real", - "BASECAMP_TOKEN": "test-token-not-real", "HOMEBREW_PREFIX": "/opt", - } - env := BuildEnv(BaseEnv, lookupFrom(host), map[string]string{"PATH": "/usr/bin", "EXTRA": "1", "BAD=NAME": "x"}) - assert.Equal(t, []string{"EXTRA=1", "HOME=/home/x", "PATH=/usr/bin"}, env) -} - -func TestStartWorkerNeverInheritsTheConnectorsEnvironment(t *testing.T) { - t.Setenv("CONNECTOR_CANARY_NOT_REAL", "leaked") - out := filepath.Join(t.TempDir(), "env.txt") - w, err := StartWorker(context.Background(), SessionConfig{Scope: Scope{WorkDir: t.TempDir()}}, - Command{Path: "/bin/sh", Args: []string{"-c", "env > " + out}, Env: []string{"ONLY=this"}}) - require.NoError(t, err) - <-w.Done() - data, err := os.ReadFile(out) - require.NoError(t, err) - assert.NotContains(t, string(data), "CONNECTOR_CANARY_NOT_REAL") - assert.Contains(t, string(data), "ONLY=this") - - // A nil Env is not "inherit". - w, err = StartWorker(context.Background(), SessionConfig{Scope: Scope{WorkDir: t.TempDir()}}, - Command{Path: "/bin/sh", Args: []string{"-c", "env > " + out}}) - require.NoError(t, err) - <-w.Done() - data, err = os.ReadFile(out) - require.NoError(t, err) - assert.NotContains(t, string(data), "CONNECTOR_CANARY_NOT_REAL") -} - -type refusingLauncher struct{} - -func (refusingLauncher) Launch(context.Context, LaunchRequest) (Launched, error) { - return Launched{}, errors.New("scope refused") -} -func (refusingLauncher) Receipts(context.Context, string) ([]Receipt, error) { return nil, nil } - -func TestAStartThatRanNothingIsErrNotStarted(t *testing.T) { - _, err := StartWorker(context.Background(), SessionConfig{Scope: Scope{WorkDir: t.TempDir()}}, Command{Path: "/nonexistent/claude-not-here"}) - assert.ErrorIs(t, err, ErrNotStarted) - _, err = StartWorker(context.Background(), SessionConfig{Launcher: refusingLauncher{}, Scope: Scope{WorkDir: t.TempDir()}}, Command{Path: "/bin/true"}) - assert.ErrorIs(t, err, ErrNotStarted) - _, err = StartWorker(context.Background(), SessionConfig{}, Command{Path: "/bin/true"}) - assert.ErrorIs(t, err, ErrNotStarted, "the direct launcher needs the record's directory") -} - -func alive(pid int) bool { return syscall.Kill(pid, 0) == nil } - -// startWithChild starts a shell that starts a long child, and returns the -// worker and the child's pid. -func startWithChild(t *testing.T) (*Worker, int) { - t.Helper() - pidFile := filepath.Join(t.TempDir(), "child") - w, err := StartWorker(context.Background(), SessionConfig{Scope: Scope{WorkDir: t.TempDir()}}, - Command{Path: "/bin/sh", Args: []string{"-c", "sleep 300 & echo $! > " + pidFile + "; wait"}, Env: []string{"PATH=/bin:/usr/bin"}}) - require.NoError(t, err) - var child int - require.Eventually(t, func() bool { - data, err := os.ReadFile(pidFile) - if err != nil || len(strings.TrimSpace(string(data))) == 0 { - return false - } - child, err = strconv.Atoi(strings.TrimSpace(string(data))) - return err == nil - }, 5*time.Second, 10*time.Millisecond) - return w, child -} - -func TestTerminateEndsTheWholeProcessGroup(t *testing.T) { - w, child := startWithChild(t) - assert.Equal(t, w.Process().PID, w.Process().PGID) - w.Terminate(time.Second) - assert.Eventually(t, func() bool { return !alive(child) }, 5*time.Second, 20*time.Millisecond, "the worker's own children go with it") -} - -func TestTerminateRecordedLeavesAReusedPidAlone(t *testing.T) { - cmd := exec.CommandContext(context.Background(), "/bin/sleep", "300") - cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} - require.NoError(t, cmd.Start()) - t.Cleanup(func() { _ = cmd.Process.Kill(); _ = cmd.Wait() }) - // The kernel's own identity for it, which is what a record carries. - p, err := LookupProcess(cmd.Process.Pid) - require.NoError(t, err) - require.True(t, p.StartedExact) - - reused := p - reused.StartedAt = p.StartedAt.Add(-time.Hour) - signaled, err := TerminateRecorded(reused, time.Second) - assert.False(t, signaled, "a recorded start time that does not match is another process") - assert.ErrorIs(t, err, ErrGroupOutlivedLeader, "and a group still holding that id is not this worker's to end") - assert.True(t, alive(cmd.Process.Pid)) - - signaled, err = TerminateRecorded(p, 2*time.Second) - require.NoError(t, err) - assert.True(t, signaled) - _ = cmd.Wait() -} - -func TestTerminateReturnsWhenADescendantLeftTheGroupHoldingTheOutput(t *testing.T) { - python, err := exec.LookPath("python3") - if err != nil { - t.Skip("python3 is needed to start a descendant in a new session") - } - pidFile := filepath.Join(t.TempDir(), "escaped") - script := "import os,sys,time\nif os.fork()==0:\n os.setsid()\n open(sys.argv[1],'w').write(str(os.getpid()))\n time.sleep(300)\nelse:\n time.sleep(300)\n" - w, err := StartWorker(context.Background(), SessionConfig{Scope: Scope{WorkDir: t.TempDir()}}, - Command{Path: python, Args: []string{"-c", script, pidFile}, Env: []string{"PATH=/bin:/usr/bin"}}) - require.NoError(t, err) - var escaped int - require.Eventually(t, func() bool { - data, err := os.ReadFile(pidFile) - if err != nil { - return false - } - escaped, err = strconv.Atoi(strings.TrimSpace(string(data))) - return err == nil - }, 5*time.Second, 10*time.Millisecond) - t.Cleanup(func() { _ = syscall.Kill(escaped, syscall.SIGKILL) }) - - done := make(chan struct{}) - go func() { - w.Terminate(100 * time.Millisecond) - close(done) - }() - select { - case <-done: - case <-time.After(10 * time.Second): - t.Fatal("Terminate waited on a descendant outside the worker's group") - } -} - -// Copilot r3: a process group can outlive its leader, and its members may be -// the worker's own children. -func TestAGroupThatOutlivedItsLeaderIsNotSilenceAbsence(t *testing.T) { - w, child := startWithChild(t) - leader := w.Process() - t.Cleanup(func() { _ = syscall.Kill(child, syscall.SIGKILL) }) - - // The leader alone goes; its child keeps the group. - require.NoError(t, syscall.Kill(leader.PID, syscall.SIGKILL)) - <-w.Done() - require.Eventually(t, func() bool { return processStartTimeGone(leader.PID) }, 5*time.Second, 20*time.Millisecond) - - signaled, err := TerminateRecorded(leader, time.Second) - assert.False(t, signaled) - assert.ErrorIs(t, err, ErrGroupOutlivedLeader) - assert.True(t, alive(child), "and the child is left alone for a person to decide about") -} - -// processStartTimeGone reports whether the kernel has no process by that pid. -func processStartTimeGone(pid int) bool { - _, err := processStartTime(pid) - return errors.Is(err, os.ErrNotExist) -} - -// The one-owner rule's identity question: a pid is not an identity. -func TestOwnsWorkerAnswersWhetherThisIsStillTheWorker(t *testing.T) { - w, child := startWithChild(t) - p := w.Process() - t.Cleanup(func() { _ = syscall.Kill(child, syscall.SIGKILL) }) - - owns, err := OwnsWorker(p) - require.NoError(t, err) - assert.True(t, owns, "the worker it started") - - reused := p - reused.StartedAt = p.StartedAt.Add(-time.Hour) - owns, err = OwnsWorker(reused) - assert.False(t, owns, "the same pid with another start time is another process") - assert.ErrorIs(t, err, ErrGroupOutlivedLeader, "and its group still has members") - - owns, err = OwnsWorker(Process{PID: 1 << 30, PGID: 1 << 30, StartedAt: time.Now()}) - assert.False(t, owns) - assert.NoError(t, err, "a pid that names nothing, in a group with no members, is simply gone") - - owns, err = OwnsWorker(Process{}) - assert.False(t, owns) - assert.NoError(t, err, "a session with no process here is nothing to own") -} - -// Copilot r4: only "no such process group" proves a group is gone; a probe -// that was refused is not absence. -func TestOnlyNoSuchProcessGroupProvesAbsence(t *testing.T) { - assert.NoError(t, groupProbe(4242, syscall.ESRCH), "no such group: gone") - assert.ErrorIs(t, groupProbe(4242, nil), ErrGroupOutlivedLeader, "answered: members remain") - assert.ErrorIs(t, groupProbe(4242, syscall.EPERM), ErrGroupOutlivedLeader, "refused: not proven gone") - assert.ErrorIs(t, groupProbe(4242, syscall.EINVAL), ErrGroupOutlivedLeader, "any other answer: not proven gone") -} - -// openDescriptors counts this process's open file descriptors. -func openDescriptors(t *testing.T) int { - t.Helper() - entries, err := os.ReadDir("/proc/self/fd") - if err != nil { - t.Skip("no /proc/self/fd here") - } - return len(entries) -} - -// Copilot via card 22: descriptors have an owner too. A failed start closes -// what it opened, and a terminated worker's output is released. -func TestWorkersDoNotLeakDescriptors(t *testing.T) { - before := openDescriptors(t) - for range 50 { - _, err := StartWorker(context.Background(), SessionConfig{Scope: Scope{WorkDir: t.TempDir()}}, Command{Path: "/nonexistent/claude-not-here"}) - require.ErrorIs(t, err, ErrNotStarted) - } - // At most: an earlier test's worker may release its pipes meanwhile, but - // fifty failed starts that each leaked would be fifty more. - assert.LessOrEqual(t, openDescriptors(t), before, "fifty failed starts leave no descriptor open") - - for range 5 { - w, err := StartWorker(context.Background(), SessionConfig{Scope: Scope{WorkDir: t.TempDir()}}, Command{Path: "/bin/true", Env: []string{}}) - require.NoError(t, err) - w.Terminate(time.Second) - } - assert.Eventually(t, func() bool { return openDescriptors(t) <= before }, 2*pipeWaitDelay+2*time.Second, 50*time.Millisecond, - "a terminated worker's pipes are released without anyone else closing them") -} - -// sleepInItsOwnGroup starts a process that leads a group of its own, and -// gives back the kernel's identity for it. It stands in for whatever holds a -// pid now: a worker of a later attempt, or any process of this user the -// kernel gave a recycled id to. -func sleepInItsOwnGroup(t *testing.T) (*exec.Cmd, Process) { - t.Helper() - cmd := exec.CommandContext(context.Background(), "/bin/sleep", "300") - cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} - require.NoError(t, cmd.Start()) - t.Cleanup(func() { - _ = cmd.Process.Kill() - _ = cmd.Wait() - }) - p, err := LookupProcess(cmd.Process.Pid) - require.NoError(t, err) - require.True(t, p.StartedExact, "the kernel's own start time is the identity") - return cmd, p -} - -// Copilot on #738: the driver records the kernel's start time when it can -// get one, but the comparison accepted anything within three seconds of it, -// so under fast pid reuse a stranger started just after the record was -// written passed as the worker. -func TestAProcessStartedJustAfterTheRecordIsNotTheRecordedOne(t *testing.T) { - cmd, p := sleepInItsOwnGroup(t) - - stranger := p - stranger.StartedAt = p.StartedAt.Add(-time.Second) - gone, err := ProcessGone(stranger) - require.NoError(t, err) - assert.True(t, gone, "a second between the record and the kernel is another process, not this one") - - owns, err := OwnsWorker(stranger) - assert.False(t, owns) - assert.ErrorIs(t, err, ErrGroupOutlivedLeader, "and its group is not settled around either") - - signaled, err := TerminateRecorded(stranger, 100*time.Millisecond) - assert.False(t, signaled) - assert.ErrorIs(t, err, ErrGroupOutlivedLeader) - assert.True(t, alive(cmd.Process.Pid), "nothing is signaled on a record that does not match") -} - -// The wall-clock fallback is not an identity: where the kernel would not say -// when a process started, the rule refuses to answer rather than compare -// against a stamp taken around a fork. -func TestARecordWithNoKernelStartTimeIsRefused(t *testing.T) { - cmd, p := sleepInItsOwnGroup(t) - - stamped := Process{PID: p.PID, PGID: p.PGID, StartedAt: time.Now()} - _, err := ProcessGone(stamped) - assert.ErrorIs(t, err, ErrIdentityUnknown) - - owns, err := OwnsWorker(stamped) - assert.False(t, owns) - assert.ErrorIs(t, err, ErrIdentityUnknown, "neither owned nor gone: unanswerable") - - signaled, err := TerminateRecorded(stamped, 100*time.Millisecond) - assert.False(t, signaled) - assert.ErrorIs(t, err, ErrIdentityUnknown) - assert.True(t, alive(cmd.Process.Pid)) - - // And a record with a pid but no start time at all — a ledger row - // written where the kernel could not be asked — is the same answer, not - // "gone, settle it". - owns, err = OwnsWorker(Process{PID: p.PID, PGID: p.PGID}) - assert.False(t, owns) - assert.ErrorIs(t, err, ErrIdentityUnknown) -} - -// Copilot on #738: group identity was checked once, before the grace period, -// and the SIGKILL that followed went out by the saved negative pgid however -// long the wait had been. This is what that costs once the id has changed -// hands: the record names a pid that now leads somebody else's group. -func TestALaterGroupSignalIsNotSentToAGroupTheRecordNoLongerOwns(t *testing.T) { - cmd, p := sleepInItsOwnGroup(t) - // What the connector recorded a while ago for the worker that had this - // pid before the kernel gave it away. - recorded := p - recorded.StartedAt = p.StartedAt.Add(-time.Hour) - - err := ConfirmGroupGone(recorded, 100*time.Millisecond) - assert.ErrorIs(t, err, ErrGroupOutlivedLeader, "the group is not proven gone") - // Not alive(), which counts the zombie this test has not reaped: the - // question is whether anything of that group still runs. - assert.True(t, GroupMembersRemain(p), "and the group that holds the id now is left running") - _ = cmd - - // The same rule, asked directly: nothing is signaled on a record whose - // identity cannot be established either. - assert.ErrorIs(t, signalRecordedGroup(Process{PID: p.PID, PGID: p.PGID, StartedAt: time.Now()}, syscall.SIGKILL), ErrIdentityUnknown) - assert.True(t, alive(cmd.Process.Pid)) - - // And the worker it really is may still be ended by its group. - require.NoError(t, signalRecordedGroup(p, syscall.SIGKILL)) - assert.Eventually(t, func() bool { return !GroupMembersRemain(p) }, 5*time.Second, 20*time.Millisecond) -} diff --git a/internal/connector/driver/env.go b/internal/connector/driver/env.go deleted file mode 100644 index dd267b285..000000000 --- a/internal/connector/driver/env.go +++ /dev/null @@ -1,59 +0,0 @@ -package driver - -import ( - "slices" - "strings" -) - -// BaseEnv is the environment every worker process may get from the -// connector's own: what a program needs to find its home, its tools, its -// locale and its terminal, and nothing that authenticates anyone. A driver -// adds the few variables its agent needs by name; nothing is passed by -// pattern. -var BaseEnv = []string{ - "HOME", "PATH", "USER", "LOGNAME", "SHELL", "LANG", "LC_ALL", "LC_CTYPE", - "TERM", "TMPDIR", "TZ", - "XDG_CONFIG_HOME", "XDG_DATA_HOME", "XDG_STATE_HOME", "XDG_CACHE_HOME", "XDG_RUNTIME_DIR", -} - -// BuildEnv is the environment made of the allowlisted names that lookup has, -// plus extra, which wins over a looked-up value of the same name. Its output -// is sorted, so the same inputs make the same environment. -// -// lookup is os.LookupEnv in production. A name is taken only as given: no -// prefix, no pattern, so a new variable of the host's never reaches a worker -// by resembling an allowed one. -func BuildEnv(allow []string, lookup func(string) (string, bool), extra map[string]string) []string { - values := map[string]string{} - for _, name := range allow { - if name == "" || strings.ContainsAny(name, "=\x00") { - continue - } - if v, ok := lookup(name); ok { - values[name] = v - } - } - for k, v := range extra { - if k == "" || strings.ContainsAny(k, "=\x00") { - continue - } - values[k] = v - } - out := make([]string, 0, len(values)) - for k, v := range values { - out = append(out, k+"="+v) - } - slices.Sort(out) - return out -} - -// EnvMap is BuildEnv's result as a map, for an MCPServer's Env. -func EnvMap(env []string) map[string]string { - out := make(map[string]string, len(env)) - for _, kv := range env { - if k, v, ok := strings.Cut(kv, "="); ok { - out[k] = v - } - } - return out -} diff --git a/internal/connector/driver/preflight.go b/internal/connector/driver/preflight.go deleted file mode 100644 index 285238ec4..000000000 --- a/internal/connector/driver/preflight.go +++ /dev/null @@ -1,345 +0,0 @@ -package driver - -import ( - "bytes" - "context" - "errors" - "fmt" - "io/fs" - "os" - "os/exec" - "regexp" - "slices" - "strings" - "time" -) - -// A preflight starts a worker's own program the way a session would — the -// same binary, found on the same PATH, with the same environment — and asks -// it three things without any work and without a model call: does it start, -// does it know every flag a session passes it, and is it logged in. What it -// finds is said in a person's words, with the fix, because the person reading -// it is the one who runs the agent, not the one who mentioned it. - -// Preflighter is a driver that can check its worker before any work is given -// to it. -type Preflighter interface { - Preflight(ctx context.Context, policy PermissionPolicy) Preflight -} - -// Preflight is what a worker's preflight found. -type Preflight struct { - // Product is the worker's name for a person: "Claude Code". - Product string - // Version is what the worker said its version is; empty when it did not - // start. - Version string - Checks []PreflightCheck -} - -// Failed is the first check that failed. -func (p Preflight) Failed() (PreflightCheck, bool) { - for _, c := range p.Checks { - if c.Status == PreflightFail { - return c, true - } - } - return PreflightCheck{}, false -} - -// PreflightCheck is one question a preflight asked. -type PreflightCheck struct { - Name string - Status PreflightStatus - Message string - // Hint is the fix, where one is known. - Hint string -} - -// PreflightStatus is how a check went. A warning is something the preflight -// could not settle; it never stops anything. -type PreflightStatus string - -const ( - PreflightPass PreflightStatus = "pass" - PreflightFail PreflightStatus = "fail" - PreflightWarn PreflightStatus = "warn" -) - -// The checks, by name. -const ( - PreflightStarts = "starts" - PreflightFlags = "flags" - PreflightLogin = "login" -) - -// PreflightTimeout bounds each of a preflight's runs. -const PreflightTimeout = 30 * time.Second - -// WorkerProbe is how one driver's preflight runs. -type WorkerProbe struct { - // Product is the worker's name for a person: "Claude Code". - Product string - // Binary is the program a session starts, found on PATH as a session - // finds it. - Binary string - // Env is the environment a session gets. - Env []string - // Help is the arguments that list the flags Flags must be among. - Help []string - // Flags are the flags a session passes. - Flags []string - // Update is how a person updates the worker. - Update string - // Login is the arguments that ask whether the worker is logged in, with - // no model call. - Login []string - // LoggedIn reads Login's answer: whether the worker said it is logged in, - // and whether it said anything that could be read. - LoggedIn func(ProbeResult) (loggedIn, known bool) - // LoginFix is what a person does when the worker is logged out. - LoginFix string - // LoginWarns is a login check that never fails: its answer is a guess. - LoginWarns bool - // Timeout bounds each run; PreflightTimeout when zero. - Timeout time.Duration - // Run runs one probe; RunProbe when nil. A test seam. - Run func(ctx context.Context, cmd Command, timeout time.Duration) ProbeResult -} - -// ProbeResult is one short run of a worker's program. -type ProbeResult struct { - Stdout, Stderr string - // Exit is the exit status; -1 when the program did not exit on its own. - Exit int - // NotFound is a program PATH does not have. - NotFound bool - // TimedOut is a program that did not answer in time. - TimedOut bool - // Err is why the program could not be run, when it could not. - Err error -} - -// probeOutputLimit bounds what a probe keeps of each stream. -const probeOutputLimit = 256 << 10 - -// RunProbe runs cmd with no input, bounded by timeout. The program is found as -// StartWorker finds it. -func RunProbe(ctx context.Context, cmd Command, timeout time.Duration) ProbeResult { - ctx, cancel := context.WithTimeout(ctx, timeout) - defer cancel() - ec := exec.CommandContext(ctx, cmd.Path, cmd.Args...) //nolint:gosec // G204: the driver's own binary and flags, never content - ec.Dir = cmd.Dir - ec.Env = cmd.Env - if ec.Env == nil { - ec.Env = []string{} - } - ec.WaitDelay = 2 * time.Second - group := probeInItsOwnGroup(ec) - defer group.cleanup() - var stdout, stderr limitedBuffer - stdout.max, stderr.max = probeOutputLimit, probeOutputLimit - ec.Stdout, ec.Stderr = &stdout, &stderr - err := ec.Start() - if err == nil { - group.started() - err = ec.Wait() - } - out := ProbeResult{Stdout: stdout.String(), Stderr: stderr.String(), Exit: -1} - if ec.ProcessState != nil { - out.Exit = ec.ProcessState.ExitCode() - } - var exitErr *exec.ExitError - switch { - case ctx.Err() != nil && errors.Is(ctx.Err(), context.DeadlineExceeded): - out.TimedOut = true - case errors.Is(err, exec.ErrNotFound), errors.Is(err, fs.ErrNotExist) && ec.ProcessState == nil && !isFile(ec.Path): - // Missing only when the program itself is: a program that is there - // but whose interpreter or loader is gone fails its exec with the - // same ENOENT, and is a program that can't start, not one PATH lacks. - out.NotFound = true - out.Err = err - case errors.As(err, &exitErr): - case err != nil: - out.Err = err - } - return out -} - -func isFile(path string) bool { - info, err := os.Stat(path) - return err == nil && !info.IsDir() -} - -type limitedBuffer struct { - bytes.Buffer - max int -} - -func (b *limitedBuffer) Write(p []byte) (int, error) { - if room := b.max - b.Len(); room > 0 { - b.Buffer.Write(p[:min(len(p), room)]) - } - return len(p), nil -} - -// Check runs the preflight: the start first, and the rest only when it -// started. -func (w WorkerProbe) Check(ctx context.Context) Preflight { - out := Preflight{Product: w.Product} - red := NewRedactor(Redaction{Env: w.Env}) - path, _ := exec.LookPath(w.Binary) - - started := w.run(ctx, "--version") - if c, ok := w.startFailure(started, path, red); ok { - out.Checks = append(out.Checks, c) - return out - } - out.Version = versionIn(started.Stdout) - named := w.Product - if out.Version != "" { - named += " " + out.Version - } - out.Checks = append(out.Checks, PreflightCheck{Name: PreflightStarts, Status: PreflightPass, Message: named + " starts (" + path + ")"}) - - out.Checks = append(out.Checks, w.flagsCheck(ctx, named, red)) - if len(w.Login) > 0 { - out.Checks = append(out.Checks, w.loginCheck(ctx, red)) - } - return out -} - -func (w WorkerProbe) run(ctx context.Context, args ...string) ProbeResult { - run := w.Run - if run == nil { - run = RunProbe - } - timeout := w.Timeout - if timeout <= 0 { - timeout = PreflightTimeout - } - return run(ctx, Command{Path: w.Binary, Args: args, Env: w.Env}, timeout) -} - -func (w WorkerProbe) command(args ...string) string { - return "`" + strings.Join(append([]string{w.Binary}, args...), " ") + "`" -} - -// launcherMissing is what a shell or a wrapper says when the program it -// hands over to is not there. -var launcherMissing = regexp.MustCompile(`(?i)no such file or directory|not found|cannot execute`) - -// startFailure is the start check when the worker did not start. -func (w WorkerProbe) startFailure(r ProbeResult, path string, red *Redactor) (PreflightCheck, bool) { - c := PreflightCheck{Name: PreflightStarts, Status: PreflightFail} - said := red.Stderr(r.Stderr) - version := w.command("--version") - switch { - case r.NotFound: - c.Message = fmt.Sprintf("%s isn't installed here: %s is not on the PATH the connector starts with", w.Product, w.Binary) - c.Hint = fmt.Sprintf("Install %s, or put %s on the PATH the connector starts with.", w.Product, w.Binary) - case r.TimedOut: - c.Message = fmt.Sprintf("%s didn't answer %s in time", w.Product, version) - c.Hint = fmt.Sprintf("Run %s yourself to see what it's waiting for.", version) - case r.Err != nil: - c.Message = fmt.Sprintf("%s couldn't be started: %s", w.Product, red.Sanitize(r.Err.Error())) - c.Hint = fmt.Sprintf("Run %s yourself to see why.", version) - case r.Exit == 0: - return PreflightCheck{}, false - case r.Exit == 126 || r.Exit == 127 || launcherMissing.MatchString(said): - c.Message = fmt.Sprintf("The %s on your PATH is a launcher that couldn't find %s (%s)", w.Binary, w.Product, orExit(said, r.Exit)) - c.Hint = fmt.Sprintf("Reinstall %s, or fix the launcher at %s.", w.Product, path) - default: - c.Message = fmt.Sprintf("%s couldn't start: %s exited with status %d (%s)", w.Product, version, r.Exit, orExit(said, r.Exit)) - c.Hint = fmt.Sprintf("Run %s yourself to see why.", version) - } - return c, true -} - -func orExit(said string, exit int) string { - if said == "" { - return fmt.Sprintf("it exited with status %d and said nothing", exit) - } - return said -} - -// flagsCheck asks the worker's help for every flag a session passes. A -// worker too old to know one would refuse the session it was started for. -func (w WorkerProbe) flagsCheck(ctx context.Context, named string, red *Redactor) PreflightCheck { - c := PreflightCheck{Name: PreflightFlags} - r := w.run(ctx, w.Help...) - if r.Exit != 0 || r.TimedOut || r.Err != nil { - c.Status = PreflightFail - c.Message = fmt.Sprintf("%s wouldn't list its options: %s failed (%s)", named, w.command(w.Help...), orExit(red.Stderr(r.Stderr), r.Exit)) - c.Hint = fmt.Sprintf("Run %s yourself to see why.", w.command(w.Help...)) - return c - } - missing := MissingFlags(r.Stdout+"\n"+r.Stderr, w.Flags) - if len(missing) > 0 { - c.Status = PreflightFail - c.Message = fmt.Sprintf("%s is too old for the connector — update it", named) - c.Hint = fmt.Sprintf("It doesn't know %s. %s", strings.Join(missing, ", "), w.Update) - return c - } - c.Status = PreflightPass - c.Message = fmt.Sprintf("knows all %d options the connector passes", len(w.Flags)) - return c -} - -func (w WorkerProbe) loginCheck(ctx context.Context, red *Redactor) PreflightCheck { - c := PreflightCheck{Name: PreflightLogin} - r := w.run(ctx, w.Login...) - loggedIn, known := false, false - if !r.TimedOut && r.Err == nil { - loggedIn, known = w.LoggedIn(r) - } - switch { - case known && loggedIn: - c.Status = PreflightPass - c.Message = "logged in" - case known: - c.Status = PreflightFail - c.Message = fmt.Sprintf("%s is logged out on this computer — %s", w.Product, w.LoginFix) - if w.LoginWarns { - c.Status = PreflightWarn - c.Message = fmt.Sprintf("%s may be logged out on this computer — %s", w.Product, w.LoginFix) - } - default: - c.Status = PreflightWarn - c.Message = fmt.Sprintf("couldn't tell whether %s is logged in: %s answered %s", w.Product, w.command(w.Login...), orExit(red.Stderr(r.Stderr), r.Exit)) - } - return c -} - -var versionPattern = regexp.MustCompile(`\d+\.\d+[0-9A-Za-z.+-]*`) - -// versionIn is the version a --version answer names. -func versionIn(stdout string) string { - first, _, _ := strings.Cut(strings.TrimSpace(stdout), "\n") - return versionPattern.FindString(first) -} - -// FlagsOf is every flag in a command line, once each, in order. A lone "-" -// (read the prompt from stdin) is an argument, not a flag. -func FlagsOf(args []string) []string { - var flags []string - for _, a := range args { - if len(a) > 1 && a != "--" && strings.HasPrefix(a, "-") && !slices.Contains(flags, a) { - flags = append(flags, a) - } - } - return flags -} - -// MissingFlags is the flags help does not list. A flag counts as listed only -// as a whole word: --tools is not found inside --allowed-tools. -func MissingFlags(help string, flags []string) []string { - var missing []string - for _, f := range flags { - listed := regexp.MustCompile(`(^|[\s,\[(|])` + regexp.QuoteMeta(f) + `($|[\s,=\])|<])`) - if !listed.MatchString(help) { - missing = append(missing, f) - } - } - return missing -} diff --git a/internal/connector/driver/preflight_test.go b/internal/connector/driver/preflight_test.go deleted file mode 100644 index 98f40d9b1..000000000 --- a/internal/connector/driver/preflight_test.go +++ /dev/null @@ -1,216 +0,0 @@ -//go:build unix - -package driver - -import ( - "context" - "encoding/json" - "fmt" - "os" - "path/filepath" - "strconv" - "strings" - "syscall" - "testing" - "time" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -// fakeWorker writes a program that answers a preflight as a worker would: -// its version, a help listing flags, and a login status. -func fakeWorker(t *testing.T, script string) string { - t.Helper() - path := filepath.Join(t.TempDir(), "worker") - require.NoError(t, os.WriteFile(path, []byte("#!/bin/sh\n"+script), 0o700)) - return path -} - -const fakeWorkerAnswers = ` -case "$1" in - --version) echo "2.1.283 (Claude Code)" ;; - --help) echo "Usage: worker [options]"; echo " -p, --print"; echo " --allowed-tools "; echo " --tools " ;; - auth) if [ -n "$LOGGED_IN" ]; then echo '{"loggedIn":true,"email":"someone@example.com"}'; else echo '{"loggedIn":false}'; exit 1; fi ;; -esac -` - -func testProbe(binary string, env ...string) WorkerProbe { - return WorkerProbe{ - Product: "Claude Code", - Binary: binary, - Env: append([]string{"PATH=/usr/bin:/bin"}, env...), - Help: []string{"--help"}, - Flags: []string{"-p", "--tools", "--allowed-tools"}, - Update: "Update Claude Code: claude update", - Login: []string{"auth", "status", "--json"}, - LoggedIn: func(r ProbeResult) (bool, bool) { return testLoggedIn(r) }, - LoginFix: "run `claude` and log in", - } -} - -func testLoggedIn(r ProbeResult) (bool, bool) { - var s struct { - LoggedIn *bool `json:"loggedIn"` - } - if json.Unmarshal([]byte(r.Stdout), &s) != nil || s.LoggedIn == nil { - return false, false - } - return *s.LoggedIn, true -} - -func TestPreflightPassesAWorkerThatStartsKnowsItsFlagsAndIsLoggedIn(t *testing.T) { - p := testProbe(fakeWorker(t, fakeWorkerAnswers), "LOGGED_IN=1").Check(context.Background()) - _, failed := p.Failed() - assert.False(t, failed) - assert.Equal(t, "2.1.283", p.Version) - require.Len(t, p.Checks, 3) - for _, c := range p.Checks { - assert.Equal(t, PreflightPass, c.Status, c.Name) - assert.NotContains(t, c.Message, "someone@example.com", "nothing the login answer says about the person is repeated") - } -} - -func TestPreflightRunsTheWorkerWithTheSessionsEnvironmentOnly(t *testing.T) { - t.Setenv("LOGGED_IN", "1") - p := testProbe(fakeWorker(t, fakeWorkerAnswers)).Check(context.Background()) - c, failed := p.Failed() - require.True(t, failed, "the connector's own environment is not the worker's") - assert.Equal(t, PreflightLogin, c.Name) -} - -func TestPreflightSaysWhyAWorkerCannotStart(t *testing.T) { - cases := map[string]struct { - binary string - message string - hint string - }{ - "not installed": { - binary: "basecamp-connect-no-such-worker", - message: "Claude Code isn't installed here: basecamp-connect-no-such-worker is not on the PATH the connector starts with", - hint: "Install Claude Code, or put basecamp-connect-no-such-worker on the PATH the connector starts with.", - }, - "a launcher whose target is gone": { - binary: fakeWorker(t, `exec "$HOME/.local/share/mise/installs/claude/latest/claude" "$@"`), - message: "The %s on your PATH is a launcher that couldn't find Claude Code", - }, - "a worker that fails": { - binary: fakeWorker(t, `echo "config is broken" >&2; exit 3`), - message: "Claude Code couldn't start: `%s --version` exited with status 3 (config is broken)", - }, - } - for name, tc := range cases { - t.Run(name, func(t *testing.T) { - p := testProbe(tc.binary, "HOME="+t.TempDir()).Check(context.Background()) - require.Len(t, p.Checks, 1, "nothing else is asked of a worker that did not start") - c, failed := p.Failed() - require.True(t, failed) - assert.Equal(t, PreflightStarts, c.Name) - assert.Contains(t, c.Message, sprintfIf(tc.message, tc.binary)) - if tc.hint != "" { - assert.Equal(t, tc.hint, c.Hint) - } - assert.Empty(t, p.Version) - }) - } -} - -func TestPreflightNamesTheVersionTooOldForTheConnector(t *testing.T) { - probe := testProbe(fakeWorker(t, fakeWorkerAnswers), "LOGGED_IN=1") - probe.Flags = append(probe.Flags, "--permission-prompts", "--strict-mcp-config") - c, failed := probe.Check(context.Background()).Failed() - require.True(t, failed) - assert.Equal(t, PreflightFlags, c.Name) - assert.Equal(t, "Claude Code 2.1.283 is too old for the connector — update it", c.Message) - assert.Equal(t, "It doesn't know --permission-prompts, --strict-mcp-config. Update Claude Code: claude update", c.Hint) -} - -func TestPreflightSaysAWorkerIsLoggedOut(t *testing.T) { - c, failed := testProbe(fakeWorker(t, fakeWorkerAnswers)).Check(context.Background()).Failed() - require.True(t, failed) - assert.Equal(t, PreflightLogin, c.Name) - assert.Equal(t, "Claude Code is logged out on this computer — run `claude` and log in", c.Message) - - probe := testProbe(fakeWorker(t, fakeWorkerAnswers)) - probe.LoginWarns = true - p := probe.Check(context.Background()) - _, failed = p.Failed() - assert.False(t, failed, "a login the worker can only guess at never stops anything") - assert.Equal(t, PreflightWarn, p.Checks[2].Status) -} - -func TestPreflightCannotTellALoginItCannotRead(t *testing.T) { - probe := testProbe(fakeWorker(t, `case "$1" in --version) echo 1.0.0 ;; --help) echo "-p --tools --allowed-tools" ;; *) echo "unknown command" >&2; exit 2 ;; esac`)) - p := probe.Check(context.Background()) - _, failed := p.Failed() - assert.False(t, failed) - assert.Equal(t, PreflightWarn, p.Checks[2].Status) - assert.Contains(t, p.Checks[2].Message, "couldn't tell whether Claude Code is logged in") -} - -func TestMissingFlagsMatchesWholeFlagsOnly(t *testing.T) { - help := " -p, --print\n --allowed-tools \n --mcp-config \n -c, --config " - assert.Equal(t, []string{"--tools", "--mcp"}, MissingFlags(help, []string{"-p", "--tools", "--allowed-tools", "--mcp", "--mcp-config", "-c"})) -} - -func TestFlagsOfSkipsArguments(t *testing.T) { - assert.Equal(t, []string{"--json", "-c", "--disable"}, - FlagsOf([]string{"exec", "--json", "-c", "a=1", "-c", "b=2", "--disable", "x", "-"})) -} - -func sprintfIf(format, arg string) string { - if strings.Contains(format, "%s") { - return fmt.Sprintf(format, arg) - } - return format -} - -// A probe that times out ends its whole process tree: a launcher that started -// the real worker as a child and waited must not leave that child running -// (Codex on #794). -func TestRunProbeEndsTheLaunchersChildrenWhenItTimesOut(t *testing.T) { - dir := t.TempDir() - pidFile := filepath.Join(dir, "child.pid") - launcher := filepath.Join(dir, "claude") - require.NoError(t, os.WriteFile(launcher, []byte("#!/bin/sh\nsleep 60 &\necho $! > "+pidFile+"\nwait\n"), 0o700)) - - r := RunProbe(context.Background(), Command{Path: launcher, Dir: dir}, 300*time.Millisecond) - require.True(t, r.TimedOut) - raw, err := os.ReadFile(pidFile) - require.NoError(t, err) - pid, err := strconv.Atoi(strings.TrimSpace(string(raw))) - require.NoError(t, err) - assert.Eventually(t, func() bool { return syscall.Kill(pid, 0) != nil }, 5*time.Second, 20*time.Millisecond, "the launcher's child is gone") -} - -// A program that is on PATH but can't be run — its interpreter is gone — is -// not reported as missing from PATH (Codex on #794). -func TestRunProbeTellsAMissingInterpreterFromAMissingProgram(t *testing.T) { - dir := t.TempDir() - launcher := filepath.Join(dir, "claude") - require.NoError(t, os.WriteFile(launcher, []byte("#!/nonexistent/interpreter\n"), 0o700)) - - r := RunProbe(context.Background(), Command{Path: launcher, Dir: dir}, 5*time.Second) - assert.False(t, r.NotFound, "the program is there") - require.Error(t, r.Err) - - missing := RunProbe(context.Background(), Command{Path: filepath.Join(dir, "absent"), Dir: dir}, 5*time.Second) - assert.True(t, missing.NotFound) -} - -// A launcher that leaves a child behind and exits on its own still has that -// child ended when the probe returns (Codex on #794). -func TestRunProbeEndsChildrenALauncherLeftWhenItExited(t *testing.T) { - dir := t.TempDir() - pidFile := filepath.Join(dir, "child.pid") - launcher := filepath.Join(dir, "claude") - require.NoError(t, os.WriteFile(launcher, []byte("#!/bin/sh\nsleep 60 &\necho $! > "+pidFile+"\necho 2.1.0\n"), 0o700)) - - r := RunProbe(context.Background(), Command{Path: launcher, Dir: dir}, 10*time.Second) - require.False(t, r.TimedOut) - raw, err := os.ReadFile(pidFile) - require.NoError(t, err) - pid, err := strconv.Atoi(strings.TrimSpace(string(raw))) - require.NoError(t, err) - assert.Eventually(t, func() bool { return syscall.Kill(pid, 0) != nil }, 5*time.Second, 20*time.Millisecond, "the child is gone") -} diff --git a/internal/connector/driver/probe_reuse_unix_test.go b/internal/connector/driver/probe_reuse_unix_test.go deleted file mode 100644 index b3505fb1d..000000000 --- a/internal/connector/driver/probe_reuse_unix_test.go +++ /dev/null @@ -1,69 +0,0 @@ -//go:build unix - -package driver - -import ( - "os" - "os/exec" - "path/filepath" - "strconv" - "strings" - "syscall" - "testing" - "time" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -// A probe's cleanup runs after the probe has been reaped, when its pid may -// already lead someone else's group. That group is left alone (Codex on #794). -func TestProbeCleanupLeavesAReusedPidAlone(t *testing.T) { - stranger := exec.CommandContext(t.Context(), "sleep", "30") - stranger.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} - require.NoError(t, stranger.Start()) - exited := make(chan struct{}) - go func() { _ = stranger.Wait(); close(exited) }() - t.Cleanup(func() { _ = stranger.Process.Kill(); <-exited }) - pid := stranger.Process.Pid - - // The probe that had this pid started earlier: its recorded start time is - // not the stranger's. - g := &probeGroup{recorded: true, pid: pid, startedAt: time.Unix(1, 0), exact: true} - g.cleanup() - - select { - case <-exited: - t.Fatal("the stranger was signaled") - case <-time.After(300 * time.Millisecond): - } -} - -// A launcher can exit before the probe records it, leaving a child in its -// group: its start time is gone by then, and cleanup still ends the child -// (Codex on #794). -func TestProbeCleanupEndsTheChildOfALauncherThatExitedBeforeItWasRecorded(t *testing.T) { - dir := t.TempDir() - pidFile := filepath.Join(dir, "child.pid") - launcher := exec.CommandContext(t.Context(), "/bin/sh", "-c", "sleep 60 >/dev/null 2>&1 & echo $! > "+pidFile) - g := probeInItsOwnGroup(launcher) - require.NoError(t, launcher.Start()) - var child int - require.Eventually(t, func() bool { - raw, err := os.ReadFile(pidFile) - if err != nil || !strings.HasSuffix(string(raw), "\n") { - return false - } - child, err = strconv.Atoi(strings.TrimSpace(string(raw))) - return err == nil - }, 5*time.Second, 10*time.Millisecond) - t.Cleanup(func() { _ = syscall.Kill(child, syscall.SIGKILL) }) - require.Eventually(t, func() bool { _, err := processStartTime(launcher.Process.Pid); return err != nil }, - 5*time.Second, 10*time.Millisecond, "the launcher has exited") - - g.started() - _ = launcher.Wait() - g.cleanup() - - assert.Eventually(t, func() bool { return syscall.Kill(child, 0) != nil }, 5*time.Second, 20*time.Millisecond, "the child is gone") -} diff --git a/internal/connector/driver/redact.go b/internal/connector/driver/redact.go deleted file mode 100644 index 146492a79..000000000 --- a/internal/connector/driver/redact.go +++ /dev/null @@ -1,366 +0,0 @@ -package driver - -import ( - "context" - "errors" - "fmt" - "log/slog" - "path/filepath" - "regexp" - "slices" - "strings" - "unicode" -) - -// # Redaction: what leaves a worker, and what is taken out of it first -// -// Everything that crosses out of a worker toward a person or a file — an -// error a driver returns, a log line, a dispatch status line, a tool name in -// an update, the tail of the adapter's stderr — passes through one function, -// Redactor.Sanitize, before it is written anywhere. Err, Stderr and Handler -// are Sanitize applied to an error, to stderr and to a logger; nothing else -// in the connector redacts on its own. -// -// Sanitize removes, in this order: -// -// 1. Every character that drives a terminal rather than printing on one: -// C0 but for tab and newline, DEL, and the C1 block. It goes first so -// that an escape character inside a secret cannot carry it past the -// rules below. -// 2. Every value in Redaction.Secrets, wherever it appears: the task token -// and the agent's credentials, named by whoever holds them. -// 3. Every value of the worker's environment and of its MCP servers' -// environments (Redaction.Env) that BaseEnv does not name. BaseEnv is -// the operator's home, path, locale and terminal, chosen because none of -// it authenticates anyone; everything a driver or the dispatcher adds by -// name (an API key, a config directory) is a value the agent was given, -// and is taken out. Values shorter than minEnvValue are left, since a -// one-character value would take out every letter it matches. -// 4. Every path under Redaction.Dirs — the connector's state directory, -// which holds the ledger, and its runtime directory, which holds session -// files and token sockets — to the end of the path, whether it is written -// as given or with its symlinks resolved. -// 5. Email addresses: agents volunteer the signed-in account's address -// unprompted. -// 6. Credential-shaped runs: a bearer header's value, and any unbroken run -// of 40 or more token characters. -// -// Stderr is further never passed on verbatim: only its last line is kept, -// sanitized, stripped of control characters and cut to maxStderr bytes. -// -// A nil *Redactor still applies rules 1, 5 and 6, so no caller is ever -// without the control and pattern rules. -// -// Where this can still be broken: a secret the Redactor was not told about -// and that has no credential shape (a short password, say) passes; a secret -// the agent transforms before it writes it (base64, reversed, split across -// lines) passes; and a path outside the named directories is shown as it is. -// The rule removes what the connector knows is secret; it cannot recognize a -// secret it was never shown. - -// Redaction names what a Redactor takes out. -type Redaction struct { - // Secrets are values removed wherever they appear: a task token, an - // agent credential. - Secrets []string - // Env is an environment, as KEY=VALUE, whose values are removed unless - // BaseEnv names them. - Env []string - // Dirs are directories any path under which is removed: the state and - // runtime directories. - Dirs []string -} - -// With is r with more added. -func (r Redaction) With(more Redaction) Redaction { - return Redaction{ - Secrets: append(slices.Clone(r.Secrets), more.Secrets...), - Env: append(slices.Clone(r.Env), more.Env...), - Dirs: append(slices.Clone(r.Dirs), more.Dirs...), - } -} - -// EnvOf is an MCP server's environment map as KEY=VALUE, for Redaction.Env. -func EnvOf(m map[string]string) []string { - out := make([]string, 0, len(m)) - for k, v := range m { - out = append(out, k+"="+v) - } - return out -} - -const ( - // minEnvValue is the shortest environment value removed by value. - minEnvValue = 6 - // maxStderr is the most of a worker's stderr ever passed on, per line. - maxStderr = 300 - // maxStderrLines is how many of a worker's last stderr lines Lines - // returns: enough that a refusal is not lost behind the diagnostics that - // follow it, few enough to be a bound. - maxStderrLines = 50 -) - -const ( - redactedSecret = "[redacted]" - redactedPath = "[connector path]" - redactedEmail = "[email redacted]" - redactedCred = "[credential redacted]" //nolint:gosec // G101: the placeholder that replaces a credential, not one -) - -var ( - emailPattern = regexp.MustCompile(`[A-Za-z0-9._%+\-]+@[A-Za-z0-9.\-]+\.[A-Za-z]{2,}`) - // bearerPattern is a credential-shaped run: a bearer header value or a - // long unbroken token. - bearerPattern = regexp.MustCompile(`(?i)\bbearer\s+[A-Za-z0-9._~+/\-]+=*|\b[A-Za-z0-9_\-]{40,}\b`) -) - -// Redactor applies a Redaction. Build one with NewRedactor; it is safe for -// concurrent use. -type Redactor struct { - values *strings.Replacer - paths *regexp.Regexp -} - -// NewRedactor compiles r. -func NewRedactor(r Redaction) *Redactor { - seen := map[string]bool{} - var values []string - add := func(v string) { - if v != "" && !seen[v] { - seen[v] = true - values = append(values, v) - } - } - for _, s := range r.Secrets { - add(s) - } - base := map[string]bool{} - for _, name := range BaseEnv { - base[name] = true - } - for _, kv := range r.Env { - name, value, ok := strings.Cut(kv, "=") - if ok && !base[name] && len(value) >= minEnvValue { - add(value) - } - } - // Longest first, so a value that contains another is removed whole. - slices.SortFunc(values, func(a, b string) int { return len(b) - len(a) }) - pairs := make([]string, 0, 2*len(values)) - for _, v := range values { - pairs = append(pairs, v, redactedSecret) - } - - var dirs []string - for _, d := range r.Dirs { - if d == "" { - continue - } - d = filepath.Clean(d) - dirs = append(dirs, d) - if resolved, err := filepath.EvalSymlinks(d); err == nil && resolved != d { - dirs = append(dirs, resolved) - } - } - slices.SortFunc(dirs, func(a, b string) int { return len(b) - len(a) }) - var paths *regexp.Regexp - if len(dirs) > 0 { - alternatives := make([]string, len(dirs)) - for i, d := range dirs { - alternatives[i] = regexp.QuoteMeta(d) - } - // The directory, and the rest of the path up to the first character - // that ends a path in a message: a space, a quote, a bracket, or the - // punctuation an error puts after a file name. - paths = regexp.MustCompile(`(?:` + strings.Join(alternatives, "|") + `)(?:/[^\s"'` + "`" + `)\]:;,]*)?`) - } - return &Redactor{values: strings.NewReplacer(pairs...), paths: paths} -} - -// Sanitize is the one function every text crossing out of a worker passes -// through. See the rule above. -func (r *Redactor) Sanitize(s string) string { - // Controls first: a terminal is the connector's stderr, and stripping - // them afterwards would let an escape character sitting inside a secret - // carry that secret past the replacer. - s = stripControls(s) - if r != nil { - s = r.values.Replace(s) - if r.paths != nil { - s = r.paths.ReplaceAllString(s, redactedPath) - } - } - s = emailPattern.ReplaceAllString(s, redactedEmail) - return bearerPattern.ReplaceAllString(s, redactedCred) -} - -// stripControls removes every character that drives a terminal rather than -// printing on one: C0, DEL and the C1 block. What is sanitized ends up on -// the connector's stderr and in its log, and the fields interpolated into it -// are a worker's own — the permission mode Claude reports, a tool name — so -// a malformed or hostile value must not be able to move the cursor, repaint -// the screen or start an escape sequence there. -// -// Tab and newline stay. Neither drives a terminal, and both are what a -// legitimate multi-line log field is made of; a message that must be one -// line says so itself (Redactor.line, which maps every remaining control to -// a space). Carriage return does not stay: it rewrites the line it is on, -// which is how output is made to lie about what it said. -func stripControls(s string) string { - return strings.Map(func(c rune) rune { - switch { - case c == '\t' || c == '\n': - return c - case c < 0x20, c == 0x7f, c >= 0x80 && c <= 0x9f: - return -1 - } - return c - }, s) -} - -// Stderr is what may be passed on of a worker's stderr: its last non-empty -// line, sanitized, on one line, and no longer than maxStderr bytes. -func (r *Redactor) Stderr(text string) string { - lines := r.Lines(text) - if len(lines) == 0 { - return "" - } - return lines[len(lines)-1] -} - -// Lines is what may be passed on of a worker's stderr when the LAST line is -// not enough: its last maxStderrLines non-empty lines, each sanitized, on one -// line and no longer than maxStderr bytes, oldest first. -// -// Stderr gives the last line, which is where a program that could not start -// says why. A refusal, though, is written when it happens and whatever the -// agent prints afterwards buries it, so a driver that reads refusals from -// stderr reads them here (driver.go's "Refusals"). -func (r *Redactor) Lines(text string) []string { - raw := strings.Split(text, "\n") - out := make([]string, 0, len(raw)) - for _, line := range raw { - if clean := r.line(line); clean != "" { - out = append(out, clean) - } - } - if len(out) > maxStderrLines { - out = out[len(out)-maxStderrLines:] - } - return out -} - -// line is one line of a worker's output, sanitized, on one line and bounded. -func (r *Redactor) line(text string) string { - text = r.Sanitize(strings.TrimRight(text, "\r\n")) - text = strings.Map(func(c rune) rune { - if unicode.IsControl(c) { - return ' ' - } - return c - }, text) - if len(text) > maxStderr { - text = strings.ToValidUTF8(text[len(text)-maxStderr:], "") - } - return strings.TrimSpace(text) -} - -// Err is err with its message sanitized. errors.Is still answers for every -// error err wraps, and errors.As for a *StartError, whose own error is -// sanitized in turn; nothing else of the original chain is reachable, so no -// wrapped message can carry a secret past it. -func (r *Redactor) Err(err error) error { - if err == nil { - return nil - } - var already *redactedError - if errors.As(err, &already) && already.by == r { - return err - } - return &redactedError{msg: r.Sanitize(err.Error()), orig: err, by: r} -} - -type redactedError struct { - msg string - orig error - by *Redactor -} - -func (e *redactedError) Error() string { return e.msg } - -func (e *redactedError) Is(target error) bool { return errors.Is(e.orig, target) } - -func (e *redactedError) As(target any) bool { - switch t := target.(type) { - case **StartError: - var started *StartError - if !errors.As(e.orig, &started) { - return false - } - *t = &StartError{Process: started.Process, Err: e.by.Err(started.Err)} - return true - case **redactedError: - *t = e - return true - } - return false -} - -// Format keeps %+v and %#v from reaching the original error. -func (e *redactedError) Format(f fmt.State, _ rune) { _, _ = f.Write([]byte(e.msg)) } - -// Handler is h with every message and attribute sanitized. A string, an -// error or any value that is not a number, a boolean, a time or a duration -// is written as its sanitized text. -func (r *Redactor) Handler(h slog.Handler) slog.Handler { - return &redactingHandler{next: h, r: r} -} - -type redactingHandler struct { - next slog.Handler - r *Redactor -} - -func (h *redactingHandler) Enabled(ctx context.Context, level slog.Level) bool { - return h.next.Enabled(ctx, level) -} - -func (h *redactingHandler) Handle(ctx context.Context, rec slog.Record) error { - out := slog.NewRecord(rec.Time, rec.Level, h.r.Sanitize(rec.Message), rec.PC) - rec.Attrs(func(a slog.Attr) bool { - out.AddAttrs(h.attr(a)) - return true - }) - return h.next.Handle(ctx, out) -} - -func (h *redactingHandler) WithAttrs(attrs []slog.Attr) slog.Handler { - clean := make([]slog.Attr, len(attrs)) - for i, a := range attrs { - clean[i] = h.attr(a) - } - return &redactingHandler{next: h.next.WithAttrs(clean), r: h.r} -} - -func (h *redactingHandler) WithGroup(name string) slog.Handler { - return &redactingHandler{next: h.next.WithGroup(name), r: h.r} -} - -func (h *redactingHandler) attr(a slog.Attr) slog.Attr { - v := a.Value.Resolve() - switch v.Kind() { - case slog.KindInt64, slog.KindUint64, slog.KindFloat64, slog.KindBool, slog.KindTime, slog.KindDuration: - return slog.Attr{Key: a.Key, Value: v} - case slog.KindGroup: - group := v.Group() - clean := make([]slog.Attr, len(group)) - for i, g := range group { - clean[i] = h.attr(g) - } - return slog.Attr{Key: a.Key, Value: slog.GroupValue(clean...)} - case slog.KindString: - return slog.String(a.Key, h.r.Sanitize(v.String())) - default: - return slog.String(a.Key, h.r.Sanitize(fmt.Sprint(v.Any()))) - } -} diff --git a/internal/connector/driver/redact_test.go b/internal/connector/driver/redact_test.go deleted file mode 100644 index c49077b1b..000000000 --- a/internal/connector/driver/redact_test.go +++ /dev/null @@ -1,140 +0,0 @@ -package driver - -import ( - "bytes" - "errors" - "fmt" - "log/slog" - "os" - "path/filepath" - "strings" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestTheRedactionRuleTakesOutEverythingItNames(t *testing.T) { - state := t.TempDir() - r := NewRedactor(Redaction{ - Secrets: []string{"test-token-not-real"}, - Env: []string{"ANTHROPIC_API_KEY=test-key-not-real", "HOME=/home/operator", "TZ=UTC", "SHORT=abc"}, - Dirs: []string{state}, - }) - - assert.NotContains(t, r.Sanitize("token test-token-not-real used"), "test-token-not-real", "a named secret") - assert.NotContains(t, r.Sanitize("key test-key-not-real used"), "test-key-not-real", "a value of the worker's environment") - assert.Contains(t, r.Sanitize("under /home/operator/Work"), "/home/operator/Work", "BaseEnv's values are the operator's own, not the agent's") - assert.Contains(t, r.Sanitize("abc"), "abc", "a value too short to remove safely") - assert.NotContains(t, r.Sanitize("open "+filepath.Join(state, "ledger.db")+": denied"), state, "a path under the state directory") - assert.Contains(t, r.Sanitize("open "+filepath.Join(state, "ledger.db")+": denied"), ": denied", "and the rest of the message stands") - assert.NotContains(t, r.Sanitize("logged in as someone@example.com"), "someone@example.com") - assert.NotContains(t, r.Sanitize("with Bearer abc.def-ghi"), "abc.def-ghi") - assert.NotContains(t, r.Sanitize(strings.Repeat("x", 48)), strings.Repeat("x", 48)) - - // The pattern rules hold even for a caller with no redaction of its own. - assert.NotContains(t, (*Redactor)(nil).Sanitize("someone@example.com"), "someone@example.com") -} - -func TestTheRuleFollowsADirectoryThroughItsSymlink(t *testing.T) { - resolved := t.TempDir() - link := filepath.Join(t.TempDir(), "state") - require.NoError(t, os.Symlink(resolved, link)) - r := NewRedactor(Redaction{Dirs: []string{link}}) - assert.NotContains(t, r.Sanitize("open "+filepath.Join(resolved, "ledger.db")), resolved, "the resolved path is the same directory") - assert.NotContains(t, r.Sanitize("open "+filepath.Join(link, "ledger.db")), link) -} - -func TestStderrIsNeverPassedOnVerbatim(t *testing.T) { - r := NewRedactor(Redaction{Secrets: []string{"test-token-not-real"}}) - out := r.Stderr("starting\nusing test-token-not-real\x07 now\n") - assert.NotContains(t, out, "test-token-not-real") - assert.NotContains(t, out, "starting", "only the last line") - assert.NotContains(t, out, "\x07", "no control characters") - assert.LessOrEqual(t, len(r.Stderr(strings.Repeat("y", 4000))), maxStderr) -} - -func TestARedactedErrorAnswersIsAndAsWithoutCarryingTheSecret(t *testing.T) { - r := NewRedactor(Redaction{Secrets: []string{"test-token-not-real"}}) - inner := fmt.Errorf("%w: wrote test-token-not-real", ErrUnusable) - err := r.Err(&StartError{Process: Process{PID: 42, PGID: 42}, Err: errors.Join(ErrNotStarted, inner)}) - - assert.NotContains(t, err.Error(), "test-token-not-real") - assert.NotContains(t, fmt.Sprintf("%+v", err), "test-token-not-real", "and no verbose format reaches the original") - assert.ErrorIs(t, err, ErrNotStarted) - assert.ErrorIs(t, err, ErrUnusable) - assert.Equal(t, 42, StartedProcess(err).PID, "the process a failed start left is still readable") - - var started *StartError - require.True(t, errors.As(err, &started)) - assert.NotContains(t, started.Err.Error(), "test-token-not-real", "including the error it carries") - assert.Nil(t, r.Err(nil)) -} - -func TestEveryLogRecordPassesThroughTheRule(t *testing.T) { - var buf bytes.Buffer - r := NewRedactor(Redaction{Secrets: []string{"test-token-not-real"}}) - log := slog.New(r.Handler(slog.NewJSONHandler(&buf, nil))) - log = log.With("with", "test-token-not-real") - log.WithGroup("g").Error("wrote test-token-not-real", - "text", "test-token-not-real", - "error", errors.New("test-token-not-real"), - "any", []string{"test-token-not-real"}, - "count", 3) - - out := buf.String() - assert.NotContains(t, out, "test-token-not-real") - assert.Contains(t, out, `"count":3`, "numbers stay numbers") -} - -// Card 19: a refusal an agent writes to stderr is followed by whatever it -// prints next, and the tail is only the last line. Lines keeps them all, -// bounded and sanitized. -func TestStderrLinesKeepARefusalTheDiagnosticsBury(t *testing.T) { - r := NewRedactor(Redaction{Secrets: []string{"test-token-not-real"}}) - text := "refused: exec of /bin/rm (test-token-not-real)\nreading config\x07\n\nretrying in 2s\n" - lines := r.Lines(text) - require.Len(t, lines, 3, "the empty line is not one") - assert.Contains(t, lines[0], "refused: exec of /bin/rm", "the refusal is still there, first") - assert.NotContains(t, lines[0], "test-token-not-real", "and sanitized") - assert.Equal(t, "reading config", lines[1], "control characters are stripped") - assert.Equal(t, "retrying in 2s", lines[2]) - assert.Equal(t, "retrying in 2s", r.Stderr(text), "the tail is still the last line") - - many := make([]string, 0, maxStderrLines+20) - for i := range maxStderrLines + 20 { - many = append(many, fmt.Sprintf("line %d", i)) - } - bounded := r.Lines(strings.Join(many, "\n")) - assert.Len(t, bounded, maxStderrLines, "and the whole thing is bounded") - assert.Equal(t, "line 69", bounded[len(bounded)-1], "keeping the newest") - assert.LessOrEqual(t, len(r.Lines(strings.Repeat("z", 4000))[0]), maxStderr) -} - -// Copilot on #738: Sanitize is the last thing every driver error and log -// field passes through, and the connector's stderr is a terminal. A field a -// worker chose — Claude's reported permission mode, a tool name — must not be -// able to move the cursor, repaint the screen or start an escape sequence -// there. Tab and newline stay: they are what a legitimate multi-line log -// field is made of, and neither drives a terminal. -func TestTheRuleTakesTerminalControlsOutOfEverythingItSanitizes(t *testing.T) { - r := NewRedactor(Redaction{Secrets: []string{"test-token-not-real"}}) - - out := r.Sanitize("mode \x1b[31;1mdanger\x1b[0m\a set") - assert.NotContains(t, out, "\x1b", "an escape character never reaches a terminal") - assert.NotContains(t, out, "\a", "nor a bell") - assert.Contains(t, out, "danger", "and the text itself still reads") - - assert.NotContains(t, r.Sanitize("a\u009b2Kb"), "\u009b", "the C1 block is an escape sequence of its own") - assert.NotContains(t, r.Sanitize("a\x7fb"), "\x7f", "DEL too") - assert.NotContains(t, r.Sanitize("keep\roverwrite"), "\r", "a carriage return rewrites the line it is on") - assert.Equal(t, "one\ttwo\nthree", r.Sanitize("one\ttwo\nthree"), "tab and newline are a log field's own") - - // A control character in the middle of a secret must not hide it from - // the replacer, so the controls come out first. - assert.NotContains(t, r.Sanitize("test-token\x1b-not-real"), "not-real") - - // The pattern rules hold for a caller with no redaction of its own, and - // so does this one. - assert.NotContains(t, (*Redactor)(nil).Sanitize("\x1b]0;title\a"), "\x1b") -} diff --git a/internal/connector/driver/worker.go b/internal/connector/driver/worker.go deleted file mode 100644 index 164f92501..000000000 --- a/internal/connector/driver/worker.go +++ /dev/null @@ -1,668 +0,0 @@ -//go:build unix - -package driver - -import ( - "context" - "errors" - "fmt" - "io" - "os" - "os/exec" - "strings" - "sync" - "syscall" - "time" -) - -// pipeWaitDelay bounds how long a worker that has exited is waited on for -// pipes a stray descendant still holds. -const pipeWaitDelay = 2 * time.Second - -// # One owner, one release point -// -// This is the connector's rule for a task's process tree and its ledger -// record. Both belong to one owner — the attempt — and are released at one -// point, in this order: -// -// 1. Every worker starts as the leader of its own process group -// (StartWorker), so the tree it makes can be signaled as one. -// 2. A cancel, a deadline or a shutdown ends that group: SIGTERM, a bounded -// wait, then SIGKILL, by process group id and never by name (Terminate). -// 3. The group is then CONFIRMED gone (ConfirmGroupGone). Only after that -// may the attempt be settled and its record made terminal. -// 4. A group that cannot be confirmed gone — members left, a pid whose -// identity cannot be established, a platform that cannot say — leaves the -// record HELD: live in the ledger, its conversation and a worker slot -// still its own, for a person to settle. Never terminal, never released. -// 5. A restart reaps by the same rule (TerminateRecorded, then the same -// confirmation), and asks OwnsWorker first: a pid is not an identity, so -// ownership is the pid AND the kernel's own start time for it, compared -// exactly. Everything that acts on a recorded worker asks OwnsWorker -// rather than testing a pid of its own: in this card, recovery (through -// TerminateRecorded) and the release point's second confirmation; any -// later one — status, redispatch, discard, hold — the same way. Every -// group signal that follows the first asks again (signalRecordedGroup): -// ownership established before a grace period is not ownership after it, -// because a pid freed during the grace can be leading another group by -// the time the kill goes out. -// -// The one thing this cannot cover is a descendant that leaves the group by -// calling setsid: it is outside every group signal, and the connector can -// only avoid waiting on it (WaitDelay, CloseStdout). Containment is the -// sandbox launcher's job, not this rule's. -// -// Cards that start workers or settle records use the functions here rather -// than writing their own. -// -// # What a driver promises, and where each promise can still be broken -// -// The rule above is about the release point. These are the promises the rest -// of the boundary makes, each with the paths that can still break it named, -// so a reader does not have to take "held everywhere" on trust. -// -// ## A worker's lifetime -// -// - After a start returns a Session, a process group exists whose leader is -// the worker, and the connector owns it: Process() names it, and nobody -// else may signal it. -// - After a start returns an ERROR, no process of that session exists. -// Either none was started, or the driver ended the one it started, whole -// group, before returning (Driver.NewSession). ErrNotStarted says more: -// none ever existed, so the connector may retry the start once. -// - Cancel ends the turn, not the worker, and never blocks on a worker that -// has stopped reading its input: it gives up instead, and says so. -// - Close ends the session and its group — signal, bounded wait, kill — and -// is idempotent. It never waits on the worker's cooperation. -// - A worker that goes with a turn in flight is classified by how it went: -// one that exited on its own with a non-zero status FAILED, and one that -// vanished — signaled by someone else, or gone with no status the -// connector observed — is LOST. -// - Descriptors have an owner too. A start that fails closes every -// descriptor it opened; a terminated worker's output pipe is closed by -// the Worker once its reader has had the same bound to drain it that Wait -// gives a stray descendant, whether or not the reader closed it. -// - After a crash of the connector, the group survives. A later process -// identifies it by OwnsWorker (pid AND recorded start time), ends it with -// TerminateRecorded, and confirms with ConfirmGroupGone before anything -// is settled or released. -// -// Where this can still be broken: a descendant that calls setsid leaves the -// group and no signal reaches it (there is no portable way to see it, and -// containment is the sandbox launcher's); a driver that returns an error -// after leaving a process behind breaks the start promise, which is why it is -// written on the method rather than left to each driver; and on a platform -// where process start times cannot be read — or for a worker whose start -// time the kernel would not give — OwnsWorker refuses to answer and nothing -// may be settled; the run command refuses to start on such a platform at -// all. -// -// ## Credentials -// -// Two secrets exist around a worker, and each has one carriage. -// -// - The agent's Basecamp credential stays in the CLI's credential store. It -// is never in any environment, argv, file or log the connector writes; -// the worker's MCP server, running as the agent's profile, reads it from -// that store itself. -// - A task token lives from LaunchTask to the end of its task. The ledger -// keeps only its hash. It crosses only to the worker's MCP server, and -// never to the agent process: the dispatcher serves it over a unix socket -// in the attempt's owner-only runtime directory, once per start of that -// server (an MCP host that restarts a stdio server re-runs it, so the -// bridge asks again) and at most connector.MaxTokenHandoffs times, -// only to a peer of this user in the worker's process group or descended -// from its leader (connector.ServeTaskToken), and `basecamp connect -// worker-mcp` passes it on to `basecamp mcp` over an inherited -// descriptor. It is never in an environment, never in argv, never in a -// file, and never in a log or a dispatch line. -// - The agent's own credential (ANTHROPIC_API_KEY, where one is used) is in -// the agent's environment because the agent needs it, and nowhere else -// the connector writes. -// -// drivertest.RequireNoSecret and RequireNoSecretFilesDuring are the checks: -// the environment, argv, written text, and — watched continuously, so a file -// that lives milliseconds is still caught — every file under the working and -// session directories. What comes back OUT of a worker is the redaction -// rule's (redact.go), and drivertest.RequireRedacted is its check. -// -// Where this can still be broken: an agent may copy what it was handed -// anywhere its tools can write, and any process of this user in the worker's -// group could take the token first — the group is the agent's own tree. -// -// ## The environment a worker and its MCP servers get -// -// - The connector owns both. SessionConfig.Env is the worker's whole -// environment and MCPServer.Env is each server's, and each is an -// allowlist the dispatcher built by name (BuildEnv over BaseEnv, plus the -// variables a driver names for its own agent). -// - No credential is in either: the agent's Basecamp credential stays in -// the CLI's store, and the task token travels over the socket. -// - No secret is ever in argv, which every process on the machine can read. -// -// Where this can still be broken: an agent may ADD to the environment it -// hands its MCP servers — Claude Code passes its own whole environment down, -// which carries the agent's own credentials — so the declared environment is -// a floor, not a ceiling. The bridge (`basecamp connect worker-mcp`) execs -// `basecamp mcp` with the declared environment only, so the connector's own -// server does not keep them; a third-party MCP server the operator adds to a -// worker would inherit them regardless, and the connector ships none. -// -// ## When an attempt may be adopted, settled or released -// -// - Adoption links a reply to an event; it is never evidence that work -// finished, and never makes an outcome succeeded. It needs exactly one -// reply by the agent at that destination after the event's own -// acknowledgement and before any later instruction's, it is never the -// worker's own acknowledgement, and a listing the scan limit cut short -// adopts nothing. -// - An attempt is settled and its record made terminal at one point -// (Dispatcher.release), and only after the group is confirmed gone and -// the ledger has taken the settlement. -// - An attempt that cannot be confirmed or cannot be settled stays live and -// holds its conversation and one of the connector's worker slots, until a -// person settles it. -// -// Where this can still be broken: adoption trusts Basecamp's ordering of -// replies against this machine's clock for "after the acknowledgement", so a -// clock far behind the server's could see a reply as later than it was — the -// exactly-one rule and the acknowledgement exclusion are what keep that from -// mattering; and a person who writes to the ledger by hand can of course -// strand anything. -// -// Worker is a process a spawn driver started: the leader of its own process -// group, with its stdin and stdout piped and its stderr kept, redacted, for -// diagnosis. Every spawn driver starts its agent through StartWorker, so the -// rules for processes (invariants 1, 4 and 5) live in one place. -type Worker struct { - cmd *exec.Cmd - process Process - stdin io.WriteCloser - stdout *os.File - stderr *tailBuffer - - done chan struct{} - exit Exit - killOnce sync.Once - releaseOnce sync.Once -} - -// StartWorker launches cmd through cfg's launcher, in cfg's scope, as a new -// process group. An error wrapping ErrNotStarted means no process exists; -// StartWorker returns no other error. -// -// The whole of cfg is taken rather than its launcher and scope so that the -// announcement of the worker (cfg.Started, driver invariant 7) is made here, -// once, for every driver: the connector cannot arm the task token's socket -// until it knows the process group, and a driver that announced it only when -// its handshake returned would deadlock an agent whose handshake waits on -// its MCP servers. -func StartWorker(ctx context.Context, cfg SessionConfig, cmd Command) (*Worker, error) { - launcher := cfg.Launcher - if launcher == nil { - launcher = DirectLauncher{} - } - launched, err := launcher.Launch(ctx, LaunchRequest{Scope: cfg.Scope, Command: cmd}) - if err != nil { - return nil, fmt.Errorf("%w: launcher: %w", ErrNotStarted, err) - } - c := launched.Command - if c.Path == "" { - return nil, fmt.Errorf("%w: no command", ErrNotStarted) - } - if c.Env == nil { - // exec.Cmd reads a nil Env as "inherit the connector's". A worker - // never does (invariant 1); an empty environment is written as one. - c.Env = []string{} - } - // The worker outlives the call that starts it; Terminate ends it, never - // a context. - ec := exec.CommandContext(context.WithoutCancel(ctx), c.Path, c.Args...) //nolint:gosec // G204: the driver's own binary and flags, never content - ec.Dir = c.Dir - ec.Env = c.Env - ec.SysProcAttr = newProcessGroup() - // A descendant that left the group (a daemon that called setsid) can - // hold the worker's stdout or stderr open after the worker is gone. Wait - // would block on it, and with it Terminate and every shutdown behind - // it; past this delay the pipes are closed and the worker counts as - // exited. - ec.WaitDelay = pipeWaitDelay - w := &Worker{cmd: ec, stderr: &tailBuffer{max: 8 << 10}, done: make(chan struct{})} - ec.Stderr = w.stderr - if w.stdin, err = ec.StdinPipe(); err != nil { - return nil, fmt.Errorf("%w: %w", ErrNotStarted, err) - } - // Stdout is a pipe of the Worker's own, not exec's StdoutPipe: Wait - // closes an exec pipe when the process exits, which can drop the last - // lines a worker wrote before exiting while they are still being read. - // This one closes only when the reader has everything, or CloseStdout. - readEnd, writeEnd, err := os.Pipe() - if err != nil { - // Descriptors are owned too: a start that fails closes every one it - // opened. - _ = w.stdin.Close() - return nil, fmt.Errorf("%w: %w", ErrNotStarted, err) - } - ec.Stdout = writeEnd - w.stdout = readEnd - if err := ec.Start(); err != nil { - // exec.Cmd.Start returns an error only when no process was created: - // a missing binary, a bad directory, a failed fork. - _ = w.stdin.Close() - _ = readEnd.Close() - _ = writeEnd.Close() - return nil, fmt.Errorf("%w: %w", ErrNotStarted, err) - } - // The child has its copy; this process keeps none, so the reader sees - // end of file once the worker and everything it started have closed it. - _ = writeEnd.Close() - // The kernel's own start time for this pid, not the clock: it is what - // tells this worker from a later process the kernel gives the same pid, - // and OwnsWorker compares against it exactly. Where the kernel cannot be - // asked, the wall-clock stamp is kept for a person to read and the - // identity is marked inexact: no tolerance stands in for it, because a - // tolerance wide enough to cover a stamp taken around a fork is wide - // enough to accept a stranger under fast pid reuse (Copilot). Nothing is - // signaled on an inexact identity and nothing of its attempt is released. - started, exact := time.Now(), false - if kernel, err := processStartTime(ec.Process.Pid); err == nil { - started, exact = kernel, true - } - w.process = Process{PID: ec.Process.Pid, PGID: ec.Process.Pid, StartedAt: started, StartedExact: exact} - go func() { - err := ec.Wait() - w.exit = exitOf(ec, err) - close(w.done) - }() - // The worker exists, and nothing has been asked of it yet: this is the - // earliest the connector can be told its process group, and the latest it - // can be told without a handshake that waits on the token socket waiting - // on itself (driver invariant 7). - if cfg.Started != nil { - cfg.Started(w.process) - } - return w, nil -} - -func exitOf(cmd *exec.Cmd, err error) Exit { - state := cmd.ProcessState - if state == nil { - return Exit{Code: -1, Err: err} - } - if ws, ok := state.Sys().(syscall.WaitStatus); ok && ws.Signaled() { - return Exit{Code: -1, Signaled: true} - } - var exitErr *exec.ExitError - if err != nil && !errors.As(err, &exitErr) { - return Exit{Code: state.ExitCode(), Err: err} - } - return Exit{Code: state.ExitCode()} -} - -// Process is the worker's process. -func (w *Worker) Process() Process { return w.process } - -// Stdin is the worker's standard input. -func (w *Worker) Stdin() io.WriteCloser { return w.stdin } - -// Stdout is the worker's standard output. Read it to end of file. -func (w *Worker) Stdout() io.Reader { return w.stdout } - -// CloseStdout closes the worker's output: a reader blocked on it returns, and -// the descriptor is released. -// For a worker that is gone while a descendant that left its group still -// holds the pipe. -func (w *Worker) CloseStdout() { _ = w.stdout.Close() } - -// Done is closed once the process has exited and been reaped. -func (w *Worker) Done() <-chan struct{} { return w.done } - -// Exit is how it exited; meaningful once Done is closed. -func (w *Worker) Exit() Exit { - <-w.done - return w.exit -} - -// StderrTail is what may be passed on of the worker's stderr, through r -// (Redactor.Stderr): never the text verbatim. -func (w *Worker) StderrTail(r *Redactor) string { return r.Stderr(w.stderr.String()) } - -// StderrLines is what may be passed on of the worker's stderr when its last -// line is not enough — a refusal the agent wrote before it wrote anything -// else — through r (Redactor.Lines): bounded in lines and in bytes, each -// sanitized, never the text verbatim. -func (w *Worker) StderrLines(r *Redactor) []string { return r.Lines(w.stderr.String()) } - -// Terminate ends the process group: SIGTERM, grace, SIGKILL. It returns once -// the leader is reaped. Idempotent. -func (w *Worker) Terminate(grace time.Duration) { - w.killOnce.Do(func() { - _ = w.stdin.Close() - select { - case <-w.done: - // The leader is gone and reaped, so its pid — which is its - // group's id — may be the kernel's to give away: the group is - // signaled only while it is still provably this worker's. - _ = signalRecordedGroup(w.process, syscall.SIGKILL) - return - default: - } - _ = signalRecordedGroup(w.process, syscall.SIGTERM) - select { - case <-w.done: - case <-time.After(grace): - } - // The worker may have exited and been reaped during the grace, so - // ownership is established again rather than assumed from before it. - _ = signalRecordedGroup(w.process, syscall.SIGKILL) - // The leader by its own pid as well: were it not a group leader, the - // group signal would reach nothing and Terminate would wait forever. - _ = w.cmd.Process.Kill() - }) - <-w.done - // The output pipe is the Worker's to release as well. Its reader gets the - // same bound Wait gives a stray descendant to finish draining what the - // worker wrote before it went, and then the descriptor is closed whether - // or not the reader closed it. - w.releaseOnce.Do(func() { - time.AfterFunc(pipeWaitDelay, w.CloseStdout) - }) -} - -// ErrGroupOutlivedLeader is a recorded process group whose leader is gone — -// or is a pid the kernel has since reused — while the group still has -// members. They may be the worker's own children, so the caller must not -// treat the worker as finished. -var ErrGroupOutlivedLeader = errors.New("driver: the recorded process group outlived its leader") - -// OwnsWorker answers the one-owner rule's identity question: is the process -// this record names still the worker the task owns? -// -// A pid is not an identity — the kernel reuses them — so ownership is the pid -// AND the start time the owner recorded for it. Everything that acts on a -// recorded worker (recovery, status, redispatch, discard, hold) asks this -// before it acts, rather than writing its own pid check: -// -// - (true, nil): the process is still that worker. It may be signaled. -// - (false, nil): it is gone, and its group has no members left. Its record -// may be settled. -// - (false, ErrGroupOutlivedLeader): the leader is gone or is now some other -// process, and the recorded group still has members — they may be the -// worker's children. Nothing may be settled or released. -// - (false, err): the identity cannot be established here (an unreadable -// process table, a record with no kernel start time, a platform that -// cannot say). Nothing may be settled or released either. -func OwnsWorker(p Process) (bool, error) { - if p.PID <= 0 || p.PGID <= 0 { - // There is no process here to own. - return false, nil - } - gone, err := ProcessGone(p) - if err != nil { - return false, err - } - if gone { - // The leader is gone, or its pid is somebody else's now: what is left - // of the group decides whether anything of this worker remains. - return false, groupGone(p.PGID) - } - return true, nil -} - -// ErrIdentityUnknown is a record the connector cannot tell from a later -// process that reused its pid, because no kernel start time was ever -// recorded for it. It is not "gone" and it is not "still running": it is -// unanswerable, and the one-owner rule signals nothing and releases nothing -// on an unanswerable identity. -var ErrIdentityUnknown = errors.New("driver: the recorded process has no kernel start time, so it cannot be told from a later process that reused its pid") - -// ProcessGone reports whether the process a record names is gone: no process -// by that pid, a zombie, or a later process the kernel gave the same pid. It -// asks only about that process and says nothing about its group, which is -// what a caller wants to know about a worker's MCP server — the group is the -// agent's and outlives its servers. -// -// The comparison is exact. A kernel start time is read the same way every -// time it is read, in ticks since boot, so the process that was recorded -// answers with the value recorded for it and anything else is another -// process. A record whose start time the kernel never gave (StartedExact -// false) is ErrIdentityUnknown rather than a comparison against a tolerance: -// under fast pid reuse a window wide enough to cover a wall-clock stamp is -// wide enough to accept a stranger. -// -// It is the one place the question "is this still that process?" is answered; -// OwnsWorker asks it too, and adds the group. -func ProcessGone(p Process) (bool, error) { - if p.PID <= 0 { - return true, nil - } - started, err := processStartTime(p.PID) - if err != nil { - if errors.Is(err, os.ErrNotExist) { - // No process by that pid at all: nothing of it is left, whatever - // the record says about when it started. - return true, nil - } - return false, err - } - if !p.StartedExact { - return false, fmt.Errorf("%w: pid %d", ErrIdentityUnknown, p.PID) - } - if !started.Equal(p.StartedAt) { - return true, nil - } - return false, nil -} - -// LookupProcess is a live process's identity: its pid, the process group it -// leads or belongs to, and the start time that tells it from a later process -// the kernel gave the same pid. A process that is gone — or a zombie, which -// runs nothing — is os.ErrNotExist. -// -// It is how the connector takes the identity of a process it did not start -// but knows about, such as the MCP server that took a task token from the -// socket, which an agent may have started in a process group of its own. -func LookupProcess(pid int) (Process, error) { - if pid <= 0 { - return Process{}, os.ErrNotExist - } - started, err := processStartTime(pid) - if err != nil { - return Process{}, err - } - pgid, err := syscall.Getpgid(pid) - if err != nil { - return Process{}, err - } - return Process{PID: pid, PGID: pgid, StartedAt: started, StartedExact: true}, nil -} - -// signalRecordedGroup is the one place a recorded worker's process group is -// signaled, and it establishes that the group is still that worker's every -// time — not once, before a grace period, for every signal that follows it. -// A group signal is sent by the LEADER's pid, and a pid the kernel has taken -// back can lead a group of its own: the worker this connector started a -// minute later, say, which every signal held over from the last one would -// then end. -// -// It signals in two cases and neither is an assumption: -// -// - the recorded process is alive and is still that worker (OwnsWorker), or -// - the worker LED the group, no live process holds its pid any more, and -// the group still has members. Those members are the worker's own -// orphaned children: the kernel keeps a pid allocated for as long as a -// live process uses it as its process group id, so a group id cannot -// change hands while anything is still in the group. -// -// Everything else signals nothing. A pid that is alive and is NOT the -// recorded process is the case this exists for: the id has changed hands, -// and any group under it is a stranger's — the worker this connector started -// a minute later, say. An identity that cannot be established at all -// (ErrIdentityUnknown, an unreadable process table) is an error the caller -// holds on rather than a signal. And a record that names a process which -// only belonged to the group (a taker, whose pid is not the group's id) -// proves nothing about the group once that process is gone. -// -// Where this can still be broken: between the observation and the signal the -// last member can exit and the kernel can give the pid away. There is no -// portable way to signal a group as one atomic act — pidfd is per process, -// not per group — so that window is the syscall pair's, and it is the reason -// the connector confirms rather than assumes. -func signalRecordedGroup(p Process, sig syscall.Signal) error { - switch owns, err := OwnsWorker(p); { - case owns: - case err != nil && !errors.Is(err, ErrGroupOutlivedLeader): - return err - case p.PID != p.PGID || !pidUnheld(p.PID) || !GroupMembersRemain(p): - return nil - } - return signalGroup(p.PGID, sig) -} - -// pidUnheld reports whether no live process holds the pid: there is none, or -// what is left of one is a zombie, which runs nothing and keeps the id from -// being given away until its parent reaps it. -func pidUnheld(pid int) bool { - _, err := processStartTime(pid) - return errors.Is(err, os.ErrNotExist) -} - -// TerminateRecorded ends a worker a previous connector process started, by -// the process group it recorded, and only while OwnsWorker says that group is -// still this task's worker: a pid the kernel has since given to something -// else is left alone. It reports whether it signaled anything. -func TerminateRecorded(p Process, grace time.Duration) (bool, error) { - switch owns, err := OwnsWorker(p); { - case err != nil: - return false, err - case !owns: - return false, nil - } - if err := signalGroup(p.PGID, syscall.SIGTERM); err != nil { - if errors.Is(err, syscall.ESRCH) { - return false, nil - } - return false, err - } - deadline := time.Now().Add(grace) - for time.Now().Before(deadline) { - if groupGone(p.PGID) == nil { - return true, nil - } - time.Sleep(100 * time.Millisecond) - } - // The worker may have gone during the grace and its pid been given to a - // new group leader, so this signal asks again whose group it is. - _ = signalRecordedGroup(p, syscall.SIGKILL) - return true, nil -} - -// GroupMembersRemain reports whether the process group still has members. It -// signals nothing: it is the observation the one-owner rule's step 3 and 4 -// rest on, and what a caller asks when it must not disturb the group. -// -// A probe that cannot answer — the group exists but is not ours to signal — -// counts as members remaining, because the rule releases nothing it cannot -// prove gone. -func GroupMembersRemain(p Process) bool { - return p.PGID > 1 && groupGone(p.PGID) != nil -} - -// groupGone reports nil only when the kernel says there is no such process -// group, or when every member it still lists is a zombie. Anything else — -// a member that runs, a listing that could not be read, or a probe that was -// refused — is not absence, and the rule holds rather than releases. -// -// A zombie answers a zero-signal like a live process, and one stays a member -// until its parent waits for it. The connector's own worker is such a child -// between its exit and the Wait that reaps it, so a probe that counted -// zombies could hold a finished worker for as long as that Wait is late. -func groupGone(pgid int) error { - err := signalGroup(pgid, 0) - if err == nil { - running, listErr := groupRunning(pgid) - switch { - case listErr != nil: - return fmt.Errorf("%w: %d: %w", ErrGroupOutlivedLeader, pgid, listErr) - case !running: - return nil - } - } - return groupProbe(pgid, err) -} - -// groupProbe reads what a zero-signal to a process group said. Only ESRCH — -// "no such process group" — is proof of absence; a refusal (EPERM, from a -// group this process may not signal) is a group that is probably there and -// certainly not proven gone. -func groupProbe(pgid int, err error) error { - switch { - case err == nil: - return fmt.Errorf("%w: %d", ErrGroupOutlivedLeader, pgid) - case errors.Is(err, syscall.ESRCH): - return nil - default: - return fmt.Errorf("%w: %d: %w", ErrGroupOutlivedLeader, pgid, err) - } -} - -// ConfirmGroupGone is step 3 of the one-owner rule: it answers whether a -// worker's process group is gone, and it is what every caller asks before -// settling an attempt. -// -// It signals the group once more — a worker that ignored SIGTERM gets SIGKILL -// — then waits up to grace for the last member to go. A group with members -// left is ErrGroupOutlivedLeader, and the zero Process (a session the -// connector cannot signal at all) is gone as far as this rule goes, since -// there is nothing of it here to own. -func ConfirmGroupGone(p Process, grace time.Duration) error { - if p.PGID <= 0 { - return nil - } - if err := groupGone(p.PGID); err == nil { - return nil - } - if err := signalRecordedGroup(p, syscall.SIGKILL); err != nil { - // The group is not proven gone and whose it is cannot be - // established, so it is neither signaled nor confirmed: the attempt - // is held for a person. - return err - } - deadline := time.Now().Add(grace) - // The wait backs off: each probe of a group that still has members reads - // every process's state, and a stubborn worker must not cost a busy host - // a full process listing twenty times a second for the whole grace. - for wait := 50 * time.Millisecond; ; { - err := groupGone(p.PGID) - if err == nil || time.Now().After(deadline) { - return err - } - time.Sleep(wait) - if wait < 500*time.Millisecond { - wait *= 2 - } - } -} - -// tailBuffer keeps the last max bytes written to it. -type tailBuffer struct { - mu sync.Mutex - max int - buf []byte -} - -func (b *tailBuffer) Write(p []byte) (int, error) { - b.mu.Lock() - defer b.mu.Unlock() - b.buf = append(b.buf, p...) - if over := len(b.buf) - b.max; over > 0 { - b.buf = b.buf[over:] - } - return len(p), nil -} - -func (b *tailBuffer) String() string { - b.mu.Lock() - defer b.mu.Unlock() - return strings.ToValidUTF8(string(b.buf), "") -} diff --git a/internal/connector/driver/worker_other.go b/internal/connector/driver/worker_other.go deleted file mode 100644 index ccc31de88..000000000 --- a/internal/connector/driver/worker_other.go +++ /dev/null @@ -1,63 +0,0 @@ -//go:build !unix - -package driver - -import ( - "context" - "errors" - "io" - "os/exec" - "time" -) - -var errUnsupported = errors.New("driver: workers run on Unix only (process groups)") - -// Worker is unavailable off Unix. -type Worker struct{} - -// StartWorker refuses off Unix; nothing is started. -func StartWorker(context.Context, SessionConfig, Command) (*Worker, error) { - return nil, errors.Join(ErrNotStarted, errUnsupported) -} - -func (*Worker) Process() Process { return Process{} } -func (*Worker) Stdin() io.WriteCloser { return nil } -func (*Worker) Stdout() io.Reader { return nil } -func (*Worker) CloseStdout() {} -func (*Worker) Done() <-chan struct{} { return nil } -func (*Worker) Exit() Exit { return Exit{} } -func (*Worker) StderrTail(*Redactor) string { return "" } -func (*Worker) StderrLines(*Redactor) []string { return nil } -func (*Worker) Terminate(time.Duration) {} - -// OwnsWorker cannot answer off Unix, and an identity that cannot be -// established is never acted on. -func OwnsWorker(Process) (bool, error) { return false, errUnsupported } - -// GroupMembersRemain cannot answer off Unix, and what cannot be proven gone -// is held: it answers that members remain. -func GroupMembersRemain(Process) bool { return true } - -// ConfirmGroupGone cannot answer off Unix. -func ConfirmGroupGone(Process, time.Duration) error { return errUnsupported } - -// OwnProcessGroup cannot answer off Unix. -func OwnProcessGroup() (int, bool) { return 0, false } - -// ProcessGone cannot answer off Unix, and what cannot be answered is not -// proven gone. -func ProcessGone(Process) (bool, error) { return false, errUnsupported } - -// LookupProcess cannot answer off Unix. -func LookupProcess(int) (Process, error) { return Process{}, errUnsupported } - -// TerminateRecorded does nothing off Unix. -func TerminateRecorded(Process, time.Duration) (bool, error) { return false, errUnsupported } - -// probeGroup leaves the probe as it is off Unix, where the connector doesn't -// run workers. -type probeGroup struct{} - -func probeInItsOwnGroup(*exec.Cmd) *probeGroup { return &probeGroup{} } -func (*probeGroup) started() {} -func (*probeGroup) cleanup() {} diff --git a/internal/connector/driver/worker_unix.go b/internal/connector/driver/worker_unix.go deleted file mode 100644 index 0912ee7c4..000000000 --- a/internal/connector/driver/worker_unix.go +++ /dev/null @@ -1,120 +0,0 @@ -//go:build unix - -package driver - -import ( - "errors" - "os/exec" - "runtime" - "sync" - "syscall" - "time" -) - -// newProcessGroup makes the child the leader of a new process group, so the -// whole tree it starts is signaled as one. -func newProcessGroup() *syscall.SysProcAttr { - return &syscall.SysProcAttr{Setpgid: true} -} - -// OwnProcessGroup is the connector's own process group, which nothing of a -// worker's is ever in: every worker leads a group of its own. -func OwnProcessGroup() (int, bool) { return syscall.Getpgrp(), true } - -// signalGroup signals every process in the group. A non-positive pgid is -// refused — kill(0) and kill(-1) mean this group and every process — and so -// is the connector's own group: every worker leads a group of its own -// (Setpgid), so a recorded group that is this process's own is a mistake, and -// signaling it would end the connector and everything it is supervising. -func signalGroup(pgid int, sig syscall.Signal) error { - if pgid <= 1 || pgid == syscall.Getpgrp() { - return syscall.EINVAL - } - err := syscall.Kill(-pgid, sig) - // macOS only: Linux delivers a signal to a zombie-only group, so EPERM - // there is a group this process may not signal — and with /proc mounted - // hidepid, its listing cannot see that group's members to say otherwise. - if runtime.GOOS == "darwin" && errors.Is(err, syscall.EPERM) { - // macOS refuses any signal to a group whose only members are - // zombies, where Linux delivers it; nothing in such a group runs. - // When the kernel's own listing says so, it is the absent group it - // is everywhere else. A group that runs but may not be signaled - // (another user's) still lists its members, and keeps the refusal. - if running, listErr := groupRunning(pgid); listErr == nil && !running { - return syscall.ESRCH - } - } - return err -} - -// probeGroup is a probe run as the leader of a group of its own. -type probeGroup struct { - ec *exec.Cmd - - mu sync.Mutex - recorded bool // started has run: the probe may be waited for, and reaped - pid int - startedAt time.Time - exact bool // startedAt is the kernel's, for the pid while it was the probe -} - -// probeInItsOwnGroup runs a probe as the leader of a group of its own, and -// ends the whole group when its context does. -func probeInItsOwnGroup(ec *exec.Cmd) *probeGroup { - g := &probeGroup{ec: ec} - ec.SysProcAttr = newProcessGroup() - ec.Cancel = g.kill - return g -} - -// started records who the probe is while it can't have been reaped: RunProbe -// calls it after Start and before Wait. A launcher that has already exited is -// a zombie with no start time to read; its pid is recorded all the same. -func (g *probeGroup) started() { - g.mu.Lock() - defer g.mu.Unlock() - g.pid = g.ec.Process.Pid - if at, err := processStartTime(g.pid); err == nil { - g.startedAt, g.exact = at, true - } - g.recorded = true -} - -// cleanup ends the probe's group once the probe is over, however it ended: a -// launcher that starts the real worker as a child must not leave that child -// behind. -func (g *probeGroup) cleanup() { _ = g.kill() } - -// kill ends the probe's group, and only ever the probe's (Codex on #794). It -// runs from the context watcher, which can fire after Wait has reaped the -// probe, and from cleanup, which always does, so the pid may since have been -// given to someone else: -// - not yet recorded, the probe hasn't been waited for, so its pid is still -// its own; -// - a leader still running is killed only while its start time says it is -// the probe; -// - once the leader is gone, only its group's leftovers are: while a group -// has members the kernel won't give its id to another process (Linux and -// the BSDs keep a pid in use as a process group id), so a pid nobody holds -// with members left in its group is the probe's group, and a pid someone -// holds belongs to whoever reused it. -func (g *probeGroup) kill() error { - g.mu.Lock() - defer g.mu.Unlock() - if !g.recorded { - if g.ec.Process == nil { - return nil // it never started - } - return signalGroup(g.ec.Process.Pid, syscall.SIGKILL) - } - if g.exact { - gone, err := ProcessGone(Process{PID: g.pid, PGID: g.pid, StartedAt: g.startedAt, StartedExact: true}) - if err == nil && !gone { - return signalGroup(g.pid, syscall.SIGKILL) - } - } - if pidUnheld(g.pid) && GroupMembersRemain(Process{PID: g.pid, PGID: g.pid}) { - return signalGroup(g.pid, syscall.SIGKILL) - } - return nil -} diff --git a/internal/connector/handoff.go b/internal/connector/handoff.go index b6c80f3e3..5c261bfca 100644 --- a/internal/connector/handoff.go +++ b/internal/connector/handoff.go @@ -30,7 +30,7 @@ type HandoffLine struct { Trigger string `json:"trigger"` Recording HandoffRecording `json:"recording"` - ReplyTo InstructionReply `json:"reply_to"` + ReplyTo HandoffReply `json:"reply_to"` RequesterID int64 `json:"requester_id"` // RequesterName is known when the requester wrote the recording. RequesterName string `json:"requester_name,omitempty"` @@ -52,6 +52,12 @@ type HandoffRecording struct { URL string `json:"url"` } +// HandoffReply is where the session's acknowledgement and reply go. +type HandoffReply struct { + Kind string `json:"kind"` + RecordingID int64 `json:"recording_id"` +} + // Discard reasons the handoff writes. const ( // ReasonHandedOff: the request was written to stdout for the session. @@ -212,7 +218,7 @@ func handoffLine(record Record, agentID int64) (HandoffLine, error) { Title: richtext.SanitizeTerminal(snapshot.Title), URL: richtext.SanitizeTerminal(record.Decision.RecordingURL), }, - ReplyTo: InstructionReply{Kind: record.Decision.ReplyKind, RecordingID: record.Decision.ReplyRecordingID}, + ReplyTo: HandoffReply{Kind: record.Decision.ReplyKind, RecordingID: record.Decision.ReplyRecordingID}, RequesterID: record.Decision.RequesterID, RequesterName: richtext.SanitizeTerminal(snapshot.RequesterName), Acknowledge: record.Decision.Acknowledge, diff --git a/internal/connector/handoff_test.go b/internal/connector/handoff_test.go index c95113b24..07bf73868 100644 --- a/internal/connector/handoff_test.go +++ b/internal/connector/handoff_test.go @@ -15,6 +15,14 @@ import ( "github.com/basecamp/basecamp-cli/internal/connector/ndjson" ) +// admitOn writes an admitted record on a conversation key. +func admitOn(t *testing.T, ledger *Ledger, id int64, key string) { + t.Helper() + seenRecord(t, ledger, id) + _, err := ledger.Admission().Commit(context.Background(), admittedVerdict(id, 0, key)) + require.NoError(t, err) +} + func handoffOptions(ledger *Ledger, out *bytes.Buffer) HandoffOptions { return HandoffOptions{ Ledger: ledger, @@ -181,7 +189,7 @@ func TestTheLineCarriesNoTerminalControlSequences(t *testing.T) { func TestATaskLeftOpenByTheWorkerConnectorBlocksNothing(t *testing.T) { ledger := newTestLedger(t) admitOn(t, ledger, 1, "recording:1") - launch(t, ledger, 1) // the old dispatcher started a task, then crashed + olderLaunch(t, ledger, 1) // the old dispatcher started a task, then crashed admitOn(t, ledger, 2, "recording:1") var out bytes.Buffer diff --git a/internal/connector/invariants_test.go b/internal/connector/invariants_test.go index 3b77b8680..1cad47922 100644 --- a/internal/connector/invariants_test.go +++ b/internal/connector/invariants_test.go @@ -397,7 +397,7 @@ func reachTerminal(t *testing.T, ledger *Ledger, id int64, terminal RecordState) if terminal == StateDiscarded { return ledger.SetState(ctx, id, StateDiscarded, "untrusted_author") } - dispatchForTest(t, ledger, id) + olderDispatch(t, ledger, id) return ledger.SetState(ctx, id, StateCompleted, "") } @@ -474,16 +474,14 @@ func TestInvariantE4TheLifecycleCarriesTheEdgesLaterCardsCommit(t *testing.T) { assert.Equal(t, StateQueued, record.State) }) - // A dispatched record never handed to a worker returns to admitted when - // its task is superseded. + // A dispatched record never handed to a worker returned to admitted when + // an older build superseded its task. t.Run("dispatched back to admitted", func(t *testing.T) { ledger := newTestLedger(t) ctx := context.Background() _, err := ledger.RecordSeen(ctx, testEvent(1), LanePoll) require.NoError(t, err) - grant := dispatchForTest(t, ledger, 1) - - require.NoError(t, ledger.SupersedeTask(ctx, grant.ID)) + olderSupersede(t, ledger, olderDispatch(t, ledger, 1)) record, ok, err := ledger.Get(ctx, 1) require.NoError(t, err) diff --git a/internal/connector/ledger.go b/internal/connector/ledger.go index c06460ee3..344ae29c2 100644 --- a/internal/connector/ledger.go +++ b/internal/connector/ledger.go @@ -85,7 +85,6 @@ type Ledger struct { file *openLedgerFile closed sync.Once now func() time.Time - hooks Hooks // blockedScheduleReads counts the reads the retry sweep makes against the // schedule to prune its claims. The sweep must make none when it holds no // claims: the read it used to make was over the whole live backlog, and a @@ -96,44 +95,25 @@ type Ledger struct { // OpenLedger opens (creating if absent) the ledger at path and brings its // schema up to date. func OpenLedger(path string) (*Ledger, error) { - return openLedger(context.Background(), path, true) + return openLedger(context.Background(), path) } // ErrLedgerSchema is a ledger whose schema is not the one this binary writes. var ErrLedgerSchema = errors.New("the connector ledger's schema is not the version this basecamp writes") -// OpenExistingLedger opens a ledger the connector already created, for a -// process that reads and reports into it rather than owns it — a worker's -// MCP server. It never creates the file and never migrates: a different -// basecamp binary started as a worker must not change the schema under the -// connector that holds it, so a ledger at any other schema version is -// refused. -// -// Neither the privacy check nor SQLite may create the file on this path: the -// check only inspects, and the database is opened with mode=rw, so a ledger -// removed at any moment is an error rather than a new empty one. -func OpenExistingLedger(ctx context.Context, path string) (*Ledger, error) { - return openLedger(ctx, path, false) -} - -// ledgerDSN is the SQLite URI for the ledger at path. owner opens it the way -// the connector does, creating it when absent; otherwise mode=rw makes SQLite -// refuse a file that is not there. +// ledgerDSN is the SQLite URI for the ledger at path, opened the way the +// connector does, creating it when absent. // // _txlock=immediate takes the write lock when a transaction opens rather than // on its first write. Without it two connectors racing on one file can both // start, both read, and one is refused at COMMIT with the work already done. -func ledgerDSN(path string, owner bool) string { - dsn := "file:" + path + "?_pragma=busy_timeout(5000)&_pragma=journal_mode(WAL)&_pragma=foreign_keys(1)&_txlock=immediate" - if !owner { - dsn += "&mode=rw" - } - return dsn +func ledgerDSN(path string) string { + return "file:" + path + "?_pragma=busy_timeout(5000)&_pragma=journal_mode(WAL)&_pragma=foreign_keys(1)&_txlock=immediate" } -// openLedger opens the ledger; owner is the connector itself, which creates -// and migrates it. Any other opener does neither. -func openLedger(ctx context.Context, path string, owner bool) (*Ledger, error) { +// openLedger opens the ledger as the connector does: it creates the file when +// absent and migrates its schema. +func openLedger(ctx context.Context, path string) (*Ledger, error) { if path == "" { return nil, errors.New("connector: ledger path is required") } @@ -158,12 +138,12 @@ func openLedger(ctx context.Context, path string, owner bool) (*Ledger, error) { releaseLedger(file) return nil, fmt.Errorf("connector: %s and %s are one file: %w", abs, file.key, ErrLedgerUnderAnotherName) } - if err := checkLedgerFile(file, path, abs, owner); err != nil { + if err := checkLedgerFile(file, path, abs, true); err != nil { releaseLedger(file) return nil, err } - db, err := sql.Open("sqlite", ledgerDSN(path, owner)) + db, err := sql.Open("sqlite", ledgerDSN(path)) if err != nil { releaseLedger(file) return nil, fmt.Errorf("connector: open ledger: %w", err) @@ -173,26 +153,9 @@ func openLedger(ctx context.Context, path string, owner bool) (*Ledger, error) { db.SetMaxOpenConns(1) l := &Ledger{db: db, file: file, now: time.Now} - if owner { - if err := retryBusy(func() error { return l.migrate(ctx) }); err != nil { - _ = l.Close() - return nil, err - } - } else { - var version int - err := retryBusy(func() error { - var err error - version, err = l.schemaVersion(ctx) - return err - }) - if err != nil { - _ = l.Close() - return nil, fmt.Errorf("connector: read ledger schema: %w", err) - } - if version != len(migrations) { - _ = l.Close() - return nil, fmt.Errorf("connector: ledger at schema %d, this basecamp writes %d: %w", version, len(migrations), ErrLedgerSchema) - } + if err := retryBusy(func() error { return l.migrate(ctx) }); err != nil { + _ = l.Close() + return nil, err } // The WAL and shared-memory sidecars exist now and were created under the // process umask. The private directory already keeps other users out; @@ -908,7 +871,7 @@ END; // that is the whole reason it is not blocked_at. blocked_at is when the // record entered its current run of blocked states and survives every // blocked-to-blocked verdict, which is right for what reads it (a - // person's authorization, AuthorizedBlocked) and wrong for a window: the + // person's authorization, status's authorized_blocked) and wrong for a window: the // reasons carry different ones. A record that spent a week as the // unbounded config_unreadable and then blocks read_failed would be // measured against a week-old clock and get none of the twenty-four diff --git a/internal/connector/ledger_admission.go b/internal/connector/ledger_admission.go index ec828fab8..f2210fcd9 100644 --- a/internal/connector/ledger_admission.go +++ b/internal/connector/ledger_admission.go @@ -55,9 +55,9 @@ var undecided = []RecordState{StateSeen, StateBlocked} // run. Absent information is not consent: read the other way, a record with // no schedule would be eligible immediately and on every tick after it. // Nothing gives an untimed record an attempt today. A person's redispatch -// used to (Ledger.Redispatch stamps next_retry_at as it authorizes, through -// authorizeBlocked), but no command has called it since the worker side was -// removed in #815; such a record stays blocked until a person discards it. +// used to, stamping next_retry_at as it authorized the record, and went with +// the worker side in #815; such a record stays blocked until a person +// discards it. func (a Admission) LoadUndecided(ctx context.Context, id int64) (admission.Event, bool, error) { record, ok, err := a.ledger.Get(ctx, id) if err != nil || !ok { @@ -181,27 +181,13 @@ func (a Admission) commit(ctx context.Context, v admission.Verdict, state Record if state == StateAdmitted || state == StateQueued { // A record a hold tagged for review is written held by the database // instead (ledger_hold.go). The verdict reports what was written, so - // neither the hooks nor the stdout line call it admitted. + // the stdout line does not call it admitted. var written string if err := tx.QueryRowContext(ctx, `SELECT state FROM events WHERE id = ?`, v.EventID).Scan(&written); err != nil { return "", fmt.Errorf("connector: read verdict on %d back: %w", v.EventID, err) } state = RecordState(written) } - if l.hooks.VerdictCommitted != nil { - committed := CommittedVerdict{ - EventID: v.EventID, - State: state, - Reason: string(v.Reason), - Trigger: string(v.Trigger), - Acknowledge: v.Acknowledge, - ReplyKind: string(reply.Kind), - ReplyRecordingID: reply.RecordingID, - } - if err := l.hooks.VerdictCommitted(ctx, tx, committed); err != nil { - return "", fmt.Errorf("connector: verdict hook for %d: %w", v.EventID, err) - } - } if err := tx.Commit(); err != nil { return "", fmt.Errorf("connector: commit verdict on %d: %w", v.EventID, err) } diff --git a/internal/connector/ledger_admission_test.go b/internal/connector/ledger_admission_test.go index 84c60beec..a2b124ca7 100644 --- a/internal/connector/ledger_admission_test.go +++ b/internal/connector/ledger_admission_test.go @@ -132,10 +132,10 @@ func TestAdmissionSkipsARecordPastDeciding(t *testing.T) { require.NoError(t, ledger.SetState(ctx, 1, StateDiscarded, "untrusted_author")) case StateCompleted: require.NoError(t, reachTerminal(t, ledger, 7, StateCompleted)) - dispatchForTest(t, ledger, 1) + olderDispatch(t, ledger, 1) require.NoError(t, ledger.SetState(ctx, 1, StateCompleted, "")) case StateDispatched: - dispatchForTest(t, ledger, 1) + olderDispatch(t, ledger, 1) default: require.NoError(t, ledger.SetState(ctx, 1, state, "")) } @@ -252,7 +252,7 @@ func TestAdmissionNeverDecidesARecordPastDeciding(t *testing.T) { require.NoError(t, l.SetState(context.Background(), 1, StateAdmitted, "")) }, "dispatched": func(t *testing.T, l *Ledger) { - dispatchForTest(t, l, 1) + olderDispatch(t, l, 1) }, "discarded": func(t *testing.T, l *Ledger) { require.NoError(t, l.SetState(context.Background(), 1, StateDiscarded, "by_operator")) @@ -303,21 +303,21 @@ func TestAdmissionQueuesBehindALiveConversation(t *testing.T) { {"a dispatched record is a running task", func(t *testing.T, l *Ledger) { _, err := l.Admission().Commit(context.Background(), admittedVerdict(2, 0, key)) require.NoError(t, err) - dispatchForTest(t, l, 2) + olderDispatch(t, l, 2) }, admission.StateQueued}, {"a queued record alone is not a task", func(t *testing.T, l *Ledger) { _, err := l.Admission().Commit(context.Background(), admittedVerdict(2, 0, key)) require.NoError(t, err) _, err = l.Admission().Commit(context.Background(), admittedVerdict(3, 0, key)) require.NoError(t, err) - dispatchForTest(t, l, 2) + olderDispatch(t, l, 2) require.NoError(t, l.SetState(context.Background(), 2, StateCompleted, "")) require.Equal(t, StateQueued, getRecord(t, l, 3).State) }, admission.StateAdmitted}, {"a completed task is not live", func(t *testing.T, l *Ledger) { _, err := l.Admission().Commit(context.Background(), admittedVerdict(2, 0, key)) require.NoError(t, err) - dispatchForTest(t, l, 2) + olderDispatch(t, l, 2) require.NoError(t, l.SetState(context.Background(), 2, StateCompleted, "")) }, admission.StateAdmitted}, {"a blocked record is not a task", func(t *testing.T, l *Ledger) { @@ -467,12 +467,11 @@ func TestEveryMoveKeepsTheBlockedScheduleInputs(t *testing.T) { assert.Nil(t, d.BlockedAt, "a record that left blocked is not blocked") assert.Nil(t, d.RetryAt) - // Blocked again after a dispatch that was superseded: a new window from - // now, and the verdict that follows keeps it. Superseding a task returns - // an event no worker was handed to admitted, and it is blocked from - // there. - grant := dispatchForTest(t, ledger, 1) - require.NoError(t, ledger.SupersedeTask(ctx, grant.ID)) + // Blocked again after a dispatch an older build superseded: a new window + // from now, and the verdict that follows keeps it. Superseding a task + // returned an event no worker was handed to admitted, and it is blocked + // from there. + olderSupersede(t, ledger, olderDispatch(t, ledger, 1)) require.NoError(t, ledger.SetState(ctx, 1, StateBlocked, "read_failed")) record := getRecord(t, ledger, 1) require.NotNil(t, record.Decision.BlockedAt) @@ -515,7 +514,7 @@ func TestAMoveToBlockedOrDiscardedDropsTheSnapshot(t *testing.T) { seenRecord(t, ledger, 1) _, err := ledger.Admission().Commit(ctx, admittedVerdict(1, 0, "recording:9")) require.NoError(t, err) - dispatchForTest(t, ledger, 1) + olderDispatch(t, ledger, 1) require.NoError(t, ledger.SetState(ctx, 1, StateCompleted, "")) assert.NotEmpty(t, getRecord(t, ledger, 1).Decision.Snapshot) }) @@ -580,7 +579,7 @@ func TestDropContentTakesTheVerdictToo(t *testing.T) { seenRecord(t, ledger, 1) _, err := ledger.Admission().Commit(ctx, admittedVerdict(1, 0, "recording:9")) require.NoError(t, err) - dispatchForTest(t, ledger, 1) + olderDispatch(t, ledger, 1) require.NoError(t, ledger.SetState(ctx, 1, StateCompleted, "")) dropped, err := ledger.DropContent(ctx, at.Add(time.Hour), at.Add(time.Hour)) @@ -751,18 +750,14 @@ func mentionMarkup(id int64) string { } // A connect.json that cannot be read holds its records rather than discarding -// them or answering that the project is not served, and a person's -// redispatch runs them once the file is back. -// -// Waiting for a person is what every other blocked record does, and it is -// what this asserts — deliberately, and after taking an automatic sweep back -// out. A retry that offers due blocked records is a scheduler of its own, -// with its own claiming, and it turned out to re-decide five other blocked -// reasons that have nothing to do with this change. It is carded rather than -// carried here, so nothing in the code or the comments promises a timer that -// does not run. -func TestAnUnreadableConfigHoldsItsRecordsUntilTheFileAndAPersonAreBack(t *testing.T) { +// them or answering that the project is not served, and the record's timed +// retry runs it once the file is back. +func TestAnUnreadableConfigHoldsItsRecordsUntilTheFileIsBack(t *testing.T) { ledger := newTestLedger(t) + // The ledger's clock is read by admission's worker goroutine, so the + // test moves it through an atomic rather than by writing the field. + var ahead atomic.Int64 + ledger.now = func() time.Time { return time.Now().Add(time.Duration(ahead.Load())) } queue, err := NewQueue(10, 100) require.NoError(t, err) @@ -817,12 +812,9 @@ func TestAnUnreadableConfigHoldsItsRecordsUntilTheFileAndAPersonAreBack(t *testi require.Equal(t, string(admission.ReasonConfigUnreadable), held.Reason) assert.NotEqual(t, StateDiscarded, held.State, "a discard is the one outcome repairing the file could not undo") - // The operator repairs the file and redispatches, which is the remedy - // for a blocked record and the one the holding reply names. + // The operator repairs the file, and the record's retry comes due. broken.Store(false) - res, err := ledger.Redispatch(ctx, ev.ID, "local:tester", []int64{adapterBucketID}) - require.NoError(t, err) - require.True(t, res.Rerun, "a blocked record's prerequisite is run again") + ahead.Store(int64(admission.BlockedRetryInterval)) require.NoError(t, queue.Offer(ctx, ev.ID)) require.Eventually(t, func() bool { diff --git a/internal/connector/ledger_dangerous_test.go b/internal/connector/ledger_dangerous_test.go deleted file mode 100644 index 63bf3e875..000000000 --- a/internal/connector/ledger_dangerous_test.go +++ /dev/null @@ -1,62 +0,0 @@ -package connector - -import ( - "context" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -// Only unfinished requests count, and a request is the person admission -// trusted at the gate's: the performer, else the creator. A record only -// seen is judged again under the trust the connector runs with now. -func TestUnfinishedFromOthersCountsWhatOthersAskedForAndIsStillWaiting(t *testing.T) { - ledger := newTestLedger(t) - ctx := context.Background() - const operator, other = int64(111), int64(222) - - addWhy := func(id, creator int64, performer *int64, state RecordState, reason string) { - t.Helper() - ev := testEvent(id) - ev.CreatorID = creator - ev.PerformedByID = performer - _, err := ledger.RecordSeen(ctx, ev, LaneLive) - require.NoError(t, err) - _, err = ledger.db.ExecContext(ctx, `UPDATE events SET state = ?, reason = ? WHERE id = ?`, string(state), reason, id) - require.NoError(t, err) - } - add := func(id, creator int64, performer *int64, state RecordState) { - t.Helper() - addWhy(id, creator, performer, state, "") - } - count := func() int { - t.Helper() - n, err := ledger.UnfinishedFromOthers(ctx, operator) - require.NoError(t, err) - return n - } - - add(1, operator, nil, StateQueued) - add(2, other, nil, StateCompleted) - add(3, other, nil, StateDiscarded) - add(4, other, nil, StateSeen) - op := operator - add(5, other, &op, StateQueued) // the operator assigned someone else's to-do - assert.Equal(t, 0, count()) - - add(6, other, nil, StateQueued) - assert.Equal(t, 1, count()) - oth := other - add(7, operator, &oth, StateHeld) // someone else acted on the operator's recording - assert.Equal(t, 2, count()) - - // Blocked in a project the agent doesn't serve: judged again under - // today's trust, so it never reaches a worker and doesn't count. - addWhy(8, other, nil, StateBlocked, "no_route") - assert.Equal(t, 2, count()) - // Blocked after its worker failed to start: a redispatch runs it as it - // was admitted, so it counts. - addWhy(9, other, nil, StateBlocked, ReasonSpawnFailed) - assert.Equal(t, 3, count()) -} diff --git a/internal/connector/ledger_decisions.go b/internal/connector/ledger_decisions.go index edfb8946a..5425cdceb 100644 --- a/internal/connector/ledger_decisions.go +++ b/internal/connector/ledger_decisions.go @@ -5,52 +5,34 @@ import ( "database/sql" "errors" "fmt" - "slices" "strings" - "time" - - "github.com/basecamp/basecamp-cli/internal/connector/driver" ) // ErrDecisionRefused is a decision the record's state does not accept: a -// discard, or a redispatch through Ledger.Redispatch, which no command calls -// since #815. The message says why. +// discard, or an import's entry. The message says why. var ErrDecisionRefused = errors.New("refused") -// eventTask is the latest task an event was on, as a decision reads it. +// eventTask is the latest task an event was on, as a decision reads it: a +// record a worker ran under an older build settled its outcome there. type eventTask struct { - found bool - taskID int64 - delivery Delivery - outcome Outcome - superseded bool - ended bool + found bool + outcome Outcome // completedAt is when the event's outcome settled, as stored. completedAt string - // live is the task's attempt that has not ended, if any, with its - // recorded process. - liveAttempt string - process AttemptProcess } func loadEventTask(ctx context.Context, tx *sql.Tx, eventID int64) (eventTask, error) { var ( - et eventTask - delivery, outcome string - superseded, ended sql.NullString - completed sql.NullString - attempt, startedText sql.NullString - pid, pgid sql.NullInt64 + et eventTask + outcome string + completed sql.NullString ) err := tx.QueryRowContext(ctx, ` -SELECT te.task_id, te.delivery, te.outcome, te.completed_at, t.superseded_at, t.ended_at, - a.id, a.pid, a.pgid, a.process_started +SELECT te.outcome, te.completed_at FROM task_events te JOIN tasks t ON t.id = te.task_id -LEFT JOIN attempts a ON a.task_id = t.id AND a.state <> 'ended' WHERE te.event_id = ? AND te.withdrawn_at IS NULL -ORDER BY te.task_id DESC LIMIT 1`, eventID).Scan(&et.taskID, &delivery, &outcome, &completed, &superseded, &ended, - &attempt, &pid, &pgid, &startedText) +ORDER BY te.task_id DESC LIMIT 1`, eventID).Scan(&outcome, &completed) switch { case errors.Is(err, sql.ErrNoRows): return eventTask{}, nil @@ -58,310 +40,10 @@ ORDER BY te.task_id DESC LIMIT 1`, eventID).Scan(&et.taskID, &delivery, &outcome return eventTask{}, fmt.Errorf("connector: read the task of event %d: %w", eventID, err) } et.found = true - et.delivery, et.outcome = Delivery(delivery), Outcome(outcome) - et.superseded, et.ended, et.completedAt = superseded.Valid, ended.Valid, completed.String - if attempt.Valid { - et.liveAttempt = attempt.String - et.process = AttemptProcess{PID: int(pid.Int64), PGID: int(pgid.Int64)} - if startedText.Valid { - if et.process.StartedAt, err = parseStamp(startedText.String); err != nil { - return eventTask{}, err - } - } - } + et.outcome, et.completedAt = Outcome(outcome), completed.String return et, nil } -// operatorRecord is a record with the columns a decision reads. -type operatorRecord struct { - Record - review bool - authorizedAt sql.NullString - // redispatchDecision is the redispatch waiting for the record's task to - // end; zero when none is. Only a ledger written before #815 removed the - // worker side, or Ledger.Redispatch, which no command calls now, holds one. - redispatchDecision int64 -} - -func loadOperatorRecord(ctx context.Context, tx *sql.Tx, eventID int64) (operatorRecord, error) { - record, err := loadRecord(ctx, tx, eventID) - if err != nil { - return operatorRecord{}, err - } - out := operatorRecord{Record: record} - var decision sql.NullInt64 - if err := tx.QueryRowContext(ctx, `SELECT review, authorized_at, redispatch_decision FROM events WHERE id = ?`, eventID). - Scan(&out.review, &out.authorizedAt, &decision); err != nil { - return operatorRecord{}, fmt.Errorf("connector: read event %d: %w", eventID, err) - } - out.redispatchDecision = decision.Int64 - return out, nil -} - -// RedispatchResult is what a redispatch did. -type RedispatchResult struct { - EventID int64 `json:"event_id"` - FromState RecordState `json:"from_state"` - FromReason string `json:"from_reason,omitempty"` - FromOutcome Outcome `json:"from_outcome,omitempty"` - // State is the record's state after the authorization. - State RecordState `json:"state"` - // Admitted says the record waits for a worker now. - Admitted bool `json:"admitted"` - // Pending says the record's task is still live: it is admitted in the - // transaction that ends that task. - Pending bool `json:"pending,omitempty"` - // Rerun says the record was authorized as blocked: the caller runs its - // prerequisite again (admission), which admits it when it succeeds. - Rerun bool `json:"rerun,omitempty"` - // SupersededTaskID is the task whose token this redispatch retired; zero - // when it was already retired. - SupersededTaskID int64 `json:"superseded_task_id,omitempty"` - // Worker is the replaced attempt's recorded process, still live in the - // ledger: the caller terminates it (driver.TerminateRecorded). - Worker *LiveWorker `json:"worker,omitempty"` - // Held says the hold marker stands: authorized, and nothing launches - // until release. - Held bool `json:"held,omitempty"` -} - -// LiveWorker is an attempt's recorded worker process. -type LiveWorker struct { - AttemptID string `json:"attempt_id"` - TaskID int64 `json:"task_id"` - // PID, PGID and StartedAt are the recorded process: with the start time, - // the pid is an identity (driver.OwnsWorker). - PID int `json:"pid"` - PGID int `json:"pgid"` - StartedAt time.Time `json:"started_at"` -} - -// Identity is the process the record names, for the one-owner rule. -func (w LiveWorker) Identity() driver.Process { - return recordedIdentity(w.PID, w.PGID, w.StartedAt) -} - -// Redispatch authorizes a record to run again, or for the first time, and -// records who authorized it (invariants 4 to 6). -// -// No command calls it: `connect redispatch` went with the worker side in -// #815. It stays with the schema it writes, which the ledger keeps as it is. -// -// - completed with outcome unknown or failed: the task's token is -// superseded; admitted at once when the task has ended, otherwise when it -// ends. Refused without a snapshot, or in a project connect.json does -// not serve. -// - held with its snapshot, in a served project and with no blocking -// reason: admitted. -// - blocked, or held over a blocking reason: authorized as blocked, and -// Rerun asks the caller to run what blocked it. -// - succeeded, discarded, and anything live (seen, admitted, queued, -// dispatched) are refused with ErrDecisionRefused. -func (l *Ledger) Redispatch(ctx context.Context, eventID int64, by string, served []int64) (RedispatchResult, error) { - if strings.TrimSpace(by) == "" { - return RedispatchResult{}, errors.New("connector: a redispatch records who authorized it") - } - var out RedispatchResult - err := retryBusy(func() error { - var err error - out, err = l.redispatch(ctx, eventID, by, served) - return err - }) - return out, err -} - -func (l *Ledger) redispatch(ctx context.Context, eventID int64, by string, served []int64) (RedispatchResult, error) { - tx, err := l.db.BeginTx(ctx, nil) - if err != nil { - return RedispatchResult{}, fmt.Errorf("connector: begin redispatch: %w", err) - } - defer func() { _ = tx.Rollback() }() - - record, err := loadOperatorRecord(ctx, tx, eventID) - if err != nil { - return RedispatchResult{}, err - } - task, err := loadEventTask(ctx, tx, eventID) - if err != nil { - return RedispatchResult{}, err - } - out := RedispatchResult{EventID: eventID, FromState: record.State, FromReason: record.Reason, FromOutcome: task.outcome} - refuse := func(why string) error { - return fmt.Errorf("connector: redispatch of event %d %s: %w", eventID, why, ErrDecisionRefused) - } - // Decision.Served is what admission wrote when it decided the record, so - // it says the project was served then; served is what the caller read - // from connect.json for this command. Both, because a redispatch that - // reported success and left the record for a dispatcher that will refuse - // to launch it is worse than one that refuses here (Copilot on #765). - // - // served must be a reading its caller took under connect.json's own lock - // and holds until this commits, so an unserve lands wholly before that - // read or wholly after this write; the removed `connect redispatch` did - // exactly that. This checks the set it was given; the caller is what - // makes the set current. - dispatchable := !record.ContentDropped && len(record.Decision.Snapshot) > 0 && - record.Decision.Served && slices.Contains(served, record.BucketID) && - record.Decision.ConversationKey != "" - at := l.now() - now := stamp(at) - authorize := []assignment{{column: "authorized_at", value: now}, {column: "authorized_by", value: by}} - recorded := false - - switch record.State { - case StateSeen, StateAdmitted, StateQueued, StateDispatched: - return RedispatchResult{}, refuse(fmt.Sprintf("is %s: it is live, and runs without one", record.State)) - case StateDiscarded: - return RedispatchResult{}, refuse(fmt.Sprintf("is discarded (%s)", record.Reason)) - - case StateCompleted: - switch { - case !task.found || task.delivery != DeliveryCompleted: - return RedispatchResult{}, refuse("has no settled outcome to redispatch") - case task.outcome == OutcomeSucceeded: - return RedispatchResult{}, refuse("succeeded; a success is not run again") - case task.outcome != OutcomeUnknown && task.outcome != OutcomeFailed: - return RedispatchResult{}, refuse(fmt.Sprintf("has outcome %q", task.outcome)) - case record.redispatchDecision != 0: - return RedispatchResult{}, refuse("already has a redispatch waiting for its task to end") - case !dispatchable: - return RedispatchResult{}, refuse("is missing something a dispatch needs: its content snapshot (retention dropped it, or the verdict carried none), or a project connect.json serves") - } - if !task.superseded { - // The replaced worker is refused by basecamp_connect from here on - // (invariant 5). Through supersedeTask and not a bare write to - // superseded_at: the supersession retires the task's rows in the - // same statement (tasks_supersession_retires_its_events), so an - // event this task never exposed has to be returned to admitted - // here or it is never returned at all — the settlement at the - // attempt's end reads only rows that are still live, and finds - // none. A returned event is not startable while the task it left - // has not ended, so nothing runs beside the worker being stopped. - if err := l.supersedeTask(ctx, tx, task.taskID); err != nil { - return RedispatchResult{}, err - } - out.SupersededTaskID = task.taskID - } - if task.liveAttempt != "" { - out.Worker = &LiveWorker{AttemptID: task.liveAttempt, TaskID: task.taskID, - PID: task.process.PID, PGID: task.process.PGID, StartedAt: task.process.StartedAt} - } - to := StateCompleted - if task.ended { - to = StateAdmitted - } - // The decision is the authorization the database checks: the record - // names it, and only a decision made after the outcome settled lets a - // completed record move (invariant 4). - decisionID, err := insertDecision(ctx, tx, decision{action: "redispatch", eventID: eventID, by: by, at: notBefore(now, task.completedAt), - fromState: record.State, fromReason: record.Reason, fromOutcome: task.outcome, toState: to, - supersededTask: out.SupersededTaskID, note: pendingNote(task)}) - if err != nil { - return RedispatchResult{}, err - } - recorded = true - if _, err := tx.ExecContext(ctx, `UPDATE events SET authorized_at = ?, authorized_by = ?, redispatch_decision = ? WHERE id = ?`, now, by, decisionID, eventID); err != nil { - return RedispatchResult{}, fmt.Errorf("connector: authorize event %d: %w", eventID, err) - } - if task.ended { - moved, err := l.move(ctx, tx, transition{id: eventID, state: StateAdmitted, from: []RecordState{StateCompleted}, byOperator: true, - set: []assignment{{column: "redispatch_decision", value: nil}}}) - if err != nil { - return RedispatchResult{}, err - } - if !moved { - return RedispatchResult{}, fmt.Errorf("connector: admit event %d: %w", eventID, ErrNotATransition) - } - out.Admitted = true - } else { - out.Pending = true - } - - case StateHeld: - if record.Reason == "" && dispatchable { - // Queued behind a live conversation, as admission would write it. - target := StateAdmitted - var live bool - if err := tx.QueryRowContext(ctx, `SELECT EXISTS (SELECT 1 FROM events WHERE conversation_key = ? AND id <> ? AND state IN ('admitted', 'dispatched'))`, - record.Decision.ConversationKey, eventID).Scan(&live); err != nil { - return RedispatchResult{}, fmt.Errorf("connector: read conversation of %d: %w", eventID, err) - } - if live { - target = StateQueued - } - moved, err := l.move(ctx, tx, transition{id: eventID, state: target, from: []RecordState{StateHeld}, byOperator: true, set: authorize}) - if err != nil { - return RedispatchResult{}, err - } - if !moved { - return RedispatchResult{}, fmt.Errorf("connector: admit event %d: %w", eventID, ErrNotATransition) - } - out.Admitted = target == StateAdmitted - break - } - // A held record carries no reason today (events_review_is_held clears - // it), but the spec's "held over a blocking reason" is a record a - // migration may yet write, and it re-runs what blocked it. - reason := record.Reason - if reason == "" { - reason = "held_incomplete" - } - moved, err := l.move(ctx, tx, transition{id: eventID, state: StateBlocked, reason: reason, from: []RecordState{StateHeld}, byOperator: true}) - if err != nil { - return RedispatchResult{}, err - } - if !moved { - return RedispatchResult{}, fmt.Errorf("connector: authorize event %d: %w", eventID, ErrNotATransition) - } - if err := authorizeBlocked(ctx, tx, eventID, now, by); err != nil { - return RedispatchResult{}, err - } - out.Rerun = true - - case StateBlocked: - // The record keeps its state; what blocked it runs again. Writing - // the authorization is not a state change and leaves the revision - // the re-run loads at. - if err := authorizeBlocked(ctx, tx, eventID, now, by); err != nil { - return RedispatchResult{}, err - } - out.Rerun = true - - default: - return RedispatchResult{}, refuse(fmt.Sprintf("is in a state %q this build does not know", record.State)) - } - - if err := tx.QueryRowContext(ctx, `SELECT state FROM events WHERE id = ?`, eventID).Scan(&out.State); err != nil { - return RedispatchResult{}, fmt.Errorf("connector: read event %d back: %w", eventID, err) - } - if _, out.Held, err = readHold(ctx, tx); err != nil { - return RedispatchResult{}, err - } - if !recorded { - note := "" - if out.Rerun { - note = "prerequisite runs again" - } - if err := recordDecision(ctx, tx, decision{action: "redispatch", eventID: eventID, by: by, at: now, - fromState: record.State, fromReason: record.Reason, fromOutcome: task.outcome, toState: out.State, - supersededTask: out.SupersededTaskID, note: note}); err != nil { - return RedispatchResult{}, err - } - } - // A notice already posted that asked for this redispatch is answered by - // one, in this transaction, as every other lifecycle message is written - // in the transaction of the transition that calls for it. - if l.hooks.RecordDecided != nil { - if err := l.hooks.RecordDecided(ctx, tx, RecordDecision{EventID: eventID, Action: DecisionRedispatch, At: at}); err != nil { - return RedispatchResult{}, err - } - } - if err := tx.Commit(); err != nil { - return RedispatchResult{}, fmt.Errorf("connector: commit redispatch of %d: %w", eventID, err) - } - return out, nil -} - // DiscardResult is what a discard did. type DiscardResult struct { EventID int64 `json:"event_id"` @@ -399,7 +81,7 @@ func (l *Ledger) discard(ctx context.Context, eventID int64, by string) (Discard } defer func() { _ = tx.Rollback() }() - record, err := loadOperatorRecord(ctx, tx, eventID) + record, err := loadRecord(ctx, tx, eventID) if err != nil { return DiscardResult{}, err } @@ -457,45 +139,12 @@ WHERE event_id = ? AND state = 'pending' AND kind IN ('guard_ack', 'holding_repl return DiscardResult{}, err } out.Canceled = int(canceled) - // What it already said, and that asked for this decision, is retracted: - // the ask is answered, so it stops standing on the card as an open one. - if l.hooks.RecordDecided != nil { - if err := l.hooks.RecordDecided(ctx, tx, RecordDecision{EventID: eventID, Action: DecisionDiscard, At: at}); err != nil { - return DiscardResult{}, err - } - } if err := tx.Commit(); err != nil { return DiscardResult{}, fmt.Errorf("connector: commit discard of %d: %w", eventID, err) } return out, nil } -// AuthorizedBlocked lists blocked records a person authorized, oldest first: -// authorized since the record entered its current run of blocked states, so -// an authorization that answered an earlier outcome does not count. -// It was for running a redispatched record's prerequisite again when the -// redispatch's own run did not settle it. Nothing calls it. The timed -// retry that now runs (Intake.sweepBlockedRetries) is not it: that schedule -// is keyed on the reason, not on who authorized the record, and the reasons -// it leaves alone — no_route, unroutable, bucket_mismatch — are the ones a -// re-run on a timer could only repeat the same answer for. -func (l *Ledger) AuthorizedBlocked(ctx context.Context, limit int) ([]int64, error) { - rows, err := l.db.QueryContext(ctx, `SELECT id FROM events WHERE state = 'blocked' AND authorized_at >= blocked_at ORDER BY id LIMIT ?`, limit) - if err != nil { - return nil, fmt.Errorf("connector: authorized blocked records: %w", err) - } - defer func() { _ = rows.Close() }() - var ids []int64 - for rows.Next() { - var id int64 - if err := rows.Scan(&id); err != nil { - return nil, err - } - ids = append(ids, id) - } - return ids, rows.Err() -} - // notBefore is now, or the stored time an outcome settled when that is later: // a decision is never recorded as made before the outcome it decides on, even // with a clock that stepped back. @@ -505,34 +154,3 @@ func notBefore(now, settled string) string { } return now } - -func pendingNote(task eventTask) string { - if task.ended { - return "" - } - return fmt.Sprintf("waits for task %d to end", task.taskID) -} - -// authorizeBlocked records a person's authorization on a blocked record, never -// dated before the record entered its current run of blocked states: an -// authorization counts for that block only when it is not older than it -// (AuthorizedBlocked), and neither a move's own later stamp nor a clock that -// stepped back may make a fresh one look stale. -// It also makes the record due now, whatever it was blocked on. A person -// asking for a rerun is not a timer and does not wait for one: the removed -// redispatch command ran admission itself straight after this, and a blocked -// record with no attempt owed it is not one admission will load -// (LoadUndecided): every untimed reason, no_route above all, the one a -// person is most likely to redispatch. Writing "due now" rather than -// leaving the schedule as it was also keeps a rerun that never happened on -// the sweep's list instead of stranding it. Its only caller is -// Ledger.Redispatch, which no command calls since #815. -func authorizeBlocked(ctx context.Context, tx *sql.Tx, eventID int64, now, by string) error { - if _, err := tx.ExecContext(ctx, ` -UPDATE events SET authorized_at = MAX(?, COALESCE(blocked_at, '')), authorized_by = ?, - next_retry_at = ? -WHERE id = ? AND state = 'blocked'`, now, by, now, eventID); err != nil { - return fmt.Errorf("connector: authorize event %d: %w", eventID, err) - } - return nil -} diff --git a/internal/connector/ledger_dispatch.go b/internal/connector/ledger_dispatch.go index 18addbf81..bf06666aa 100644 --- a/internal/connector/ledger_dispatch.go +++ b/internal/connector/ledger_dispatch.go @@ -2,25 +2,14 @@ package connector import ( "context" - "crypto/rand" - "crypto/sha256" "database/sql" - "encoding/base64" - "encoding/hex" - "encoding/json" - "errors" "fmt" "html" - "net/url" "os" "path/filepath" "slices" "strconv" "strings" - "time" - - "modernc.org/sqlite" - sqlite3 "modernc.org/sqlite/lib" "github.com/basecamp/basecamp-sdk/go/pkg/basecamp" ) @@ -28,9 +17,13 @@ import ( // The dispatch lifecycle, as one state machine. // // Three things have state here, and a fourth is the guard on one of them. -// This comment is the contract; lifecycle (ledger_events.go), move, the -// schema's triggers and index, and the functions below enforce exactly it, -// and TestDispatchLifecycleTable tries every transition against it. +// Since #815 removed the worker side, this build writes no task, no task event +// and no delivery: the dispatcher and the worker's basecamp_connect calls that +// did are gone. Ledgers an older build wrote still hold them, and the schema's +// triggers and index that held those writes to this contract are kept as they +// shipped, so this is still the shape of what such a ledger can contain. +// lifecycle (ledger_events.go) and move enforce the record table for every +// write this build still makes. // // # Record (events.state) // @@ -38,141 +31,104 @@ import ( // seen admitted admission verdict, conversation not live // seen queued admission verdict, conversation live // seen blocked admission verdict -// seen discarded admission verdict +// seen discarded admission, operator verdict, or import // blocked admitted admission re-decided // blocked queued admission re-decided, conversation live // blocked blocked admission re-decided, still blocked // blocked discarded admission, operator verdict, or discard -// blocked dispatched — an edge the lifecycle map has and no -// caller can take: a task needs an -// instruction, and blocked has none -// admitted dispatched dispatcher (CreateTask) joins a task -// queued dispatched dispatcher (CreateTask) joins a task -// dispatched dispatched dispatcher (CreateTask) redispatch onto a new task -// dispatched admitted dispatcher (SupersedeTask) never handed to a worker, task retired -// dispatched admitted dispatcher (withdrawExposure) exposed at launch, spawn proven failed: retry -// dispatched blocked dispatcher (withdrawExposure) exposed at launch, spawn failed again -// dispatched completed worker (complete_dispatch), dispatcher the outcome, reported or settled -// admitted queued lifecycle bookkeeping — -// admitted blocked — an edge the lifecycle map allows and no -// caller takes today -// admitted discarded operator discard -// queued blocked — an edge the lifecycle map allows and no -// caller takes today -// queued discarded operator discard -// completed — nobody terminal -// discarded — nobody terminal +// blocked dispatched nobody an edge the lifecycle map has and no +// caller takes +// admitted dispatched an older build's dispatcher joined a task +// queued dispatched an older build's dispatcher joined a task +// dispatched dispatched an older build's dispatcher redispatched onto a new task +// dispatched admitted an older build's dispatcher never handed to a worker, task retired, +// or exposed at launch with a spawn +// proven failed: its one retry +// dispatched blocked an older build's dispatcher exposed at launch, spawn failed again +// dispatched completed an older build's worker or dispatcher the outcome, reported or settled +// admitted queued lifecycle bookkeeping - +// admitted blocked nobody an edge the lifecycle map allows and no +// caller takes +// admitted discarded handoff, operator handed to the session, or import +// queued blocked nobody an edge the lifecycle map allows and no +// caller takes +// queued discarded handoff, operator handed to the session, or import +// completed - nobody terminal +// discarded - nobody terminal // // Writing the state a record already has is a repeat and always allowed. Any // pair not in the table is refused. Into dispatched and out of it, the task // decides, as a rule about the moves rather than about the state: a record -// enters dispatched only when a live task already carries it, and leaves it — -// other than to completed — only when none does +// enters dispatched only when a live task already carries it, and leaves it, +// other than to completed, only when none does // (events_dispatched_while_on_a_live_task, and move). A record can therefore // be dispatched with no live task, and one case is expected: work a worker was // handed, whose task was superseded, waiting for its outcome (invariant 7). // // # Task (tasks) // -// live created by the dispatcher (CreateTask); its token is valid -// superseded by the dispatcher or an operator's redispatch (SupersedeTask); -// its token is refused; terminal, and the row is never deleted +// live created by an older build's dispatcher; its token was valid +// superseded by that dispatcher or a person's redispatch; terminal, and +// the row is never deleted // -// An operator's redispatch is the dispatcher's own two writes in one -// transaction: supersedeTask on the live task — which refuses its token from -// then on, retires its rows, and returns what no worker was handed to -// admitted — and createTask for the events being run again. It can therefore -// redispatch an admitted, queued or dispatched record, which covers held work -// waiting for its outcome and the spawn-failure retry. A completed or -// discarded record it cannot: those are terminal here, so redispatching one -// is a decision this ledger does not carry (plan step 21 owns it). +// A redispatch superseded the live task, which refused its token, retired its +// rows and returned what no worker was handed to admitted, and created a task +// for the events being run again, in one transaction. // // # Delivery (task_events.delivery), per event on a task // -// admitted → exposed worker (get_dispatch), dispatcher at launch -// exposed → delivered worker (ack_dispatch); the id the worker points at, -// when it has one, is written with this move or never -// (task_events_acknowledgement_settles_once), so an id -// arriving after the acknowledgement is a second -// report and refused -// exposed → completed worker (complete_dispatch) -// exposed → completed dispatcher settlement (worker gone before ack) -// delivered → completed worker (complete_dispatch), dispatcher settlement -// exposed → withdrawn dispatcher (withdrawExposure): the spawn failed -// before any worker process existed. The database -// holds what it can see: an exposure written at -// launch that no worker ever pulled (pulled_at, set by -// a worker's first get_dispatch), on a superseded task, -// with no live task carrying the event; once, and the -// row moves no more. That no process existed at all is -// the driver's report, which the dispatcher acts on. +// admitted → exposed the worker's get_dispatch, or the dispatcher at launch +// exposed → delivered the worker's ack_dispatch; the id the worker pointed +// at, when it had one, was written with this move or +// never (task_events_acknowledgement_settles_once) +// exposed → completed the worker's complete_dispatch, or the dispatcher's +// settlement of a worker gone before it acknowledged +// delivered → completed the same two +// exposed → withdrawn the dispatcher, for a spawn that failed before any +// worker process existed: an exposure written at +// launch that no worker pulled (pulled_at), on a +// superseded task, with no live task carrying the +// event; once, and the row moves no more // // Forward only, and never skipping exposure: nothing a worker was never -// handed is acknowledged or completed. A row is retired (retired_at) -// when its task is superseded; a worker can touch only its live task's rows. +// handed was acknowledged or completed. A row is retired (retired_at) when +// its task is superseded. // // # Guard (task_events.guard) // -// armed → canceled worker (get_dispatch) -// armed → fired connector's thirty-second acknowledgement +// armed → canceled the worker's get_dispatch +// armed → fired the connector's thirty-second acknowledgement // -// '' (none) and armed are only ever written when the row is created. +// '' (none) and armed were only ever written when the row was created. // // # Invariants // // 1. One live task per event: task_events_one_live_task. -// 2. One task per conversation, and in the conversation's order: an event -// joins a task only if no live task holds its conversation, no record on -// that conversation is dispatched outside this task, and no admitted -// record older than it is waiting there — admission queues a record -// precisely because an earlier one is next, and a queued record does not -// go around it (conversationsAreFree, and -// task_events_one_live_task_per_conversation). The conversation a task +// 2. One task per conversation, and in the conversation's order +// (task_events_one_live_task_per_conversation). The conversation a task // holds is written on the task's own rows, because retention clears a // terminal record's conversation_key while its task is still live. -// 3. The token is valid only while its task is live, checked inside every -// worker call's own transaction. +// 3. A task's token was valid only while the task was live. // 4. Nothing leaves dispatched while a worker may still act: a record with a // delivery exposed or delivered, on any task and not withdrawn, leaves // dispatched only to completed (move, and the -// events_handed_work_settles_first trigger). The one release is the -// spec's automatic retry: an exposure written at launch whose spawn -// failed before any worker process existed is withdrawn, and the record -// returns to admitted for its one retry, or goes to blocked after a -// second failure (withdrawExposure) — the only way from dispatched to -// blocked. -// 5. A worker acts only on its own task's rows, reports only what it -// pulled, and a reported outcome stands — the report path enforces that, -// and the delivery triggers hold the same shape for anything else writing -// to the file: forward only, never past a missing pull, with the -// dispatcher settling a completed record as the one exception. -// 6. A task is made only of instructions a worker can pull, and finished -// work is never handed out for the first time: a completed record is -// served, acknowledged and completed only by the worker that pulled it -// (pulled_at), never on the strength of an exposure written at launch. -// 7. Superseding retires the task's rows and returns only what it never +// events_handed_work_settles_first trigger). This build still holds every +// write of its own to it. The one release was the automatic retry of a +// spawn that failed before any worker process existed. +// 5. A worker acted only on its own task's rows, reported only what it +// pulled, and a reported outcome stands; the delivery triggers hold the +// same shape for anything else writing to the file. +// 6. Finished work was never handed out for the first time: a completed +// record was served, acknowledged and completed only by the worker that +// pulled it (pulled_at). +// 7. Superseding retired the task's rows and returned only what it never // exposed to admitted; what a worker was handed stays dispatched (4) and -// waits for its outcome or a redispatch, which supersedes and creates in -// one transaction (supersedeTask and createTask take the caller's). +// waits for its outcome. // 8. Completed and discarded are terminal (events_terminal_is_terminal). // // Every transaction takes the write lock as it opens (_txlock=immediate), and // each call ends it as soon as its ledger work is done. -// Delivery is an event's delivery state on a task. It moves forward only. -type Delivery string - -const ( - // DeliveryAdmitted is on the task and not yet handed to a worker. - DeliveryAdmitted Delivery = "admitted" - // DeliveryExposed was handed to a worker, which may have acted on it. - DeliveryExposed Delivery = "exposed" - // DeliveryDelivered was acknowledged by the worker. - DeliveryDelivered Delivery = "delivered" - // DeliveryCompleted has the worker's outcome. - DeliveryCompleted Delivery = "completed" -) - // Outcome is what a worker reports for an event. type Outcome string @@ -181,808 +137,6 @@ const ( OutcomeFailed Outcome = "failed" ) -// Refusals a worker's dispatch call can meet. Each is an answer, not a fault. -var ( - // ErrTaskTokenRefused is a token that names no task, or one a - // redispatch superseded. The two are not told apart: a worker holding - // either has no task. - ErrTaskTokenRefused = errors.New("the task token names no current task") - // ErrNotOnTask is an event id the token's task does not carry. Whether - // the event exists on another task is not said. - ErrNotOnTask = errors.New("the event is not on this task") - // ErrNotExposed is an acknowledgement or completion for an event the - // worker was never handed. - ErrNotExposed = errors.New("the event was not handed to this worker; call get_dispatch for it first") - // ErrReportConflict is a second report that disagrees with the first. - // Reported outcomes stand. - ErrReportConflict = errors.New("the event already has a different report") - // ErrNotDispatchable is an event whose record left the path to a worker - // after it joined the task: blocked or withdrawn, or its content dropped. - ErrNotDispatchable = errors.New("the event can no longer be dispatched") - // ErrInvalidReport is a report the worker can correct: an outcome that - // is not one of the two, a link that is not a URL, too many links. - ErrInvalidReport = errors.New("the report is not valid") - // ErrConversationBusy is an event whose conversation already has a - // dispatched record outside the task being created: a running task, or - // work a worker was handed that is not settled yet. A conversation has - // one task at a time. - ErrConversationBusy = errors.New("the event's conversation already has a task") - // ErrNoSuchTask is a task id the ledger does not hold. - ErrNoSuchTask = errors.New("no such task") - // ErrEventOnLiveTask is an event a live task already carries. Handing it - // to a second task would give two workers one instruction. - ErrEventOnLiveTask = errors.New("the event is already on a live task") -) - -// TaskGrant is a new task and the token that binds a worker to it. The token -// is returned once and stored only as a hash. -type TaskGrant struct { - ID int64 - Token string -} - -// CreateTask puts records on a new task at delivery admitted, with the -// acknowledgement guard armed for the records whose verdict asks for one, and -// moves each record to dispatched in the same transaction: a record is -// dispatched exactly while a live task carries it. -// -// An event is on at most one live task. A second task for an event whose task -// was not superseded is refused with ErrEventOnLiveTask, and nothing is -// written. The refusal is the database's own (task_events_one_live_task), so a -// retried or concurrent launch cannot get past it. A redispatch supersedes the -// old task first. -func (l *Ledger) CreateTask(ctx context.Context, eventIDs []int64) (TaskGrant, error) { - var grant TaskGrant - err := retryBusy(func() error { - tx, err := l.db.BeginTx(ctx, nil) - if err != nil { - return fmt.Errorf("connector: begin task: %w", err) - } - defer func() { _ = tx.Rollback() }() - if grant, err = l.createTask(ctx, tx, eventIDs); err != nil { - return err - } - if err := tx.Commit(); err != nil { - return fmt.Errorf("connector: commit task: %w", err) - } - return nil - }) - return grant, err -} - -// createTask writes a task inside the caller's transaction, so the dispatcher -// can write the task, its attempt and the originating event's exposure as one -// commit. Every guarantee CreateTask documents holds within tx; nothing is -// committed here, and a refusal leaves tx for the caller to roll back. -func (l *Ledger) createTask(ctx context.Context, tx *sql.Tx, eventIDs []int64) (TaskGrant, error) { - if len(eventIDs) == 0 { - return TaskGrant{}, errors.New("connector: a task needs at least one event") - } - seen := make(map[int64]bool, len(eventIDs)) - for _, id := range eventIDs { - if seen[id] { - return TaskGrant{}, fmt.Errorf("connector: event %d is named twice for one task", id) - } - seen[id] = true - } - raw := make([]byte, 32) - if _, err := rand.Read(raw); err != nil { - return TaskGrant{}, fmt.Errorf("connector: task token: %w", err) - } - token := base64.RawURLEncoding.EncodeToString(raw) - - // Read first, write after: a refusal leaves the caller's transaction as - // it found it, whatever the caller then does with it. - guards := make([]string, 0, len(eventIDs)) - conversations := make([]string, 0, len(eventIDs)) - for _, id := range eventIDs { - var ( - acknowledge, hasInstruction int - state, conversation string - ) - switch err := tx.QueryRowContext(ctx, `SELECT acknowledge, content_dropped = 0 AND snapshot IS NOT NULL AND length(snapshot) > 0, state, conversation_key FROM events WHERE id = ?`, id).Scan(&acknowledge, &hasInstruction, &state, &conversation); { - case errors.Is(err, sql.ErrNoRows): - return TaskGrant{}, fmt.Errorf("connector: task event %d: %w", id, ErrNoSuchRecord) - case err != nil: - return TaskGrant{}, fmt.Errorf("connector: task event %d: %w", id, err) - } - if s := RecordState(state); s != StateAdmitted && s != StateQueued && s != StateDispatched { - return TaskGrant{}, fmt.Errorf("connector: task event %d is %s; only admitted, queued or redispatched work joins a task", id, state) - } - var onLive bool - if err := tx.QueryRowContext(ctx, `SELECT EXISTS (SELECT 1 FROM task_events WHERE event_id = ? AND retired_at IS NULL)`, id).Scan(&onLive); err != nil { - return TaskGrant{}, fmt.Errorf("connector: task event %d: %w", id, err) - } - if onLive { - // The unique index refuses it too, whoever writes; this is the - // same refusal with the event named. - return TaskGrant{}, fmt.Errorf("connector: task event %d: %w", id, ErrEventOnLiveTask) - } - if hasInstruction == 0 { - // A task a worker could pull nothing from would read as a task - // with nothing left to do. A record without its instruction - // needs a new verdict first. - return TaskGrant{}, fmt.Errorf("connector: task event %d has no instruction: %w", id, ErrNotDispatchable) - } - guard := "" - if acknowledge != 0 { - guard = "armed" - } - guards = append(guards, guard) - conversations = append(conversations, conversation) - } - - // One task per conversation (invariant 2), asked as two questions, both - // before anything is written: a refusal leaves the caller's transaction - // untouched, and the records this call dispatches never count themselves. - if err := l.conversationsAreFree(ctx, tx, eventIDs, conversations); err != nil { - return TaskGrant{}, err - } - - res, err := tx.ExecContext(ctx, `INSERT INTO tasks (token_sha256, created_at) VALUES (?, ?)`, tokenHash(token), l.timestamp()) - if err != nil { - return TaskGrant{}, fmt.Errorf("connector: create task: %w", err) - } - taskID, err := res.LastInsertId() - if err != nil { - return TaskGrant{}, fmt.Errorf("connector: create task: %w", err) - } - for i, id := range eventIDs { - if _, err := tx.ExecContext(ctx, `INSERT INTO task_events (task_id, event_id, guard, conversation_key) VALUES (?, ?, ?, ?)`, taskID, id, guards[i], conversations[i]); err != nil { - if isConstraint(err) { - return TaskGrant{}, fmt.Errorf("connector: task event %d: %w", id, ErrEventOnLiveTask) - } - return TaskGrant{}, fmt.Errorf("connector: task event %d: %w", id, err) - } - } - for _, id := range eventIDs { - // Admitted or queued work joins a task; a dispatched record whose - // task was superseded joins its replacement. - // The pre-pass read this state; the move is what makes it so under a - // concurrent writer, and refuses if it changed underneath. - moved, err := l.move(ctx, tx, transition{id: id, state: StateDispatched, from: []RecordState{StateAdmitted, StateQueued, StateDispatched}}) - if err != nil { - return TaskGrant{}, err - } - if !moved { - return TaskGrant{}, fmt.Errorf("connector: task event %d changed state while its task was being written", id) - } - } - return TaskGrant{ID: taskID, Token: token}, nil -} - -// conversationsAreFree asks whether the conversations these events are on are -// free for a task of their own. -// -// Two questions, because the answer lives in two places. A record waiting for -// a worker or still dispatched is in events: admitted means admission decided -// it goes next and a queued sibling may not jump ahead of it, and dispatched -// means work a worker may still be holding, whether or not a live task -// carries it (invariant 7). A conversation a live task already holds is in -// task_events, which keeps the conversation of every row it was written for — -// the record's own conversation_key is cleared once retention reaches it, and -// a task whose records are all terminal is still a worker's task until it is -// superseded. -func (l *Ledger) conversationsAreFree(ctx context.Context, tx *sql.Tx, eventIDs []int64, conversations []string) error { - placeholders := strings.TrimSuffix(strings.Repeat("?, ", len(eventIDs)), ", ") - for i, conversation := range conversations { - if conversation == "" { - continue - } - args := make([]any, 0, len(eventIDs)+1) - args = append(args, conversation) - for _, id := range eventIDs { - args = append(args, id) - } - - var ( - other int64 - otherState string - ) - //nolint:gosec // G202: placeholders, not values - // - // An older admitted sibling is next on its conversation, and a queued - // one does not go around it: admission queues a record precisely - // because an earlier one holds the conversation. A dispatched sibling - // counts whatever its id, because a worker may still be holding it. - switch err := tx.QueryRowContext(ctx, ` -SELECT id, state FROM events -WHERE conversation_key = ? AND id NOT IN (`+placeholders+`) - AND (state = 'dispatched' OR (state = 'admitted' AND id < ?)) -ORDER BY id LIMIT 1`, append(args, eventIDs[i])...).Scan(&other, &otherState); { - case err == nil && RecordState(otherState) == StateDispatched: - return fmt.Errorf("connector: event %d is dispatched on the same conversation: %w", other, ErrConversationBusy) - case err == nil: - return fmt.Errorf("connector: event %d is admitted ahead of event %d on the same conversation: %w", other, eventIDs[i], ErrConversationBusy) - case !errors.Is(err, sql.ErrNoRows): - return fmt.Errorf("connector: read conversations: %w", err) - } - - var task int64 - //nolint:gosec // G202: placeholders, not values - switch err := tx.QueryRowContext(ctx, ` -SELECT task_id FROM task_events -WHERE conversation_key = ? AND retired_at IS NULL AND event_id NOT IN (`+placeholders+`) -ORDER BY task_id LIMIT 1`, args...).Scan(&task); { - case err == nil: - return fmt.Errorf("connector: task %d is live on the conversation of event %d: %w", task, eventIDs[i], ErrConversationBusy) - case !errors.Is(err, sql.ErrNoRows): - return fmt.Errorf("connector: read live tasks: %w", err) - } - } - return nil -} - -// SupersedeTask retires a task: its token is refused from then on, and its -// events are free to join a new task. An event this task never exposed -// returns to admitted, to be dispatched again once its conversation is free — -// unless something else still holds it dispatched, which invariant 4 decides -// and this call does not override: an earlier task's exposure that was never -// withdrawn keeps the record dispatched, waiting for its outcome, and a -// record a person or a later verdict has moved stays where it was put. An -// event a worker was handed stays dispatched for the same reason: that worker -// may have acted on it, and it waits for its settlement or a redispatch. -func (l *Ledger) SupersedeTask(ctx context.Context, taskID int64) error { - return retryBusy(func() error { - tx, err := l.db.BeginTx(ctx, nil) - if err != nil { - return fmt.Errorf("connector: begin supersede: %w", err) - } - defer func() { _ = tx.Rollback() }() - if err := l.supersedeTask(ctx, tx, taskID); err != nil { - return err - } - return tx.Commit() - }) -} - -// supersedeTask is SupersedeTask inside the caller's transaction, so a -// redispatch can retire the old task and create the new one in one commit. -func (l *Ledger) supersedeTask(ctx context.Context, tx *sql.Tx, taskID int64) error { - var exists bool - if err := tx.QueryRowContext(ctx, `SELECT EXISTS (SELECT 1 FROM tasks WHERE id = ?)`, taskID).Scan(&exists); err != nil { - return fmt.Errorf("connector: supersede task %d: %w", taskID, err) - } - if !exists { - return fmt.Errorf("connector: supersede task %d: %w", taskID, ErrNoSuchTask) - } - rows, err := tx.QueryContext(ctx, `SELECT event_id FROM task_events WHERE task_id = ? AND retired_at IS NULL AND delivery = 'admitted'`, taskID) - if err != nil { - return fmt.Errorf("connector: supersede task %d: %w", taskID, err) - } - var unexposed []int64 - for rows.Next() { - var id int64 - if err := rows.Scan(&id); err != nil { - _ = rows.Close() - return fmt.Errorf("connector: supersede task %d: %w", taskID, err) - } - unexposed = append(unexposed, id) - } - if err := rows.Err(); err != nil { - _ = rows.Close() - // Retiring the task on a partial list would leave the rest dispatched - // on no live task, never exposed and never returned. - return fmt.Errorf("connector: supersede task %d: %w", taskID, err) - } - if err := rows.Close(); err != nil { - return fmt.Errorf("connector: supersede task %d: %w", taskID, err) - } - - now := l.timestamp() - if _, err := tx.ExecContext(ctx, `UPDATE tasks SET superseded_at = COALESCE(superseded_at, ?) WHERE id = ?`, now, taskID); err != nil { - return fmt.Errorf("connector: supersede task %d: %w", taskID, err) - } - for _, id := range unexposed { - // Only a record still dispatched moves, and only if nothing else - // holds it: move's own refusals — a worker holding it on another - // task, a record a person or a later verdict moved — are the answer, - // not an error, so the result is read for what it is and not acted - // on. - if _, err := l.move(ctx, tx, transition{id: id, state: StateAdmitted, from: []RecordState{StateDispatched}}); err != nil { - return err - } - } - return nil -} - -// withdrawExposure releases an event whose worker never existed: the driver -// proved the spawn itself failed, after the dispatcher had written the -// originating event exposed at launch. It runs in the caller's transaction, -// after supersedeTask on the same task. The record goes to admitted, to be -// retried once, or — after a second failure — to blocked with reason. The -// database refuses the marker for anything but a launch exposure no worker -// pulled, on a superseded task, while no live task carries the event, and -// refuses it twice. So the order is supersede, withdraw, then create the -// retry task. -func (l *Ledger) withdrawExposure(ctx context.Context, tx *sql.Tx, taskID, eventID int64, to RecordState, reason string) error { - if to != StateAdmitted && to != StateBlocked { - return fmt.Errorf("connector: withdraw event %d: a withdrawn event is retried (admitted) or blocked, not %s", eventID, to) - } - res, err := tx.ExecContext(ctx, `UPDATE task_events SET withdrawn_at = ? WHERE task_id = ? AND event_id = ?`, l.timestamp(), taskID, eventID) - if err != nil { - return fmt.Errorf("connector: withdraw event %d: %w", eventID, err) - } - if n, err := res.RowsAffected(); err != nil || n == 0 { - return fmt.Errorf("connector: withdraw event %d: %w", eventID, ErrNotOnTask) - } - moved, err := l.move(ctx, tx, transition{id: eventID, state: to, reason: reason, from: []RecordState{StateDispatched}}) - if err != nil { - return err - } - if !moved { - return fmt.Errorf("connector: withdraw event %d: another worker still holds it: %w", eventID, ErrHeldByWorker) - } - return nil -} - -// isConstraint reports a SQLite constraint violation. -func isConstraint(err error) bool { - var sqliteErr *sqlite.Error - return errors.As(err, &sqliteErr) && sqliteErr.Code()&0xff == sqlite3.SQLITE_CONSTRAINT -} - -func tokenHash(token string) string { - sum := sha256.Sum256([]byte(token)) - return hex.EncodeToString(sum[:]) -} - -// TaskDispatch is the ledger as one worker sees it: the events on the task its -// token names, and nothing else. There is no listing; a worker never reads -// other tasks. -type TaskDispatch struct { - ledger *Ledger - hash string - agentID int64 - - // afterTx runs when a call has closed its transaction and is building its - // answer. A test seam: it is where the ledger must already be free. - afterTx func() -} - -// Dispatch binds the ledger to a worker's task token, refusing one that names -// no live task now. The token is checked again on every call, in the call's -// own transaction, so a redispatch that supersedes it later takes effect at -// once. agentID is the agent's Person id, whose own mentions are stripped from -// the instructions handed out. -func (l *Ledger) Dispatch(ctx context.Context, token string, agentID int64) (*TaskDispatch, error) { - if strings.TrimSpace(token) == "" { - return nil, errors.New("connector: a dispatch needs the task token") - } - if agentID <= 0 { - return nil, errors.New("connector: a dispatch needs the agent's Person id") - } - d := &TaskDispatch{ledger: l, hash: tokenHash(token), agentID: agentID} - var live int - if err := l.db.QueryRowContext(ctx, `SELECT COUNT(*) FROM tasks WHERE token_sha256 = ? AND superseded_at IS NULL`, d.hash).Scan(&live); err != nil { - return nil, fmt.Errorf("connector: resolve task token: %w", err) - } - if live == 0 { - return nil, fmt.Errorf("connector: %w", ErrTaskTokenRefused) - } - return d, nil -} - -// Instruction is what get_dispatch hands a worker. It is an allowlist: every -// field is named here, and nothing the ledger holds reaches a worker unless -// it is one of them. No directory, no feed position, no token. -type Instruction struct { - EventID int64 `json:"event_id"` - EventType string `json:"event_type"` - Trigger string `json:"trigger"` - Class string `json:"class,omitempty"` - - Recording InstructionRecording `json:"recording"` - ReplyTo InstructionReply `json:"reply_to"` - RequesterID int64 `json:"requester_id"` - - // Acknowledge says a person asked for something: the worker acknowledges - // it and reports the id through ack_dispatch. - Acknowledge bool `json:"acknowledge"` - // GuardAcknowledged says the connector's guard already acknowledged the - // event, so the worker does not acknowledge it again. - GuardAcknowledged bool `json:"guard_acknowledged"` - Delivery Delivery `json:"delivery"` - - // Content is the instruction as admission read it, with the agent's own - // mention stripped. The live recording may be newer. - Content string `json:"content"` - ContentUpdatedAt time.Time `json:"content_updated_at"` -} - -// InstructionRecording points at the recording the event is about. -type InstructionRecording struct { - BucketID int64 `json:"bucket_id"` - RecordingID int64 `json:"recording_id"` - Type string `json:"type"` - Title string `json:"title"` - URL string `json:"url"` -} - -// InstructionReply is where the worker's acknowledgement and reply go. -type InstructionReply struct { - Kind string `json:"kind"` - RecordingID int64 `json:"recording_id"` -} - -// Receipt is the delivery state an acknowledgement or completion left. -type Receipt struct { - EventID int64 `json:"event_id"` - Delivery Delivery `json:"delivery"` - AckID *int64 `json:"ack_id,omitempty"` - Outcome Outcome `json:"outcome,omitempty"` - ReplyID *int64 `json:"reply_id,omitempty"` - Links []string `json:"links,omitempty"` -} - -// Completion is a worker's report for one event. -type Completion struct { - Outcome Outcome - Links []string - ReplyID *int64 -} - -// Completion limits: a report is a handful of links, not a document. -const ( - maxCompletionLinks = 20 - maxLinkLength = 2048 -) - -// Get returns the instruction for eventID, or for the earliest event on the -// task not yet acknowledged when eventID is zero; ok is false when there is -// none. Handing out an event that was never exposed writes exposed before the -// instruction is returned, and cancels an armed guard. A repeat returns the -// same instruction and writes nothing. -// -// Only an event whose record is still on the way to a worker is handed out: -// dispatched, or completed, with its content. The earliest skips any other, -// so one event withdrawn or blocked never hides the rest of the task. -func (d *TaskDispatch) Get(ctx context.Context, eventID int64) (Instruction, bool, error) { - var ( - out Instruction - ok bool - ) - err := retryBusy(func() error { - var err error - out, ok, err = d.get(ctx, eventID) - return err - }) - return out, ok, err -} - -type taskEvent struct { - pulled bool - delivery Delivery - guard string - ackID sql.NullInt64 - outcome string - links string - replyID sql.NullInt64 -} - -// get is one get_dispatch: the ledger work in a transaction, the instruction -// built once it is closed. -func (d *TaskDispatch) get(ctx context.Context, eventID int64) (Instruction, bool, error) { - l := d.ledger - tx, err := l.db.BeginTx(ctx, nil) - if err != nil { - return Instruction{}, false, fmt.Errorf("connector: begin get_dispatch: %w", err) - } - defer func() { _ = tx.Rollback() }() - taskID, err := d.task(ctx, tx) - if err != nil { - return Instruction{}, false, err - } - - if eventID == 0 { - err := tx.QueryRowContext(ctx, ` -SELECT te.event_id FROM task_events te JOIN events e ON e.id = te.event_id -WHERE te.task_id = ? AND te.retired_at IS NULL AND te.delivery IN ('admitted', 'exposed') AND `+servableSQL+` -ORDER BY te.event_id LIMIT 1`, taskID).Scan(&eventID) - if errors.Is(err, sql.ErrNoRows) { - return Instruction{}, false, nil - } - if err != nil { - return Instruction{}, false, fmt.Errorf("connector: get_dispatch: %w", err) - } - } - te, err := loadTaskEvent(ctx, tx, taskID, eventID) - if err != nil { - return Instruction{}, false, err - } - record, err := loadRecord(ctx, tx, eventID) - if err != nil { - return Instruction{}, false, err - } - if !servable(record, te.pulled) { - return Instruction{}, false, fmt.Errorf("connector: event %d: %w", eventID, ErrNotDispatchable) - } - - now := l.timestamp() - wrote := false - if te.delivery == DeliveryAdmitted { - // Written before anything about the event leaves this call: a worker - // handed an instruction may act on it whether or not it reports. - if _, err := tx.ExecContext(ctx, `UPDATE task_events SET delivery = 'exposed', exposed_at = ? WHERE task_id = ? AND event_id = ? AND delivery = 'admitted'`, now, taskID, eventID); err != nil { - return Instruction{}, false, fmt.Errorf("connector: expose event %d: %w", eventID, err) - } - wrote = true - } - if !te.pulled { - // The first pull by a worker is recorded even when the dispatcher - // already exposed the event at launch: from here on a worker has the - // instruction, and the exposure can no longer be withdrawn as a - // spawn that failed before any worker existed. - if _, err := tx.ExecContext(ctx, `UPDATE task_events SET pulled_at = ? WHERE task_id = ? AND event_id = ? AND pulled_at IS NULL`, now, taskID, eventID); err != nil { - // A hold refuses a first pull (ledger_hold.go): the worker is - // told the connector is held, not given the instruction. - if held, holdErr := isHeld(ctx, tx); holdErr == nil && held { - return Instruction{}, false, fmt.Errorf("connector: event %d: %w", eventID, ErrHeld) - } - return Instruction{}, false, fmt.Errorf("connector: record the pull of %d: %w", eventID, err) - } - wrote = true - } - if te.guard == "armed" { - if _, err := tx.ExecContext(ctx, `UPDATE task_events SET guard = 'canceled' WHERE task_id = ? AND event_id = ? AND guard = 'armed'`, taskID, eventID); err != nil { - return Instruction{}, false, fmt.Errorf("connector: cancel guard on %d: %w", eventID, err) - } - wrote = true - } - if wrote { - // Read back what the writes left rather than what was read before - // them: the instruction reports the row, not this call's expectation - // of it. - if te, err = loadTaskEvent(ctx, tx, taskID, eventID); err != nil { - return Instruction{}, false, err - } - if err := tx.Commit(); err != nil { - return Instruction{}, false, fmt.Errorf("connector: commit get_dispatch: %w", err) - } - } else if err := tx.Rollback(); err != nil && !errors.Is(err, sql.ErrTxDone) { - return Instruction{}, false, fmt.Errorf("connector: end get_dispatch: %w", err) - } - // The transaction is over. Every transaction here takes the write lock as - // it opens, so decoding the snapshot and stripping the agent's mention — - // the only work in this call that is not a query — happens with the - // ledger free for the connector and for other workers. - if d.afterTx != nil { - d.afterTx() - } - - var snapshot struct { - Type string `json:"type"` - Title string `json:"title"` - AppURL string `json:"app_url"` - Content string `json:"content"` - UpdatedAt time.Time `json:"updated_at"` - } - if err := json.Unmarshal(record.Decision.Snapshot, &snapshot); err != nil { - return Instruction{}, false, fmt.Errorf("connector: event %d snapshot: %w", eventID, err) - } - return Instruction{ - EventID: record.ID, - EventType: record.EventType, - Trigger: record.Decision.Trigger, - Class: record.Decision.Class, - Recording: InstructionRecording{ - BucketID: record.BucketID, - RecordingID: record.RecordingID, - Type: snapshot.Type, - Title: snapshot.Title, - URL: record.Decision.RecordingURL, - }, - ReplyTo: InstructionReply{Kind: record.Decision.ReplyKind, RecordingID: record.Decision.ReplyRecordingID}, - RequesterID: record.Decision.RequesterID, - Acknowledge: record.Decision.Acknowledge, - GuardAcknowledged: te.guard == "fired", - Delivery: te.delivery, - Content: StripMentionsOf(snapshot.Content, d.agentID), - ContentUpdatedAt: snapshot.UpdatedAt, - }, true, nil -} - -// servable is whether an event on a task is handed to its worker: its record -// is dispatched, or completed after this worker pulled it — finished work is -// never handed out for the first time — and it still has its instruction. servableSQL is the same rule over task_events te and events e, -// for the earliest-event query; the two are kept side by side so they cannot -// drift. -func servable(record Record, pulled bool) bool { - // Exposure at launch is the dispatcher's write, not a worker's pull, so - // finished work is served again only to a worker that already had it. - state := record.State == StateDispatched || (record.State == StateCompleted && pulled) - return state && !record.ContentDropped && len(record.Decision.Snapshot) > 0 -} - -const servableSQL = `(e.state = 'dispatched' OR (e.state = 'completed' AND te.pulled_at IS NOT NULL)) - AND e.content_dropped = 0 AND e.snapshot IS NOT NULL AND length(e.snapshot) > 0` - -// Ack records the worker's acknowledgement: delivery moves to delivered, and -// ackID, when given, is the worker's own boost or comment. A repeat — a lost -// tool response retried — answers the same receipt. -func (d *TaskDispatch) Ack(ctx context.Context, eventID int64, ackID *int64) (Receipt, error) { - var out Receipt - err := retryBusy(func() error { - var err error - out, err = d.report(ctx, eventID, func(ctx context.Context, tx *sql.Tx, taskID int64, te taskEvent) (bool, error) { - if ackID != nil && te.ackID.Valid && te.ackID.Int64 != *ackID { - return false, fmt.Errorf("connector: event %d acknowledged as %d: %w", eventID, te.ackID.Int64, ErrReportConflict) - } - if ackID != nil && !te.ackID.Valid && te.delivery != DeliveryExposed { - // The acknowledgement settles with the delivery, id and all: - // this event was acknowledged with nothing to point at, and - // an id arriving afterwards is a second, different report, - // not the same call retried. - return false, fmt.Errorf("connector: event %d was acknowledged without an acknowledgement id: %w", eventID, ErrReportConflict) - } - if te.delivery != DeliveryExposed { - // Acknowledged already, and by whatever this call says: an - // id equal to the one recorded, or none to record. The - // receipt is the answer. - return false, nil - } - _, err := tx.ExecContext(ctx, ` -UPDATE task_events -SET delivery = CASE WHEN delivery = 'exposed' THEN 'delivered' ELSE delivery END, - delivered_at = COALESCE(delivered_at, ?), - ack_id = COALESCE(ack_id, ?) -WHERE task_id = ? AND event_id = ?`, d.ledger.timestamp(), nullableID(ackID), taskID, eventID) - return true, err - }) - return err - }) - return out, err -} - -// Complete records the worker's outcome and acknowledges the event if it was -// not already. The report is recorded whatever has happened to the record -// since the worker was handed it, because it is what the worker did; the -// record moves to completed when it is dispatched. A repeat of the same report -// answers the same receipt; a different one is refused, because a reported -// outcome stands. -func (d *TaskDispatch) Complete(ctx context.Context, eventID int64, c Completion) (Receipt, error) { - var out Receipt - err := retryBusy(func() error { - var err error - out, err = d.report(ctx, eventID, func(ctx context.Context, tx *sql.Tx, taskID int64, te taskEvent) (bool, error) { - // Validated inside the call's transaction, after the token and - // the exposure are: a worker whose task was superseded is told - // that, whatever it sent. - if c.Outcome != OutcomeSucceeded && c.Outcome != OutcomeFailed { - return false, fmt.Errorf("connector: outcome must be %q or %q: %w", OutcomeSucceeded, OutcomeFailed, ErrInvalidReport) - } - links, err := normalizeLinks(c.Links) - if err != nil { - return false, err - } - encoded, err := json.Marshal(links) - if err != nil { - return false, err - } - if te.delivery == DeliveryCompleted { - if te.outcome == string(c.Outcome) && te.links == string(encoded) && sameID(te.replyID, c.ReplyID) { - return false, nil - } - return false, fmt.Errorf("connector: event %d completed as %s: %w", eventID, te.outcome, ErrReportConflict) - } - // The delivery first, the record after: while the record is still - // dispatched, task_events_exposure_comes_first reads this as a - // worker's completion and holds it to the pull. Completing the - // record first would make every completion look like the - // dispatcher settling one. - now := d.ledger.timestamp() - if _, err = tx.ExecContext(ctx, ` -UPDATE task_events -SET delivery = 'completed', delivered_at = COALESCE(delivered_at, ?), completed_at = ?, - outcome = ?, links = ?, reply_id = ? -WHERE task_id = ? AND event_id = ?`, now, now, string(c.Outcome), string(encoded), nullableID(c.ReplyID), taskID, eventID); err != nil { - return false, err - } - if _, err := d.ledger.move(ctx, tx, transition{id: eventID, state: StateCompleted, from: []RecordState{StateDispatched}}); err != nil { - return false, err - } - return true, nil - }) - return err - }) - return out, err -} - -// report runs an acknowledgement or completion in one transaction: the token -// checked, the event found on the task and known to have been exposed, apply -// deciding whether to write, and the receipt read back. -func (d *TaskDispatch) report(ctx context.Context, eventID int64, apply func(context.Context, *sql.Tx, int64, taskEvent) (bool, error)) (Receipt, error) { - if eventID <= 0 { - return Receipt{}, errors.New("connector: event_id must name an event") - } - tx, err := d.ledger.db.BeginTx(ctx, nil) - if err != nil { - return Receipt{}, fmt.Errorf("connector: begin report: %w", err) - } - defer func() { _ = tx.Rollback() }() - taskID, err := d.task(ctx, tx) - if err != nil { - return Receipt{}, err - } - te, err := loadTaskEvent(ctx, tx, taskID, eventID) - if err != nil { - return Receipt{}, err - } - if !te.pulled { - // Exposure at launch is not a worker having the instruction: the - // pull is. A worker reports only what it pulled. - return Receipt{}, fmt.Errorf("connector: event %d: %w", eventID, ErrNotExposed) - } - wrote, err := apply(ctx, tx, taskID, te) - if err != nil { - // A refusal the worker can read is passed on as it is: wrapping it - // would put this package's name in the middle of the message the - // worker is shown. - for _, refusal := range []error{ErrInvalidReport, ErrReportConflict, ErrNotDispatchable, ErrHeldByWorker} { - if errors.Is(err, refusal) { - return Receipt{}, err - } - } - return Receipt{}, fmt.Errorf("connector: report on event %d: %w", eventID, err) - } - if wrote { - if te, err = loadTaskEvent(ctx, tx, taskID, eventID); err != nil { - return Receipt{}, err - } - if err := tx.Commit(); err != nil { - return Receipt{}, fmt.Errorf("connector: commit report on %d: %w", eventID, err) - } - } else if err := tx.Rollback(); err != nil && !errors.Is(err, sql.ErrTxDone) { - return Receipt{}, fmt.Errorf("connector: end report on %d: %w", eventID, err) - } - // The ledger is free before the receipt is built, as in get. - if d.afterTx != nil { - d.afterTx() - } - receipt := Receipt{EventID: eventID, Delivery: te.delivery, Outcome: Outcome(te.outcome)} - if te.ackID.Valid { - receipt.AckID = &te.ackID.Int64 - } - if te.replyID.Valid { - receipt.ReplyID = &te.replyID.Int64 - } - if err := json.Unmarshal([]byte(te.links), &receipt.Links); err != nil { - return Receipt{}, fmt.Errorf("connector: event %d links: %w", eventID, err) - } - return receipt, nil -} - -// task resolves the token to its task inside tx, or refuses it. -func (d *TaskDispatch) task(ctx context.Context, tx *sql.Tx) (int64, error) { - var id int64 - err := tx.QueryRowContext(ctx, `SELECT id FROM tasks WHERE token_sha256 = ? AND superseded_at IS NULL`, d.hash).Scan(&id) - if errors.Is(err, sql.ErrNoRows) { - return 0, fmt.Errorf("connector: %w", ErrTaskTokenRefused) - } - if err != nil { - return 0, fmt.Errorf("connector: resolve task token: %w", err) - } - return id, nil -} - -func loadTaskEvent(ctx context.Context, tx *sql.Tx, taskID, eventID int64) (taskEvent, error) { - var ( - te taskEvent - delivery string - ) - err := tx.QueryRowContext(ctx, ` -SELECT delivery, guard, ack_id, outcome, links, reply_id, pulled_at IS NOT NULL -FROM task_events WHERE task_id = ? AND event_id = ? AND retired_at IS NULL`, taskID, eventID).Scan(&delivery, &te.guard, &te.ackID, &te.outcome, &te.links, &te.replyID, &te.pulled) - if errors.Is(err, sql.ErrNoRows) { - return te, fmt.Errorf("connector: event %d: %w", eventID, ErrNotOnTask) - } - if err != nil { - return te, fmt.Errorf("connector: load task event %d: %w", eventID, err) - } - te.delivery = Delivery(delivery) - return te, nil -} - func loadRecord(ctx context.Context, tx *sql.Tx, id int64) (Record, error) { rows, err := tx.QueryContext(ctx, selectRecords+` WHERE id = ?`, id) if err != nil { @@ -998,38 +152,6 @@ func loadRecord(ctx context.Context, tx *sql.Tx, id int64) (Record, error) { return records[0], nil } -func normalizeLinks(links []string) ([]string, error) { - if len(links) > maxCompletionLinks { - return nil, fmt.Errorf("connector: at most %d links: %w", maxCompletionLinks, ErrInvalidReport) - } - out := make([]string, 0, len(links)) - for _, link := range links { - if len(link) > maxLinkLength { - return nil, fmt.Errorf("connector: a link is at most %d characters: %w", maxLinkLength, ErrInvalidReport) - } - u, err := url.Parse(link) - if err != nil || (u.Scheme != "https" && u.Scheme != "http") || u.Host == "" { - return nil, fmt.Errorf("connector: link %q is not an http(s) URL: %w", link, ErrInvalidReport) - } - out = append(out, link) - } - return out, nil -} - -func nullableID(id *int64) any { - if id == nil { - return nil - } - return *id -} - -func sameID(stored sql.NullInt64, given *int64) bool { - if given == nil { - return !stored.Valid - } - return stored.Valid && stored.Int64 == *given -} - // StripMentionsOf removes every mention of personID from rich text, and // leaves the rest as it was. A worker handed its own mention reads an // instruction addressed to itself, which says nothing the dispatch does not @@ -1293,50 +415,6 @@ func StateDirName(accountID string, agentID int64) string { return accountID + "-" + strconv.FormatInt(agentID, 10) } -// ErrNotAStateDir is a directory that is not a connector state directory for -// the account asked about. StateDirError carries why. -var ErrNotAStateDir = errors.New("not the connector's state directory for this account") - -// StateDirError says which rule a state directory failed, in fields a caller -// can build its own message from rather than by reading this one. -type StateDirError struct { - // Dir is the directory as given, made absolute. - Dir string - // Root is the connector's state root, the only place a state directory - // lives. - Root string - // Account is the account the directory names, empty when it names none; - // Want is the account it had to name. - Account, Want string - // Why is the rule it failed. - Why StateDirProblem -} - -// StateDirProblem is why a state directory was refused. -type StateDirProblem string - -const ( - // StateDirElsewhere is a directory outside the state root. - StateDirElsewhere StateDirProblem = "outside the connector's state root" - // StateDirMisnamed is a directory not named -. - StateDirMisnamed StateDirProblem = "not named -" - // StateDirOtherAccount is another account's state directory. - StateDirOtherAccount StateDirProblem = "another account's" -) - -func (e *StateDirError) Error() string { - switch e.Why { - case StateDirElsewhere: - return fmt.Sprintf("%s is not inside %s", e.Dir, e.Root) - case StateDirOtherAccount: - return fmt.Sprintf("%s belongs to account %s, not %s", e.Dir, e.Account, e.Want) - default: - return fmt.Sprintf("%s is not named -", e.Dir) - } -} - -func (e *StateDirError) Unwrap() error { return ErrNotAStateDir } - // StateRoot is where every connector state directory lives: // $XDG_STATE_HOME/basecamp/connect, or ~/.local/state/basecamp/connect when // XDG_STATE_HOME is unset or not absolute, as the XDG specification says. @@ -1352,49 +430,5 @@ func StateRoot() (string, error) { return filepath.Join(filepath.Clean(base), "basecamp", "connect"), nil } -// ResolveStateDir is the one place a state directory is accepted: dir must -// be exactly StateRoot/-, and its account must be -// accountID, compared as numbers. It returns the directory made absolute — -// which is the one every caller should go on to use — and the agent's Person -// id. -// -// The location is part of the check, not only the name. A directory named -// for this account anywhere else — a copy of another account's ledger renamed -// to match — is refused, because the name is what binds a ledger to an -// account and anyone can choose a name. -func ResolveStateDir(dir, accountID string) (string, int64, error) { - root, err := StateRoot() - if err != nil { - return "", 0, err - } - abs, err := filepath.Abs(dir) - if err != nil { - return "", 0, fmt.Errorf("connector: state directory %q: %w", dir, err) - } - refuse := func(why StateDirProblem, account string) (string, int64, error) { - return "", 0, &StateDirError{Dir: abs, Root: root, Account: account, Want: accountID, Why: why} - } - if filepath.Dir(abs) != root { - return refuse(StateDirElsewhere, "") - } - account, agent, ok := strings.Cut(filepath.Base(abs), "-") - agentID, err := strconv.ParseInt(agent, 10, 64) - if !ok || err != nil || agentID <= 0 || agent != strconv.FormatInt(agentID, 10) { - // The agent is spelled one way, so one directory answers to one name: - // "+52007412" and "052007412" are other directories, not this one. - return refuse(StateDirMisnamed, account) - } - given, errGiven := strconv.ParseUint(account, 10, 64) - if errGiven != nil || given == 0 { - // Not an account at all: the name is wrong, not another account's. - return refuse(StateDirMisnamed, account) - } - want, errWant := strconv.ParseUint(accountID, 10, 64) - if errWant != nil || given != want { - return refuse(StateDirOtherAccount, account) - } - return abs, agentID, nil -} - // LedgerFile is the ledger's file name inside the state directory. const LedgerFile = "ledger.db" diff --git a/internal/connector/ledger_dispatch_test.go b/internal/connector/ledger_dispatch_test.go index 7042e2141..561183623 100644 --- a/internal/connector/ledger_dispatch_test.go +++ b/internal/connector/ledger_dispatch_test.go @@ -2,798 +2,40 @@ package connector import ( "context" - "database/sql" - "encoding/json" "fmt" - "os" - "path/filepath" - "sort" "strings" "testing" - "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - - "github.com/basecamp/basecamp-cli/internal/connector/admission" ) const otherPersonID int64 = 1001 -// dispatchFixture is a task holding an originating mention (event 1) and a -// follow-up queued on the same conversation (event 2), both still admitted on -// the task. -type dispatchFixture struct { - ledger *Ledger - path string - grant TaskGrant - d *TaskDispatch -} - -func newDispatchFixture(t *testing.T) dispatchFixture { - t.Helper() - path := filepath.Join(t.TempDir(), "state", "connector.db") - ledger, err := OpenLedger(path) - require.NoError(t, err) - t.Cleanup(func() { _ = ledger.Close() }) - ctx := context.Background() - for _, id := range []int64{1, 2} { - seenRecord(t, ledger, id) - v := admittedVerdict(id, 0, "recording:10304028989") - v.Snapshot.Content = "
" + mentionMarkup(adapterAgentID) + " please ask " + mentionMarkup(otherPersonID) + " about it
" - _, err := ledger.Admission().Commit(ctx, v) - require.NoError(t, err) - } - require.Equal(t, StateQueued, getRecord(t, ledger, 2).State) - grant, err := ledger.CreateTask(ctx, []int64{1, 2}) - require.NoError(t, err) - d, err := ledger.Dispatch(ctx, grant.Token, adapterAgentID) - require.NoError(t, err) - return dispatchFixture{ledger: ledger, path: path, grant: grant, d: d} -} - -type taskEventRow struct { - Delivery string - Guard string - ExposedAt *string - DeliveredAt *string - CompletedAt *string -} - -func (f dispatchFixture) row(t *testing.T, eventID int64) taskEventRow { - t.Helper() - return f.rowContext(context.Background(), t, eventID) -} - -func (f dispatchFixture) rowContext(ctx context.Context, t *testing.T, eventID int64) taskEventRow { - t.Helper() - var r taskEventRow - require.NoError(t, f.ledger.db.QueryRowContext(ctx, `SELECT delivery, guard, exposed_at, delivered_at, completed_at FROM task_events WHERE task_id = ? AND event_id = ?`, - f.grant.ID, eventID).Scan(&r.Delivery, &r.Guard, &r.ExposedAt, &r.DeliveredAt, &r.CompletedAt)) - return r -} - -// Done when: get_dispatch writes exposed on first call and nothing on repeats. -func TestGetDispatchExposesOnceAndRepeatsWriteNothing(t *testing.T) { - f := newDispatchFixture(t) - ctx := context.Background() - t0 := time.Date(2026, 9, 17, 12, 0, 0, 0, time.UTC) - f.ledger.now = func() time.Time { return t0 } - record := getRecord(t, f.ledger, 2) - require.Equal(t, StateDispatched, record.State) - - first, ok, err := f.d.Get(ctx, 2) - require.NoError(t, err) - require.True(t, ok) - assert.Equal(t, DeliveryExposed, first.Delivery) - row := f.row(t, 2) - assert.Equal(t, "exposed", row.Delivery) - require.NotNil(t, row.ExposedAt) - assert.Equal(t, "admitted", f.row(t, 1).Delivery, "the other event is not exposed") - - f.ledger.now = func() time.Time { return t0.Add(time.Minute) } - again, ok, err := f.d.Get(ctx, 2) - require.NoError(t, err) - require.True(t, ok) - assert.Equal(t, first, again, "a repeat returns the same instruction") - assert.Equal(t, row, f.row(t, 2), "and marks nothing further") - assert.Equal(t, record.Revision, getRecord(t, f.ledger, 2).Revision, "exposure is the task's, not the record's") -} - -func TestGetDispatchWithoutAnIDIsTheEarliestNotAcknowledged(t *testing.T) { - f := newDispatchFixture(t) - ctx := context.Background() - - got, ok, err := f.d.Get(ctx, 0) - require.NoError(t, err) - require.True(t, ok) - assert.Equal(t, int64(1), got.EventID) - - // Exposed but not acknowledged is still the earliest. - got, _, err = f.d.Get(ctx, 0) - require.NoError(t, err) - assert.Equal(t, int64(1), got.EventID) - - _, err = f.d.Ack(ctx, 1, nil) - require.NoError(t, err) - got, ok, err = f.d.Get(ctx, 0) - require.NoError(t, err) - require.True(t, ok) - assert.Equal(t, int64(2), got.EventID) - - _, err = f.d.Complete(ctx, 2, Completion{Outcome: OutcomeSucceeded}) - require.NoError(t, err) - _, ok, err = f.d.Get(ctx, 0) - require.NoError(t, err) - assert.False(t, ok, "nothing is left to acknowledge") -} - -// The instruction is an allowlist: the fields a worker needs, the agent's own -// mention stripped, and nothing that is a route, a position or a token. -func TestGetDispatchHandsOutOnlyTheAllowlist(t *testing.T) { - f := newDispatchFixture(t) - - got, ok, err := f.d.Get(context.Background(), 1) - require.NoError(t, err) - require.True(t, ok) - - assert.Equal(t, Instruction{ - EventID: 1, EventType: "comment.created", Trigger: "mentioned", Class: "internal", - Recording: InstructionRecording{ - BucketID: adapterBucketID, RecordingID: 10304028972, Type: "Comment", Title: "A comment", - URL: "https://app.basecamp.com/2914079/buckets/48699913/recordings/10304028972", - }, - ReplyTo: InstructionReply{Kind: "comment", RecordingID: 10304028989}, - RequesterID: adapterOperatorID, - Acknowledge: true, - Delivery: DeliveryExposed, - Content: "
please ask " + mentionMarkup(otherPersonID) + " about it
", - ContentUpdatedAt: time.Date(2026, 9, 16, 10, 0, 0, 0, time.UTC), - }, got) - - encoded, err := json.Marshal(got) - require.NoError(t, err) - var fields map[string]any - require.NoError(t, json.Unmarshal(encoded, &fields)) - keys := make([]string, 0, len(fields)) - for k := range fields { - keys = append(keys, k) - } - sort.Strings(keys) - assert.Equal(t, []string{"acknowledge", "class", "content", "content_updated_at", "delivery", "event_id", "event_type", - "guard_acknowledged", "recording", "reply_to", "requester_id", "trigger"}, keys) - assert.NotContains(t, string(encoded), "/work/connector", "no route") - assert.NotContains(t, string(encoded), f.grant.Token, "no token") -} - -func TestGetDispatchCancelsTheGuardAndReportsAFiredOne(t *testing.T) { - f := newDispatchFixture(t) - ctx := context.Background() - require.Equal(t, "armed", f.row(t, 1).Guard, "an acknowledged trigger arms the guard") - - got, _, err := f.d.Get(ctx, 1) - require.NoError(t, err) - assert.False(t, got.GuardAcknowledged) - assert.Equal(t, "canceled", f.row(t, 1).Guard) - - // The connector fired the guard on event 2 before the worker asked. - _, err = f.ledger.db.ExecContext(context.Background(), `UPDATE task_events SET guard = 'fired' WHERE task_id = ? AND event_id = 2`, f.grant.ID) - require.NoError(t, err) - got, _, err = f.d.Get(ctx, 2) - require.NoError(t, err) - assert.True(t, got.GuardAcknowledged) - assert.Equal(t, "fired", f.row(t, 2).Guard, "a fired guard stays fired") -} - -func TestGetDispatchCancelsAnArmedGuardOnAnAlreadyExposedEvent(t *testing.T) { - f := newDispatchFixture(t) - // Exposed at launch by the dispatcher, guard still armed. - _, err := f.ledger.db.ExecContext(context.Background(), `UPDATE task_events SET delivery = 'exposed' WHERE event_id = 1`) - require.NoError(t, err) - - _, _, err = f.d.Get(context.Background(), 1) - require.NoError(t, err) - assert.Equal(t, "canceled", f.row(t, 1).Guard) -} - -// Done when: ack_dispatch and complete_dispatch move the delivery state. -func TestAckAndCompleteMoveTheDelivery(t *testing.T) { - f := newDispatchFixture(t) - ctx := context.Background() - _, _, err := f.d.Get(ctx, 1) - require.NoError(t, err) - - ackID := int64(9001) - receipt, err := f.d.Ack(ctx, 1, &ackID) - require.NoError(t, err) - assert.Equal(t, Receipt{EventID: 1, Delivery: DeliveryDelivered, AckID: &ackID, Links: []string{}}, receipt) - delivered := f.row(t, 1) - assert.Equal(t, "delivered", delivered.Delivery) - require.NotNil(t, delivered.DeliveredAt) - - // A lost tool response, retried. - again, err := f.d.Ack(ctx, 1, &ackID) - require.NoError(t, err) - assert.Equal(t, receipt, again) - assert.Equal(t, delivered, f.row(t, 1)) - - replyID := int64(9002) - done, err := f.d.Complete(ctx, 1, Completion{Outcome: OutcomeSucceeded, Links: []string{"https://github.com/basecamp/basecamp-cli/pull/1"}, ReplyID: &replyID}) - require.NoError(t, err) - assert.Equal(t, DeliveryCompleted, done.Delivery) - assert.Equal(t, OutcomeSucceeded, done.Outcome) - assert.Equal(t, &replyID, done.ReplyID) - assert.Equal(t, &ackID, done.AckID) - assert.Equal(t, StateCompleted, getRecord(t, f.ledger, 1).State) - completed := f.row(t, 1) - assert.Equal(t, delivered.DeliveredAt, completed.DeliveredAt) - - repeat, err := f.d.Complete(ctx, 1, Completion{Outcome: OutcomeSucceeded, Links: []string{"https://github.com/basecamp/basecamp-cli/pull/1"}, ReplyID: &replyID}) - require.NoError(t, err) - assert.Equal(t, done, repeat) - assert.Equal(t, completed, f.row(t, 1)) -} - -func TestCompleteAlsoAcknowledges(t *testing.T) { - f := newDispatchFixture(t) - ctx := context.Background() - _, _, err := f.d.Get(ctx, 2) - require.NoError(t, err) - - receipt, err := f.d.Complete(ctx, 2, Completion{Outcome: OutcomeFailed}) - require.NoError(t, err) - assert.Equal(t, DeliveryCompleted, receipt.Delivery) - row := f.row(t, 2) - assert.NotNil(t, row.DeliveredAt) - assert.NotNil(t, row.CompletedAt) -} - -func TestAReportedOutcomeStands(t *testing.T) { - f := newDispatchFixture(t) - ctx := context.Background() - _, _, err := f.d.Get(ctx, 1) - require.NoError(t, err) - first := int64(1) - _, err = f.d.Ack(ctx, 1, &first) - require.NoError(t, err) - second := int64(2) - _, err = f.d.Ack(ctx, 1, &second) - assert.ErrorIs(t, err, ErrReportConflict) - - _, err = f.d.Complete(ctx, 1, Completion{Outcome: OutcomeSucceeded}) - require.NoError(t, err) - for _, c := range []Completion{ - {Outcome: OutcomeFailed}, - {Outcome: OutcomeSucceeded, Links: []string{"https://example.com/a"}}, - {Outcome: OutcomeSucceeded, ReplyID: &second}, - } { - _, err = f.d.Complete(ctx, 1, c) - assert.ErrorIs(t, err, ErrReportConflict) - } -} - -func TestAReportNeedsTheEventHandedOut(t *testing.T) { - f := newDispatchFixture(t) - ctx := context.Background() - - _, err := f.d.Ack(ctx, 2, nil) - assert.ErrorIs(t, err, ErrNotExposed) - _, err = f.d.Complete(ctx, 2, Completion{Outcome: OutcomeSucceeded}) - assert.ErrorIs(t, err, ErrNotExposed) - assert.Equal(t, "admitted", f.row(t, 2).Delivery) -} - -// Done when: a superseded token is refused. -func TestASupersededTokenIsRefused(t *testing.T) { - f := newDispatchFixture(t) - ctx := context.Background() - _, _, err := f.d.Get(ctx, 1) - require.NoError(t, err) - - require.NoError(t, f.ledger.SupersedeTask(ctx, f.grant.ID)) - - _, _, err = f.d.Get(ctx, 2) - assert.ErrorIs(t, err, ErrTaskTokenRefused) - _, err = f.d.Ack(ctx, 1, nil) - assert.ErrorIs(t, err, ErrTaskTokenRefused) - _, err = f.d.Complete(ctx, 1, Completion{Outcome: OutcomeSucceeded}) - assert.ErrorIs(t, err, ErrTaskTokenRefused) - assert.Equal(t, "admitted", f.row(t, 2).Delivery, "a refused get exposes nothing") - assert.Equal(t, "exposed", f.row(t, 1).Delivery) - - _, err = f.ledger.Dispatch(ctx, "not-a-token", adapterAgentID) - assert.ErrorIs(t, err, ErrTaskTokenRefused, "refused when bound, not only on use") - _, err = f.ledger.Dispatch(ctx, f.grant.Token, adapterAgentID) - assert.ErrorIs(t, err, ErrTaskTokenRefused) -} - -func TestAWorkerSeesOnlyItsOwnTask(t *testing.T) { - f := newDispatchFixture(t) - ctx := context.Background() - seenRecord(t, f.ledger, 3) - _, err := f.ledger.Admission().Commit(ctx, admittedVerdict(3, 0, "recording:other")) - require.NoError(t, err) - _, err = f.ledger.CreateTask(ctx, []int64{3}) - require.NoError(t, err) - - _, _, err = f.d.Get(ctx, 3) - assert.ErrorIs(t, err, ErrNotOnTask) - _, err = f.d.Ack(ctx, 3, nil) - assert.ErrorIs(t, err, ErrNotOnTask) - _, _, err = f.d.Get(ctx, 404) - assert.ErrorIs(t, err, ErrNotOnTask, "an unknown event reads the same as another task's") -} - -func TestAnEventThatLeftThePathIsNotHandedOut(t *testing.T) { - f := newDispatchFixture(t) - ctx := context.Background() - - // A record on a live task cannot be taken off the path around its task. - require.ErrorIs(t, f.ledger.SetState(ctx, 2, StateBlocked, "no_route"), ErrOnALiveTask) - - // It can be settled before a worker pulls it: finished work is not - // handed out for the first time. - require.NoError(t, f.ledger.SetState(ctx, 2, StateCompleted, "")) - _, _, err := f.d.Get(ctx, 2) - assert.ErrorIs(t, err, ErrNotDispatchable) - assert.Equal(t, "admitted", f.row(t, 2).Delivery) -} - -// Retention took the instruction: a completed event asked for again answers -// that it can no longer be dispatched, never an empty instruction. -func TestAnEventWhoseContentWasDroppedIsNotHandedOut(t *testing.T) { - f := newDispatchFixture(t) - ctx := context.Background() - at := time.Date(2026, 9, 17, 12, 0, 0, 0, time.UTC) - f.ledger.now = func() time.Time { return at } - _, _, err := f.d.Get(ctx, 1) - require.NoError(t, err) - _, err = f.d.Complete(ctx, 1, Completion{Outcome: OutcomeSucceeded}) - require.NoError(t, err) - dropped, err := f.ledger.DropContent(ctx, at.Add(time.Hour), at.Add(time.Hour)) - require.NoError(t, err) - require.Equal(t, 1, dropped) - - _, _, err = f.d.Get(ctx, 1) - assert.ErrorIs(t, err, ErrNotDispatchable) -} - -// A record is dispatched exactly while a live task carries it: joining a task -// moves it there, in the task's own transaction. -func TestCreateTaskDispatchesItsRecords(t *testing.T) { - f := newDispatchFixture(t) - assert.Equal(t, StateDispatched, getRecord(t, f.ledger, 1).State) - assert.Equal(t, StateDispatched, getRecord(t, f.ledger, 2).State, "the queued follow-up too") -} - -// An event is on at most one live task. A retried launch is refused and -// writes nothing; after a redispatch supersedes the task, it joins a new one. -func TestAnEventIsOnOneLiveTask(t *testing.T) { - f := newDispatchFixture(t) - ctx := context.Background() - countTasks := func() int { - var n int - require.NoError(t, f.ledger.db.QueryRowContext(ctx, `SELECT COUNT(*) FROM tasks`).Scan(&n)) - return n - } - - _, err := f.ledger.CreateTask(ctx, []int64{1}) - assert.ErrorIs(t, err, ErrEventOnLiveTask) - _, err = f.ledger.CreateTask(ctx, []int64{2, 1}) - assert.ErrorIs(t, err, ErrEventOnLiveTask) - assert.Equal(t, 1, countTasks(), "a refused task leaves no task behind") - - // The database refuses it too, whoever writes. - _, err = f.ledger.db.ExecContext(ctx, `INSERT INTO tasks (id, token_sha256, created_at) VALUES (99, 'x', 'now')`) - require.NoError(t, err) - _, err = f.ledger.db.ExecContext(ctx, `INSERT INTO task_events (task_id, event_id) VALUES (99, 1)`) - require.Error(t, err) - - require.NoError(t, f.ledger.SupersedeTask(ctx, f.grant.ID)) - grant, err := f.ledger.CreateTask(ctx, []int64{1, 2}) - require.NoError(t, err) - d, err := f.ledger.Dispatch(ctx, grant.Token, adapterAgentID) - require.NoError(t, err) - got, ok, err := d.Get(ctx, 0) - require.NoError(t, err) - require.True(t, ok) - assert.Equal(t, int64(1), got.EventID) - _, _, err = f.d.Get(ctx, 1) - assert.ErrorIs(t, err, ErrTaskTokenRefused) -} - -// The dispatcher writes a task with its attempt and exposure in one commit; -// a task created in a transaction that rolls back leaves nothing, and does not -// hold the event. -func TestCreateTaskInsideACallersTransaction(t *testing.T) { - ledger := newTestLedger(t) - ctx := context.Background() - seenRecord(t, ledger, 1) - _, err := ledger.Admission().Commit(ctx, admittedVerdict(1, 0, "recording:9")) - require.NoError(t, err) - - tx, err := ledger.db.BeginTx(ctx, nil) - require.NoError(t, err) - _, err = ledger.createTask(ctx, tx, []int64{1}) - require.NoError(t, err) - require.NoError(t, tx.Rollback()) - assert.Equal(t, StateAdmitted, getRecord(t, ledger, 1).State) - - tx, err = ledger.db.BeginTx(ctx, nil) - require.NoError(t, err) - _, err = ledger.createTask(ctx, tx, []int64{1}) - require.NoError(t, err) - _, err = ledger.createTask(ctx, tx, []int64{1}) - require.ErrorIs(t, err, ErrEventOnLiveTask, "one live task per event within a transaction too") - require.NoError(t, tx.Rollback()) - - tx, err = ledger.db.BeginTx(ctx, nil) - require.NoError(t, err) - grant, err := ledger.createTask(ctx, tx, []int64{1}) - require.NoError(t, err) - require.NoError(t, tx.Commit()) - _, err = ledger.Dispatch(ctx, grant.Token, adapterAgentID) - require.NoError(t, err) - assert.Equal(t, StateDispatched, getRecord(t, ledger, 1).State) -} - -// A task is only ever made of instructions a worker can pull. A record that -// lost its snapshot on the way through blocked is refused, not dispatched as -// an empty task. -func TestCreateTaskRefusesARecordWithoutItsInstruction(t *testing.T) { - f := newDispatchFixture(t) - ctx := context.Background() - require.NoError(t, f.ledger.SupersedeTask(ctx, f.grant.ID)) - require.NoError(t, f.ledger.SetState(ctx, 1, StateBlocked, "read_failed"), "never handed, it left dispatched with its task") - require.NoError(t, f.ledger.SetState(ctx, 1, StateAdmitted, "")) - - _, err := f.ledger.CreateTask(ctx, []int64{1}) - require.ErrorIs(t, err, ErrNotDispatchable) - assert.Equal(t, StateAdmitted, getRecord(t, f.ledger, 1).State) - - _, err = f.ledger.CreateTask(ctx, []int64{2, 2}) - require.Error(t, err) - assert.NotErrorIs(t, err, ErrEventOnLiveTask, "a duplicate id is not another task's event") -} - -// Superseding a task returns what it never exposed to admitted and leaves -// what a worker was handed dispatched, so no record is left dispatched on no -// task without a worker having seen it. -func TestSupersedingReturnsUnexposedWork(t *testing.T) { - f := newDispatchFixture(t) - ctx := context.Background() - _, _, err := f.d.Get(ctx, 1) - require.NoError(t, err) - - require.NoError(t, f.ledger.SupersedeTask(ctx, f.grant.ID)) - - assert.Equal(t, StateDispatched, getRecord(t, f.ledger, 1).State, "a worker saw it") - assert.Equal(t, StateAdmitted, getRecord(t, f.ledger, 2).State, "never exposed, it is work again") - require.NoError(t, f.ledger.SupersedeTask(ctx, f.grant.ID), "a repeat is harmless") - assert.Equal(t, StateAdmitted, getRecord(t, f.ledger, 2).State) - - // The conversation is not free while event 1 waits for settlement. - _, err = f.ledger.CreateTask(ctx, []int64{2}) - require.ErrorIs(t, err, ErrConversationBusy) - require.NoError(t, f.ledger.SetState(ctx, 1, StateCompleted, "")) - grant, err := f.ledger.CreateTask(ctx, []int64{2}) - require.NoError(t, err) - d, err := f.ledger.Dispatch(ctx, grant.Token, adapterAgentID) - require.NoError(t, err) - got, ok, err := d.Get(ctx, 0) - require.NoError(t, err) - require.True(t, ok) - assert.Equal(t, int64(2), got.EventID) -} - -// A conversation has one task at a time. Siblings a supersede returned wait -// behind the exposed event a worker was handed, and two of them are never -// launched as two tasks. -func TestAConversationHasOneTaskAtATime(t *testing.T) { - ledger := newTestLedger(t) - ctx := context.Background() - const key = "recording:10304028989" - for _, id := range []int64{1, 2, 3} { - seenRecord(t, ledger, id) - _, err := ledger.Admission().Commit(ctx, admittedVerdict(id, 0, key)) - require.NoError(t, err) - } - grant, err := ledger.CreateTask(ctx, []int64{1, 2, 3}) - require.NoError(t, err) - d, err := ledger.Dispatch(ctx, grant.Token, adapterAgentID) - require.NoError(t, err) - _, _, err = d.Get(ctx, 1) - require.NoError(t, err) - require.NoError(t, ledger.SupersedeTask(ctx, grant.ID)) - - _, err = ledger.CreateTask(ctx, []int64{2}) - require.ErrorIs(t, err, ErrConversationBusy, "event 1 was handed to a worker and is not settled") - _, err = ledger.CreateTask(ctx, []int64{2, 3}) - require.ErrorIs(t, err, ErrConversationBusy) - assert.Equal(t, StateAdmitted, getRecord(t, ledger, 2).State, "a refused task moves nothing") - - // A redispatch that takes the whole conversation is one task. - _, err = ledger.CreateTask(ctx, []int64{1, 2, 3}) - require.NoError(t, err) - - // Two separate launches on a free conversation: the second is refused. - other := newTestLedger(t) - for _, id := range []int64{1, 2} { - seenRecord(t, other, id) - _, err := other.Admission().Commit(ctx, admittedVerdict(id, 0, key)) - require.NoError(t, err) - } - _, err = other.CreateTask(ctx, []int64{1}) - require.NoError(t, err) - _, err = other.CreateTask(ctx, []int64{2}) - require.ErrorIs(t, err, ErrConversationBusy) -} - -// Asked for by id or as the earliest, an event is served by one rule. -func TestTheEarliestAndAnExplicitGetAgree(t *testing.T) { - f := newDispatchFixture(t) - ctx := context.Background() - _, _, err := f.d.Get(ctx, 1) - require.NoError(t, err) - // Settled elsewhere while the worker had it, before it acknowledged. - require.NoError(t, f.ledger.SetState(ctx, 1, StateCompleted, "")) - - byID, ok, err := f.d.Get(ctx, 1) - require.NoError(t, err) - require.True(t, ok) - earliest, ok, err := f.d.Get(ctx, 0) - require.NoError(t, err) - require.True(t, ok) - assert.Equal(t, byID, earliest) -} - -// Finished work is never handed out for the first time: a record completed -// before this worker was exposed to it is refused, and one it completed -// itself is served again. -func TestCompletedWorkIsServedOnlyToItsWorker(t *testing.T) { - f := newDispatchFixture(t) - ctx := context.Background() - require.NoError(t, f.ledger.SetState(ctx, 2, StateCompleted, "")) - - _, _, err := f.d.Get(ctx, 2) - assert.ErrorIs(t, err, ErrNotDispatchable) - assert.Equal(t, "admitted", f.row(t, 2).Delivery) - - _, _, err = f.d.Get(ctx, 1) - require.NoError(t, err) - _, err = f.d.Complete(ctx, 1, Completion{Outcome: OutcomeSucceeded}) - require.NoError(t, err) - got, ok, err := f.d.Get(ctx, 1) - require.NoError(t, err) - require.True(t, ok) - assert.Equal(t, DeliveryCompleted, got.Delivery) -} - -func TestConcurrentLaunchesOfOneEventMakeOneTask(t *testing.T) { - path := filepath.Join(t.TempDir(), "state", "connector.db") - ledger, err := OpenLedger(path) - require.NoError(t, err) - t.Cleanup(func() { _ = ledger.Close() }) - ctx := context.Background() - seenRecord(t, ledger, 1) - _, err = ledger.Admission().Commit(ctx, admittedVerdict(1, 0, "recording:9")) - require.NoError(t, err) - - // Each racer has its own handle on the file, so SQLite sees the race - // rather than database/sql's one connection serializing it. - const racers = 8 - errs := make([]error, racers) - done := make(chan struct{}) - for i := range racers { - go func() { - defer func() { done <- struct{}{} }() - handle, err := OpenExistingLedger(ctx, path) - if err != nil { - errs[i] = err - return - } - defer handle.Close() - _, errs[i] = handle.CreateTask(ctx, []int64{1}) - }() - } - for range racers { - <-done - } - created := 0 - for _, err := range errs { - if err == nil { - created++ - continue - } - assert.ErrorIs(t, err, ErrEventOnLiveTask) - } - assert.Equal(t, 1, created) - var live int - require.NoError(t, ledger.db.QueryRowContext(ctx, `SELECT COUNT(*) FROM task_events WHERE event_id = 1 AND retired_at IS NULL`).Scan(&live)) - assert.Equal(t, 1, live) -} - -// One event that left the path never hides the rest of the task. -func TestTheEarliestSkipsAnEventThatLeftThePath(t *testing.T) { - f := newDispatchFixture(t) - ctx := context.Background() - require.NoError(t, f.ledger.SetState(ctx, 1, StateCompleted, ""), "settled before any worker pulled it") - - got, ok, err := f.d.Get(ctx, 0) - require.NoError(t, err) - require.True(t, ok) - assert.Equal(t, int64(2), got.EventID) -} - -// A worker's report is what it did: a record the dispatcher settled as -// completed while the worker was still going keeps its settlement, and the -// report the worker then sends is recorded against it. -func TestAReportIsRecordedAfterSettlement(t *testing.T) { - f := newDispatchFixture(t) - ctx := context.Background() - _, _, err := f.d.Get(ctx, 2) - require.NoError(t, err) - require.NoError(t, f.ledger.SetState(ctx, 2, StateCompleted, "")) - - _, err = f.d.Ack(ctx, 2, nil) - require.NoError(t, err) - receipt, err := f.d.Complete(ctx, 2, Completion{Outcome: OutcomeFailed}) - require.NoError(t, err) - assert.Equal(t, DeliveryCompleted, receipt.Delivery) - assert.Equal(t, OutcomeFailed, receipt.Outcome) - assert.Equal(t, StateCompleted, getRecord(t, f.ledger, 2).State) -} - // Invariant 4: nothing leaves dispatched while a worker may still act on it. -// A handed record cannot be withdrawn, blocked or requeued — not through the -// ledger's write, and not around it — so its conversation stays busy and no -// sibling starts a second task until its outcome is in. +// A record an older build handed to a worker cannot be withdrawn, blocked or +// requeued, not through the ledger's write and not around it, until its +// outcome is in, even after its task was superseded. func TestAHandedRecordLeavesDispatchedOnlyWhenCompleted(t *testing.T) { - f := newDispatchFixture(t) + l := newTestLedger(t) ctx := context.Background() - _, _, err := f.d.Get(ctx, 1) - require.NoError(t, err) - require.NoError(t, f.ledger.SupersedeTask(ctx, f.grant.ID), "superseding does not release what a worker holds") + opAdmit(t, l, 1, "recording:10304028989") + task := olderLaunch(t, l, 1) + olderSupersede(t, l, task.TaskID) + require.Equal(t, StateDispatched, getRecord(t, l, 1).State, "superseding does not release what a worker holds") for _, to := range []struct { state RecordState reason string }{{StateAdmitted, ""}, {StateBlocked, "read_failed"}} { - err := f.ledger.SetState(ctx, 1, to.state, to.reason) + err := l.SetState(ctx, 1, to.state, to.reason) require.ErrorIs(t, err, ErrHeldByWorker, "to %s", to.state) - _, err = f.ledger.db.ExecContext(ctx, `UPDATE events SET state = ? WHERE id = 1`, string(to.state)) + _, err = l.db.ExecContext(ctx, `UPDATE events SET state = ? WHERE id = 1`, string(to.state)) require.Error(t, err, "the database refuses it too") } - assert.Equal(t, StateDispatched, getRecord(t, f.ledger, 1).State) - _, err = f.ledger.CreateTask(ctx, []int64{2}) - require.ErrorIs(t, err, ErrConversationBusy, "the sibling waits for the handed event") + assert.Equal(t, StateDispatched, getRecord(t, l, 1).State) - require.NoError(t, f.ledger.SetState(ctx, 1, StateCompleted, "")) - _, err = f.ledger.CreateTask(ctx, []int64{2}) - require.NoError(t, err) -} - -// A worker's open never leaves a ledger behind where there was none: not -// through the privacy check, and not through SQLite, whichever the file -// disappears before. -func TestOpenExistingLedgerCreatesNothing(t *testing.T) { - dir := filepath.Join(t.TempDir(), "state") - require.NoError(t, os.Mkdir(dir, 0o700)) - path := filepath.Join(dir, LedgerFile) - - _, err := OpenExistingLedger(context.Background(), path) - require.ErrorIs(t, err, os.ErrNotExist) - entries, err := os.ReadDir(dir) - require.NoError(t, err) - assert.Empty(t, entries, "the privacy check created nothing") - - // The file gone after the check: SQLite itself must refuse. - db, err := sql.Open("sqlite", ledgerDSN(path, false)) - require.NoError(t, err) - defer db.Close() - require.Error(t, db.PingContext(context.Background())) - entries, err = os.ReadDir(dir) - require.NoError(t, err) - assert.Empty(t, entries, "SQLite created nothing") - - owner, err := sql.Open("sqlite", ledgerDSN(path, true)) - require.NoError(t, err) - defer owner.Close() - require.NoError(t, owner.PingContext(context.Background()), "the connector's own open still creates") -} - -func TestOpenExistingLedgerNeverCreatesOrMigrates(t *testing.T) { - dir := t.TempDir() + "/state" - _, err := OpenExistingLedger(context.Background(), dir+"/missing.db") - require.Error(t, err) - - path := dir + "/connector.db" - all := migrations - migrations = all[:4] - old, err := OpenLedger(path) - migrations = all - require.NoError(t, err) - require.NoError(t, old.Close()) - - _, err = OpenExistingLedger(context.Background(), path) - require.ErrorIs(t, err, ErrLedgerSchema) - again, err := OpenLedger(path) - require.NoError(t, err) - version, err := again.SchemaVersion(context.Background()) - require.NoError(t, err) - require.NoError(t, again.Close()) - assert.Equal(t, len(migrations), version, "migrated only by the owner's open") - - current, err := OpenExistingLedger(context.Background(), path) - require.NoError(t, err) - require.NoError(t, current.Close()) -} - -func TestDeliveryNeverGoesBack(t *testing.T) { - f := newDispatchFixture(t) - _, _, err := f.d.Get(context.Background(), 1) - require.NoError(t, err) - - _, err = f.ledger.db.ExecContext(context.Background(), `UPDATE task_events SET delivery = 'admitted' WHERE event_id = 1`) - require.Error(t, err) - assert.Contains(t, err.Error(), "never goes back") -} - -func TestCreateTaskTakesOnlyWorkWaitingForAWorker(t *testing.T) { - ledger := newTestLedger(t) - ctx := context.Background() - seenRecord(t, ledger, 1) - - _, err := ledger.CreateTask(ctx, []int64{1}) - require.Error(t, err) - _, err = ledger.CreateTask(ctx, []int64{404}) - assert.ErrorIs(t, err, ErrNoSuchRecord) - _, err = ledger.CreateTask(ctx, nil) - require.Error(t, err) - var tasks int - require.NoError(t, ledger.db.QueryRowContext(context.Background(), `SELECT COUNT(*) FROM tasks`).Scan(&tasks)) - assert.Zero(t, tasks, "a refused task leaves nothing behind") - - blocked := blockedVerdict(1, 0, admission.ReasonNoRoute) - _, err = ledger.Admission().Commit(ctx, blocked) - require.NoError(t, err) - _, err = ledger.CreateTask(ctx, []int64{1}) - require.Error(t, err) -} - -func TestTheTokenIsStoredOnlyAsAHash(t *testing.T) { - f := newDispatchFixture(t) - var stored string - require.NoError(t, f.ledger.db.QueryRowContext(context.Background(), `SELECT token_sha256 FROM tasks WHERE id = ?`, f.grant.ID).Scan(&stored)) - assert.NotContains(t, stored, f.grant.Token) - assert.Len(t, stored, 64) - assert.GreaterOrEqual(t, len(f.grant.Token), 43, "32 random bytes") -} - -func TestCompleteRefusesMalformedReports(t *testing.T) { - f := newDispatchFixture(t) - ctx := context.Background() - _, _, err := f.d.Get(ctx, 1) - require.NoError(t, err) - - tooMany := make([]string, maxCompletionLinks+1) - for i := range tooMany { - tooMany[i] = "https://example.com/" - } - for name, c := range map[string]Completion{ - "no outcome": {}, - "unknown outcome": {Outcome: "unknown"}, - "not a URL": {Outcome: OutcomeSucceeded, Links: []string{"javascript:alert(1)"}}, - "too many links": {Outcome: OutcomeSucceeded, Links: tooMany}, - "a link too long": {Outcome: OutcomeSucceeded, Links: []string{"https://example.com/" + strings.Repeat("a", maxLinkLength)}}, - } { - t.Run(name, func(t *testing.T) { - _, err := f.d.Complete(ctx, 1, c) - require.ErrorIs(t, err, ErrInvalidReport) - assert.NotContains(t, err.Error(), "report on event", "what the worker got wrong reaches it without the ledger's own wrapping") - assert.Equal(t, "exposed", f.rowContext(ctx, t, 1).Delivery) - }) - } + require.NoError(t, l.SetState(ctx, 1, StateCompleted, "")) } func TestStripMentionsOf(t *testing.T) { @@ -839,532 +81,3 @@ func entityEncodedSGID(mention string) string { i := strings.Index(mention, `sgid="`) + len(`sgid="`) return mention[:i] + fmt.Sprintf("&#x%x;", mention[i]) + mention[i+1:] } - -func TestResolveStateDirAcceptsOnlyTheCanonicalDirectory(t *testing.T) { - home := t.TempDir() - t.Setenv("XDG_STATE_HOME", home) - root, err := StateRoot() - require.NoError(t, err) - assert.Equal(t, filepath.Join(home, "basecamp", "connect"), root) - canonical := filepath.Join(root, StateDirName("999", adapterAgentID)) - - resolved, agentID, err := ResolveStateDir(canonical, "999") - require.NoError(t, err) - assert.Equal(t, adapterAgentID, agentID) - assert.Equal(t, canonical, resolved) - resolved, agentID, err = ResolveStateDir(canonical+"/../"+StateDirName("999", adapterAgentID)+"/", "0999") - require.NoError(t, err, "accounts compare as numbers, and a trailing slash or a .. is the same directory") - assert.Equal(t, adapterAgentID, agentID) - assert.Equal(t, canonical, resolved, "the directory every caller goes on to use is the one that was checked") - - for name, dir := range map[string]string{ - "outside the root": filepath.Join(t.TempDir(), StateDirName("999", adapterAgentID)), - "nested in the root": filepath.Join(root, "x", StateDirName("999", adapterAgentID)), - "another account": filepath.Join(root, StateDirName("1000", adapterAgentID)), - "no agent": filepath.Join(root, "999-"), - "no account": filepath.Join(root, "-52007412"), - "not a number": filepath.Join(root, "999-abc"), - "a signed agent": filepath.Join(root, "999-+52007412"), - "a padded agent": filepath.Join(root, "999-052007412"), - "the root itself": root, - "above the root": filepath.Join(root, "..", StateDirName("999", adapterAgentID)), - } { - t.Run(name, func(t *testing.T) { - _, _, err := ResolveStateDir(dir, "999") - assert.ErrorIs(t, err, ErrNotAStateDir) - var refusal *StateDirError - require.ErrorAs(t, err, &refusal, "the refusal says why in fields, not in a message to be parsed") - assert.Equal(t, root, refusal.Root) - assert.Equal(t, "999", refusal.Want) - if name == "another account" { - assert.Equal(t, StateDirOtherAccount, refusal.Why) - assert.Equal(t, "1000", refusal.Account) - } - if name == "outside the root" { - assert.Equal(t, StateDirElsewhere, refusal.Why) - } - }) - } - - t.Setenv("XDG_STATE_HOME", "relative/state") - root, err = StateRoot() - require.NoError(t, err) - assert.True(t, filepath.IsAbs(root), "a relative XDG_STATE_HOME is ignored, as the specification says") -} - -// The ledger is free while a call builds its answer: every transaction here -// takes the write lock as it opens, and decoding the snapshot and stripping -// the agent's mention must not be done holding it. -func TestGetDispatchDoesNotHoldTheLedgerWhileItBuildsItsAnswer(t *testing.T) { - f := newDispatchFixture(t) - ctx := context.Background() - other, err := OpenExistingLedger(ctx, f.path) - require.NoError(t, err) - t.Cleanup(func() { _ = other.Close() }) - - writes := 0 - f.d.afterTx = func() { - // Intake, writing while the worker builds its instruction. It waits - // for the write lock, so a transaction still open here fails this - // (after the busy timeout) rather than deadlocking. - writes++ - fresh, err := other.RecordSeen(ctx, testEvent(int64(100+writes)), LanePoll) - require.NoError(t, err) - require.True(t, fresh) - } - - // The call that writes (exposure), and the repeat that writes nothing. - _, _, err = f.d.Get(ctx, 1) - require.NoError(t, err) - _, _, err = f.d.Get(ctx, 1) - require.NoError(t, err) - assert.Equal(t, 2, writes) -} - -// Acknowledgements and completions free the ledger before building their -// receipt too — on the retry that writes nothing as much as on the first. -func TestReportsDoNotHoldTheLedgerWhileTheyBuildTheirReceipt(t *testing.T) { - f := newDispatchFixture(t) - ctx := context.Background() - other, err := OpenExistingLedger(ctx, f.path) - require.NoError(t, err) - t.Cleanup(func() { _ = other.Close() }) - _, _, err = f.d.Get(ctx, 1) - require.NoError(t, err) - - writes := 0 - f.d.afterTx = func() { - writes++ - fresh, err := other.RecordSeen(ctx, testEvent(int64(200+writes)), LanePoll) - require.NoError(t, err) - require.True(t, fresh) - } - for range 2 { - _, err = f.d.Ack(ctx, 1, nil) - require.NoError(t, err) - } - for range 2 { - _, err = f.d.Complete(ctx, 1, Completion{Outcome: OutcomeSucceeded}) - require.NoError(t, err) - } - assert.Equal(t, 4, writes) -} - -// The spec's automatic retry, as a transition table of its own: the -// dispatcher writes the originating event exposed at launch; the spawn is -// proven to have failed before any worker process existed; the exposure is -// withdrawn and the event launched again, once — and after a second failure it -// is blocked. Without the withdrawal marker, an exposure holds the record in -// dispatched and neither the retry nor the block is possible. -func TestASpawnThatFailedIsRetriedOnceThenBlocked(t *testing.T) { - f := newDispatchFixture(t) - ctx := context.Background() - exposeAtLaunch := func(taskID int64) { - t.Helper() - _, err := f.ledger.db.ExecContext(ctx, `UPDATE task_events SET delivery = 'exposed', exposed_at = 'launch' WHERE task_id = ? AND event_id = 1`, taskID) - require.NoError(t, err) - } - exposeAtLaunch(f.grant.ID) - - // First failure: supersede, withdraw, launch again — one transaction. - tx, err := f.ledger.db.BeginTx(ctx, nil) - require.NoError(t, err) - require.NoError(t, f.ledger.supersedeTask(ctx, tx, f.grant.ID)) - require.NoError(t, f.ledger.withdrawExposure(ctx, tx, f.grant.ID, 1, StateAdmitted, "")) - retry, err := f.ledger.createTask(ctx, tx, []int64{1, 2}) - require.NoError(t, err) - require.NoError(t, tx.Commit()) - var onRetry int - require.NoError(t, f.ledger.db.QueryRowContext(ctx, `SELECT COUNT(*) FROM task_events WHERE task_id = ? AND event_id = 1 AND retired_at IS NULL`, retry.ID).Scan(&onRetry)) - assert.Equal(t, 1, onRetry, "the retry carries the originating event again") - assert.Equal(t, StateDispatched, getRecord(t, f.ledger, 1).State) - exposeAtLaunch(retry.ID) - - // Second failure: supersede, withdraw, block. - tx, err = f.ledger.db.BeginTx(ctx, nil) - require.NoError(t, err) - require.NoError(t, f.ledger.supersedeTask(ctx, tx, retry.ID)) - require.NoError(t, f.ledger.withdrawExposure(ctx, tx, retry.ID, 1, StateBlocked, "spawn_failed")) - require.NoError(t, tx.Commit()) - record := getRecord(t, f.ledger, 1) - assert.Equal(t, StateBlocked, record.State) - assert.Equal(t, "spawn_failed", record.Reason) -} - -// What the database can see of "no worker existed", it holds: a withdrawal is -// refused once a worker pulled the instruction, and refused while a live task -// carries the event — so the only order is supersede, withdraw, create. -func TestAWithdrawalIsRefusedWhenAWorkerCouldHaveTheInstruction(t *testing.T) { - t.Run("a worker pulled it", func(t *testing.T) { - f := newDispatchFixture(t) - ctx := context.Background() - _, err := f.ledger.db.ExecContext(ctx, `UPDATE task_events SET delivery = 'exposed' WHERE event_id = 1`) - require.NoError(t, err) - _, _, err = f.d.Get(ctx, 1) - require.NoError(t, err, "the worker pulls an event exposed at launch") - - tx, err := f.ledger.db.BeginTx(ctx, nil) - require.NoError(t, err) - defer func() { _ = tx.Rollback() }() - require.NoError(t, f.ledger.supersedeTask(ctx, tx, f.grant.ID)) - require.Error(t, f.ledger.withdrawExposure(ctx, tx, f.grant.ID, 1, StateAdmitted, "")) - }) - - t.Run("a live task already carries it", func(t *testing.T) { - f := newDispatchFixture(t) - ctx := context.Background() - _, err := f.ledger.db.ExecContext(ctx, `UPDATE task_events SET delivery = 'exposed' WHERE event_id = 1`) - require.NoError(t, err) - - tx, err := f.ledger.db.BeginTx(ctx, nil) - require.NoError(t, err) - defer func() { _ = tx.Rollback() }() - require.NoError(t, f.ledger.supersedeTask(ctx, tx, f.grant.ID)) - _, err = f.ledger.createTask(ctx, tx, []int64{1, 2}) - require.NoError(t, err) - // The database refuses it whoever writes, and on an untouched row. - _, err = tx.ExecContext(ctx, `UPDATE task_events SET withdrawn_at = 'raw' WHERE task_id = ? AND event_id = 1`, f.grant.ID) - require.Error(t, err) - require.Error(t, f.ledger.withdrawExposure(ctx, tx, f.grant.ID, 1, StateAdmitted, ""), "create before withdraw is the wrong order") - }) - - t.Run("a pull is recorded once, and a repeat writes nothing", func(t *testing.T) { - f := newDispatchFixture(t) - ctx := context.Background() - _, _, err := f.d.Get(ctx, 1) - require.NoError(t, err) - var first string - require.NoError(t, f.ledger.db.QueryRowContext(ctx, `SELECT pulled_at FROM task_events WHERE event_id = 1`).Scan(&first)) - _, _, err = f.d.Get(ctx, 1) - require.NoError(t, err) - var again string - require.NoError(t, f.ledger.db.QueryRowContext(ctx, `SELECT pulled_at FROM task_events WHERE event_id = 1`).Scan(&again)) - assert.Equal(t, first, again) - _, err = f.ledger.db.ExecContext(ctx, `UPDATE task_events SET pulled_at = 'later' WHERE event_id = 1`) - require.Error(t, err) - }) -} - -// A worker whose task was superseded is told so, whatever it sends: the token -// is checked before the report is. -func TestASupersededWorkerIsRefusedBeforeItsReportIsRead(t *testing.T) { - f := newDispatchFixture(t) - ctx := context.Background() - _, _, err := f.d.Get(ctx, 1) - require.NoError(t, err) - require.NoError(t, f.ledger.SupersedeTask(ctx, f.grant.ID)) - - for name, c := range map[string]Completion{ - "no outcome": {}, - "not a URL": {Outcome: OutcomeSucceeded, Links: []string{"javascript:alert(1)"}}, - } { - t.Run(name, func(t *testing.T) { - _, err := f.d.Complete(ctx, 1, c) - require.ErrorIs(t, err, ErrTaskTokenRefused) - assert.NotErrorIs(t, err, ErrInvalidReport) - }) - } -} - -// dispatchForTest dispatches a record the only way a record is dispatched: on -// a task. A record a test walked to admitted by hand has no instruction, so one -// is attached first; a seen record is admitted first. -func dispatchForTest(t *testing.T, ledger *Ledger, id int64) TaskGrant { - t.Helper() - ctx := context.Background() - _, err := ledger.db.ExecContext(ctx, `UPDATE events -SET snapshot = COALESCE(snapshot, CAST('{"content":"do it"}' AS BLOB)), - conversation_key = CASE WHEN conversation_key = '' THEN 'recording:' || id ELSE conversation_key END -WHERE id = ?`, id) - require.NoError(t, err) - if getRecord(t, ledger, id).State == StateSeen { - require.NoError(t, ledger.SetState(ctx, id, StateAdmitted, "")) - } - grant, err := ledger.CreateTask(ctx, []int64{id}) - require.NoError(t, err) - return grant -} - -// createTask writes nothing when it refuses: the caller's transaction is as -// it found it, whatever the caller does with it next. -func TestCreateTaskWritesNothingWhenItRefuses(t *testing.T) { - f := newDispatchFixture(t) - ctx := context.Background() - seenRecord(t, f.ledger, 3) - _, err := f.ledger.Admission().Commit(ctx, admittedVerdict(3, 0, "recording:3")) - require.NoError(t, err) - dispatchForTest(t, f.ledger, 3) - require.NoError(t, f.ledger.SetState(ctx, 3, StateCompleted, ""), "settled, and its snapshot is still there") - - var tasksBefore, rowsBefore int - require.NoError(t, f.ledger.db.QueryRowContext(ctx, `SELECT COUNT(*) FROM tasks`).Scan(&tasksBefore)) - require.NoError(t, f.ledger.db.QueryRowContext(ctx, `SELECT COUNT(*) FROM task_events`).Scan(&rowsBefore)) - - tx, err := f.ledger.db.BeginTx(ctx, nil) - require.NoError(t, err) - _, err = f.ledger.createTask(ctx, tx, []int64{3}) - require.Error(t, err) - assert.Contains(t, err.Error(), "completed") - var tasksAfter, rowsAfter int - require.NoError(t, tx.QueryRowContext(ctx, `SELECT COUNT(*) FROM tasks`).Scan(&tasksAfter)) - require.NoError(t, tx.QueryRowContext(ctx, `SELECT COUNT(*) FROM task_events`).Scan(&rowsAfter)) - require.NoError(t, tx.Rollback()) - assert.Equal(t, tasksBefore, tasksAfter, "no task row") - assert.Equal(t, rowsBefore, rowsAfter, "no task event row") -} - -// Exposure written at launch is the dispatcher's word, not a worker's pull. -// Until the worker pulls, finished work is not served to it, and it can -// neither acknowledge nor complete anything. -func TestALaunchExposureIsNotAPull(t *testing.T) { - f := newDispatchFixture(t) - ctx := context.Background() - exposeAtLaunch := func(id int64) { - t.Helper() - _, err := f.ledger.db.ExecContext(ctx, `UPDATE task_events SET delivery = 'exposed', exposed_at = 'launch' WHERE event_id = ?`, id) - require.NoError(t, err) - } - exposeAtLaunch(1) - exposeAtLaunch(2) - - _, err := f.d.Ack(ctx, 1, nil) - assert.ErrorIs(t, err, ErrNotExposed, "nothing was pulled yet") - _, err = f.d.Complete(ctx, 1, Completion{Outcome: OutcomeSucceeded}) - assert.ErrorIs(t, err, ErrNotExposed) - - // Settled before the worker ever pulled it: not served, and not the - // earliest either. - require.NoError(t, f.ledger.SetState(ctx, 1, StateCompleted, "")) - _, _, err = f.d.Get(ctx, 1) - assert.ErrorIs(t, err, ErrNotDispatchable) - got, ok, err := f.d.Get(ctx, 0) - require.NoError(t, err) - require.True(t, ok) - assert.Equal(t, int64(2), got.EventID) - - // Event 2 the worker did pull, just now. Settled after that, it is served - // again to the worker that has it, and its report is taken. - require.NoError(t, f.ledger.SetState(ctx, 2, StateCompleted, "")) - again, ok, err := f.d.Get(ctx, 2) - require.NoError(t, err) - require.True(t, ok) - assert.Equal(t, int64(2), again.EventID) - _, err = f.d.Ack(ctx, 2, nil) - require.NoError(t, err) -} - -// A task is live until it is superseded, whatever became of its records, so a -// conversation whose only event was settled before the worker pulled it is -// still busy. -func TestALiveTaskKeepsItsConversationBusy(t *testing.T) { - f := newDispatchFixture(t) - ctx := context.Background() - require.NoError(t, f.ledger.SetState(ctx, 1, StateCompleted, "")) - require.NoError(t, f.ledger.SetState(ctx, 2, StateCompleted, "")) - seenRecord(t, f.ledger, 3) - _, err := f.ledger.Admission().Commit(ctx, admittedVerdict(3, 0, "recording:10304028989")) - require.NoError(t, err) - - _, err = f.ledger.CreateTask(ctx, []int64{3}) - require.ErrorIs(t, err, ErrConversationBusy, "the first task is still live") - - require.NoError(t, f.ledger.SupersedeTask(ctx, f.grant.ID)) - _, err = f.ledger.CreateTask(ctx, []int64{3}) - require.NoError(t, err) -} - -// A task is made of work waiting for a worker, on a live task, and its rows -// stay where they were written — the database says so too. -func TestTheDatabaseRefusesAttachingWorkToTheWrongTask(t *testing.T) { - f := newDispatchFixture(t) - ctx := context.Background() - seenRecord(t, f.ledger, 3) - - _, err := f.ledger.db.ExecContext(ctx, `INSERT INTO task_events (task_id, event_id) VALUES (?, 3)`, f.grant.ID) - require.Error(t, err, "a seen record is not work waiting for a worker") - - other, err := f.ledger.CreateTask(ctx, []int64{}) - require.Error(t, err) - require.NoError(t, f.ledger.SupersedeTask(ctx, f.grant.ID)) - _, err = f.ledger.Admission().Commit(ctx, admittedVerdict(3, 0, "recording:3")) - require.NoError(t, err) - _, err = f.ledger.db.ExecContext(ctx, `INSERT INTO task_events (task_id, event_id) VALUES (?, 3)`, f.grant.ID) - require.Error(t, err, "a superseded task takes no new work") - _ = other - - _, err = f.ledger.db.ExecContext(ctx, `UPDATE task_events SET task_id = 99 WHERE event_id = 1`) - require.Error(t, err, "a task event does not move between tasks") - _, err = f.ledger.db.ExecContext(ctx, `UPDATE task_events SET event_id = 3 WHERE event_id = 1`) - require.Error(t, err, "nor between events") -} - -// An instruction is content, not an empty blob: a record with one would be -// dispatched and never servable. -func TestCreateTaskRefusesAnEmptyInstruction(t *testing.T) { - f := newDispatchFixture(t) - ctx := context.Background() - require.NoError(t, f.ledger.SupersedeTask(ctx, f.grant.ID)) - _, err := f.ledger.db.ExecContext(ctx, `UPDATE events SET snapshot = CAST('' AS BLOB) WHERE id = 1`) - require.NoError(t, err) - - _, err = f.ledger.CreateTask(ctx, []int64{1}) - require.ErrorIs(t, err, ErrNotDispatchable) -} - -// A reported outcome stands, and so does what came with it: an -// acknowledgement arriving after the completion is refused, not written. -func TestAnAcknowledgementAfterTheOutcomeIsRefused(t *testing.T) { - f := newDispatchFixture(t) - ctx := context.Background() - _, _, err := f.d.Get(ctx, 1) - require.NoError(t, err) - _, err = f.d.Complete(ctx, 1, Completion{Outcome: OutcomeSucceeded}) - require.NoError(t, err) - - late := int64(4242) - _, err = f.d.Ack(ctx, 1, &late) - - require.ErrorIs(t, err, ErrReportConflict) - var ack *int64 - require.NoError(t, f.ledger.db.QueryRowContext(ctx, `SELECT ack_id FROM task_events WHERE event_id = 1`).Scan(&ack)) - assert.Nil(t, ack, "nothing was written") -} - -// A withdrawn exposure is finished: the record it belonged to may be running -// again on a new task, and the old row does not follow it. -func TestAWithdrawnExposureDoesNotFollowTheRetry(t *testing.T) { - f := newDispatchFixture(t) - ctx := context.Background() - _, err := f.ledger.db.ExecContext(ctx, `UPDATE task_events SET delivery = 'exposed', exposed_at = 'launch' WHERE event_id = 1`) - require.NoError(t, err) - - tx, err := f.ledger.db.BeginTx(ctx, nil) - require.NoError(t, err) - require.NoError(t, f.ledger.supersedeTask(ctx, tx, f.grant.ID)) - require.NoError(t, f.ledger.withdrawExposure(ctx, tx, f.grant.ID, 1, StateAdmitted, "")) - retry, err := f.ledger.createTask(ctx, tx, []int64{1}) - require.NoError(t, err) - require.NoError(t, tx.Commit()) - - // The retry's worker pulls and completes it, so the record is completed. - d, err := f.ledger.Dispatch(ctx, retry.Token, adapterAgentID) - require.NoError(t, err) - _, _, err = d.Get(ctx, 1) - require.NoError(t, err) - _, err = d.Complete(ctx, 1, Completion{Outcome: OutcomeSucceeded}) - require.NoError(t, err) - - // The withdrawn row on the old task stays where it was. - _, err = f.ledger.db.ExecContext(ctx, `UPDATE task_events SET delivery = 'completed' WHERE task_id = ? AND event_id = 1`, f.grant.ID) - require.Error(t, err) - var delivery string - require.NoError(t, f.ledger.db.QueryRowContext(ctx, `SELECT delivery FROM task_events WHERE task_id = ? AND event_id = 1`, f.grant.ID).Scan(&delivery)) - assert.Equal(t, "exposed", delivery) -} - -// A queued record does not go around the admitted one that holds its -// conversation: admission queued it precisely because that one is next. -func TestAQueuedRecordDoesNotJumpItsConversation(t *testing.T) { - ctx := context.Background() - ledger, err := OpenLedger(filepath.Join(t.TempDir(), "state", "connector.db")) - require.NoError(t, err) - t.Cleanup(func() { _ = ledger.Close() }) - for _, id := range []int64{1, 2} { - seenRecord(t, ledger, id) - _, err := ledger.Admission().Commit(ctx, admittedVerdict(id, 0, "recording:10304028989")) - require.NoError(t, err) - } - require.Equal(t, StateQueued, getRecord(t, ledger, 2).State) - - _, err = ledger.CreateTask(ctx, []int64{2}) - - require.ErrorIs(t, err, ErrConversationBusy) - assert.Contains(t, err.Error(), "event 1 is admitted ahead of event 2") - _, err = ledger.CreateTask(ctx, []int64{1}) - assert.NoError(t, err, "and the record that is next dispatches") -} - -// A task is its worker's until it is superseded, even after retention has -// cleared what its records said: the conversation a task holds is written on -// the task's own rows, where retention does not reach. -func TestRetentionDoesNotFreeALiveTasksConversation(t *testing.T) { - f := newDispatchFixture(t) - ctx := context.Background() - for _, id := range []int64{1, 2} { - _, _, err := f.d.Get(ctx, id) - require.NoError(t, err) - _, err = f.d.Complete(ctx, id, Completion{Outcome: OutcomeSucceeded}) - require.NoError(t, err) - } - dropped, err := f.ledger.DropContent(ctx, time.Now().Add(time.Hour), time.Now().Add(time.Hour)) - require.NoError(t, err) - require.Equal(t, 2, dropped, "the records' payloads are gone, conversation and all") - - seenRecord(t, f.ledger, 3) - _, err = f.ledger.Admission().Commit(ctx, admittedVerdict(3, 0, "recording:10304028989")) - require.NoError(t, err) - _, err = f.ledger.CreateTask(ctx, []int64{3}) - - require.ErrorIs(t, err, ErrConversationBusy) - assert.Contains(t, err.Error(), "is live on the conversation of event 3") - require.NoError(t, f.ledger.SupersedeTask(ctx, f.grant.ID)) - _, err = f.ledger.CreateTask(ctx, []int64{3}) - assert.NoError(t, err, "and the conversation is free once the task is not") -} - -// An acknowledgement with an id, after one without, is a second report, not -// the same call retried: the receipt stands and nothing is written. -func TestAnAcknowledgementIdAfterTheAcknowledgementIsRefused(t *testing.T) { - f := newDispatchFixture(t) - ctx := context.Background() - _, _, err := f.d.Get(ctx, 1) - require.NoError(t, err) - _, err = f.d.Ack(ctx, 1, nil) - require.NoError(t, err) - - late := int64(4242) - _, err = f.d.Ack(ctx, 1, &late) - - require.ErrorIs(t, err, ErrReportConflict) - assert.Contains(t, err.Error(), "acknowledged without an acknowledgement id") - var ack *int64 - require.NoError(t, f.ledger.db.QueryRowContext(ctx, `SELECT ack_id FROM task_events WHERE event_id = 1`).Scan(&ack)) - assert.Nil(t, ack, "nothing was written") -} - -// A retry that has nothing to point at answers the receipt it already has: no -// id is not a different id. -func TestAnAcknowledgementWithoutAnIdAfterOneWithItIsTheSameReceipt(t *testing.T) { - f := newDispatchFixture(t) - ctx := context.Background() - _, _, err := f.d.Get(ctx, 1) - require.NoError(t, err) - posted := int64(4242) - first, err := f.d.Ack(ctx, 1, &posted) - require.NoError(t, err) - - again, err := f.d.Ack(ctx, 1, nil) - - require.NoError(t, err) - assert.Equal(t, first, again) - var ack *int64 - require.NoError(t, f.ledger.db.QueryRowContext(ctx, `SELECT ack_id FROM task_events WHERE event_id = 1`).Scan(&ack)) - require.NotNil(t, ack) - assert.Equal(t, posted, *ack, "and the id it did report is still there") -} - -// The same rule holds for anything else writing to the file: an id is written -// with the acknowledgement or not at all. -func TestTheSchemaKeepsAnAcknowledgementIdWithItsAcknowledgement(t *testing.T) { - f := newDispatchFixture(t) - ctx := context.Background() - _, _, err := f.d.Get(ctx, 1) - require.NoError(t, err) - _, err = f.d.Ack(ctx, 1, nil) - require.NoError(t, err) - - _, err = f.ledger.db.ExecContext(ctx, `UPDATE task_events SET ack_id = 4242 WHERE event_id = 1`) - - require.Error(t, err) - assert.Contains(t, err.Error(), "written with the acknowledgement, once") -} diff --git a/internal/connector/ledger_events.go b/internal/connector/ledger_events.go index 59f8a0f6b..24dfff8a9 100644 --- a/internal/connector/ledger_events.go +++ b/internal/connector/ledger_events.go @@ -457,7 +457,7 @@ type transition struct { // set is further columns written in the same statement. set []assignment // byOperator adds the edges only a person's decision has (operatorEdges): - // out of held, and out of completed by a redispatch or a discard. The + // out of held, and out of completed by a discard or an import. The // database refuses them to anything that does not also write the // decision (ledger_hold.go). byOperator bool @@ -554,15 +554,16 @@ func (l *Ledger) move(ctx context.Context, db dbtx, t transition) (bool, error) switch t.state { case StateDispatched: // A record is dispatched exactly while a live task carries it: it - // enters dispatched only with its task row already written - // (createTask), and a repeat is a repeat. + // enters dispatched only with its task row already written, which an + // older build's dispatcher did and this build never does, and a + // repeat is a repeat. query.WriteString(" AND (state = 'dispatched' OR " + onLiveTask + ")") case StateCompleted: default: // Invariant 4 of the dispatch lifecycle (ledger_dispatch.go): a - // record a worker was handed leaves dispatched only to completed — + // record a worker was handed leaves dispatched only to completed, // and any other record leaves it only once no live task carries it - // (supersedeTask retires the row first). + // (an older build's supersession retired the row first). query.WriteString(" AND NOT (" + heldByWorker + ")") query.WriteString(" AND NOT (state = 'dispatched' AND " + onLiveTask + ")") } diff --git a/internal/connector/ledger_hold.go b/internal/connector/ledger_hold.go index c31dbdfbe..4ea1f43e2 100644 --- a/internal/connector/ledger_hold.go +++ b/internal/connector/ledger_hold.go @@ -12,7 +12,9 @@ import ( // The hold marker, intake generations, the review tag, and people's decisions // on records: the ledger half of `basecamp connect --hold`, `release`, -// `redispatch`, `discard`, `shadow promote` and `import`. +// `discard`, `shadow promote` and `import`. `connect redispatch` went with +// the worker side in #815; what it wrote into a ledger is still read, and +// withdrawn by a hold, a discard or an import. // // # Invariants // @@ -24,15 +26,11 @@ import ( // by a task's end returning it, by anything — is written held instead, by // a trigger, in the same statement. A held record is not startable. // 2. The hold marker stops dispatch and posting at the database. While it -// stands no attempt row can be written, no task takes a follow-up, no -// event is handed to a worker for the first time — neither a first -// exposure nor a first pull of an event the launch exposed, so a worker a -// crashed connector left running is told nothing new — and no outbox -// intent can move to sending. It lives in the ledger, so every -// start respects it, and only Release clears it. What it does not stop is -// what such a worker already holds: an instruction it was handed before -// the hold, and its own Basecamp credential. Ending it is the one-owner -// rule's (driver/worker.go), and a person can hurry it with redispatch. +// stands its triggers refuse an attempt row, a task's follow-up, an +// event's first exposure or first pull, and an outbox intent moving to +// sending. This build writes none of those; the triggers stay as an +// older build's ledger has them. The marker lives in the ledger, so every +// start respects it, and only Release clears it. // 3. A hold is one transaction: the marker, a new intake generation, the // review tag on every non-terminal record of the generations before it // (clearing any earlier authorization, a redispatch still waiting for its @@ -40,20 +38,20 @@ import ( // 4. A person's decision is one transaction with the state change it makes, // and it records who decided. A terminal record leaves its state only // against a decision row made after its outcome settled: completed to -// admitted by the redispatch the record names, which the move consumes; +// admitted by a redispatch an older build recorded, which the end of the +// record's task consumes (tasks_end_applies_redispatch); // completed(unknown) to discarded(by_operator) by a discard, and // completed(unknown or failed) to discarded(imported_done) by an import's // done decision. Discarded never leaves. A trigger refuses every other // edge. -// 5. A redispatch never runs two workers for one event. The replaced task's -// token is superseded in the authorization's transaction, and an event -// whose task is still live is not admitted until that task ends: the -// authorization waits on the record and a trigger applies it in the -// transaction that ends the task. One live task per conversation keeps -// the new task from starting before then. -// 6. Admitted means dispatchable. A redispatch admits only a record that -// still has its snapshot and a served project; anything else is decided -// again by admission, whose verdict stands. +// 5. A redispatch an older build recorded for an event whose task was still +// live waits on the record (redispatch_decision) for that task to end, +// and a trigger applies it then. Status counts what waits +// (redispatch_pending); nothing in this build ends a task, so it waits +// until a hold, a discard or an import withdraws it. +// 6. Admitted means dispatchable: that trigger admits only a record that +// still has its snapshot, and retention keeps the snapshot of a record a +// redispatch waits on. // 7. Shadow promote and import are atomic under a crash: each is one ledger // transaction, and promote exposes the shadow ledger at the normal path // only after its hold committed, by one rename (promote.go). @@ -216,10 +214,6 @@ BEGIN END; ` -// ErrHeld is the hold marker refusing to hand a worker something new. It is -// not a failure of the task: nothing more is handed over until release. -var ErrHeld = errors.New("the connector is held") - // Reasons a person's decision writes. const ( // ReasonByOperator is a record a person closed without running it. @@ -232,17 +226,12 @@ const ( // operatorEdges are the moves only a person's decision makes, by target: the // states a record may leave for it. The lifecycle's own edges (ledger_events.go) // are what the connector does by itself; these are never taken automatically. +// +// A discard or an import's done decision closes a held record or a worker's +// outcome an older build left. The redispatch edges out of completed and held +// went with `connect redispatch`; a hold writes held through the review tag's +// trigger, not through a move. var operatorEdges = map[RecordState][]RecordState{ - // A redispatch admits a completed record, or a held one with its snapshot - // (queued when its conversation is live). - StateAdmitted: {StateCompleted, StateHeld}, - StateQueued: {StateHeld}, - // A hold holds what was waiting for a worker. - StateHeld: {StateAdmitted, StateQueued}, - // A redispatch of a record held over a blocking reason runs it again as - // blocked. - StateBlocked: {StateHeld}, - // A discard closes a held record or an unknown outcome. StateDiscarded: {StateHeld, StateCompleted}, } @@ -426,20 +415,6 @@ func (l *Ledger) Release(ctx context.Context, by string) (ReleaseResult, error) return out, err } -// isHeld reports whether the hold marker stands, inside a caller's -// transaction: what a refused write asks before it calls itself a failure. -func isHeld(ctx context.Context, q rowQuerier) (bool, error) { - _, ok, err := readHold(ctx, q) - return ok, err -} - -// Held reports whether the hold marker stands. Its signature is -// OutboxOptions.Paused's. -func (l *Ledger) Held(ctx context.Context) (bool, error) { - _, ok, err := l.HoldMarker(ctx) - return ok, err -} - // HoldMarker reads the standing hold, if any. func (l *Ledger) HoldMarker(ctx context.Context) (Hold, bool, error) { return readHold(ctx, l.db) @@ -481,21 +456,16 @@ type decision struct { note string } -func recordDecision(ctx context.Context, tx Tx, d decision) error { - _, err := insertDecision(ctx, tx, d) - return err -} - -func insertDecision(ctx context.Context, tx Tx, d decision) (int64, error) { - res, err := tx.ExecContext(ctx, ` +func recordDecision(ctx context.Context, tx *sql.Tx, d decision) error { + _, err := tx.ExecContext(ctx, ` INSERT INTO decisions (action, event_id, decided_by, decided_at, from_state, from_reason, from_outcome, to_state, superseded_task_id, note) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, d.action, nullableID64(d.eventID), d.by, d.at, string(d.fromState), d.fromReason, string(d.fromOutcome), string(d.toState), nullableID64(d.supersededTask), d.note) if err != nil { - return 0, fmt.Errorf("connector: record the decision: %w", err) + return fmt.Errorf("connector: record the decision: %w", err) } - return res.LastInsertId() + return nil } // Connection states the run command reports for status. diff --git a/internal/connector/ledger_import.go b/internal/connector/ledger_import.go index 8b6946e24..efa5eef3b 100644 --- a/internal/connector/ledger_import.go +++ b/internal/connector/ledger_import.go @@ -214,11 +214,6 @@ UPDATE outbox SET state = 'canceled', finished_at = ?, note = 'discarded by a pe WHERE event_id = ? AND state = 'pending' AND kind IN ('guard_ack', 'holding_reply')`, now, e.EventID); err != nil { return ImportResult{}, fmt.Errorf("connector: cancel lifecycle messages for %d: %w", e.EventID, err) } - if l.hooks.RecordDecided != nil { - if err := l.hooks.RecordDecided(ctx, tx, RecordDecision{EventID: e.EventID, Action: DecisionDiscard, At: at}); err != nil { - return ImportResult{}, err - } - } if !recorded { if err := recordDecision(ctx, tx, decision{action: "import", eventID: e.EventID, by: by, at: now, fromState: RecordState(state), toState: toState, note: "done"}); err != nil { diff --git a/internal/connector/ledger_migrations_test.go b/internal/connector/ledger_migrations_test.go index bcf8ff960..b02d2d28d 100644 --- a/internal/connector/ledger_migrations_test.go +++ b/internal/connector/ledger_migrations_test.go @@ -27,7 +27,7 @@ const migrationsBeforeRoutePathsDropped = 12 func applyMigrationsThrough(t *testing.T, path string, n int) *sql.DB { t.Helper() require.NoError(t, os.MkdirAll(filepath.Dir(path), 0o700)) - old, err := sql.Open("sqlite", ledgerDSN(path, true)) + old, err := sql.Open("sqlite", ledgerDSN(path)) require.NoError(t, err) _, err = old.ExecContext(context.Background(), `CREATE TABLE schema_migrations (version INTEGER PRIMARY KEY, applied_at TEXT NOT NULL)`) require.NoError(t, err) diff --git a/internal/connector/ledger_owner_unix_test.go b/internal/connector/ledger_owner_unix_test.go index 0eaa66167..9612a3e30 100644 --- a/internal/connector/ledger_owner_unix_test.go +++ b/internal/connector/ledger_owner_unix_test.go @@ -60,7 +60,7 @@ func TestASecondOpenVetsTheDirectoryChain(t *testing.T) { require.NoError(t, os.Chmod(home, 0o777)) t.Cleanup(func() { _ = os.Chmod(home, 0o700) }) - _, err = OpenExistingLedger(context.Background(), path) + _, err = OpenLedgerReadOnly(context.Background(), path) require.Error(t, err) assert.Contains(t, err.Error(), "secure the ledger") @@ -80,7 +80,7 @@ func TestASecondOpenVetsTheFileMode(t *testing.T) { require.NoError(t, os.Chmod(path, 0o644)) - _, err = OpenExistingLedger(context.Background(), path) + _, err = OpenLedgerReadOnly(context.Background(), path) require.Error(t, err) assert.Contains(t, err.Error(), "can be read by other users (mode 0644)") diff --git a/internal/connector/ledger_status.go b/internal/connector/ledger_status.go index 71b636fe9..35ed97d59 100644 --- a/internal/connector/ledger_status.go +++ b/internal/connector/ledger_status.go @@ -9,7 +9,7 @@ import ( "strings" "time" - "github.com/basecamp/basecamp-cli/internal/connector/driver" + "github.com/basecamp/basecamp-cli/internal/connector/procid" ) // OpenLedgerReadOnly opens an existing ledger for reading only: no migration, @@ -170,7 +170,7 @@ type TaskStatus struct { PID int `json:"pid,omitempty"` PGID int `json:"pgid,omitempty"` // ProcessStartedAt is the start time recorded with the pid: with it, the - // pid is an identity (driver.OwnsWorker). + // pid is an identity (procid.OwnsWorker). ProcessStartedAt *time.Time `json:"process_started_at,omitempty"` // TakerPID, TakerPGID and TakerStartedAt are the process the task token // went to, where one took it: a worker's MCP server, which lives in a @@ -194,22 +194,33 @@ type TaskStatus struct { } // WorkerIdentity is the attempt's recorded worker, for the one-owner rule. -func (t TaskStatus) WorkerIdentity() driver.Process { +func (t TaskStatus) WorkerIdentity() procid.Process { return statusIdentity(t.PID, t.PGID, t.ProcessStartedAt) } // TakerIdentity is the process the task token went to, for the same rule. // It says nothing about TakerUnaccounted, which is a fact of the ledger and // not a question for the kernel: a caller asks that first. -func (t TaskStatus) TakerIdentity() driver.Process { +func (t TaskStatus) TakerIdentity() procid.Process { return statusIdentity(t.TakerPID, t.TakerPGID, t.TakerStartedAt) } -func statusIdentity(pid, pgid int, started *time.Time) driver.Process { +// statusIdentity is the one way a process read back out of the ledger becomes +// an identity the one-owner rule can answer about (procid.OwnsWorker). +// +// The exactness bit is not stored beside the stamp and does not need to be: +// an older build wrote a start time only when the kernel gave it, so a stamp +// that comes back out of the ledger is the kernel's by construction and a +// record with no stamp has no identity at all. Both read-back paths, the +// worker and the taker, go through here, because a path that rebuilds +// procid.Process by hand drops StartedExact, and a record without it is +// neither gone nor running (procid.ErrIdentityUnknown): status would call +// every live worker "unverified". +func statusIdentity(pid, pgid int, started *time.Time) procid.Process { if started == nil { - return driver.Process{PID: pid, PGID: pgid} + return procid.Process{PID: pid, PGID: pgid} } - return recordedIdentity(pid, pgid, *started) + return procid.Process{PID: pid, PGID: pgid, StartedAt: *started, StartedExact: !started.IsZero()} } // IntentStatus is a lifecycle message waiting for a person. The body is not @@ -630,21 +641,3 @@ FROM task_events WHERE task_id = ? ORDER BY event_id`, taskID) } return out, rows.Err() } - -// UnfinishedFromOthers counts the records not yet finished that someone other -// than operatorID asked for: admitted, queued, held or dispatched, or blocked -// after its worker failed to start, by the person admission trusted at the -// gate (the performer, else the creator). A record only seen, or blocked on -// any other reason, is judged again under the trust the connector runs with -// now, so it isn't counted: one stuck request from someone else would -// otherwise keep dangerous mode off for good. -func (l *Ledger) UnfinishedFromOthers(ctx context.Context, operatorID int64) (int, error) { - var n int - err := l.db.QueryRowContext(ctx, ` -SELECT COUNT(*) FROM events -WHERE (state IN (?, ?, ?, ?) OR (state = ? AND reason = ?)) - AND COALESCE(performed_by_id, creator_id) <> ?`, - string(StateAdmitted), string(StateQueued), string(StateHeld), string(StateDispatched), - string(StateBlocked), ReasonSpawnFailed, operatorID).Scan(&n) - return n, err -} diff --git a/internal/connector/ledger_tasks.go b/internal/connector/ledger_tasks.go index 83dfe164d..cbacec8f8 100644 --- a/internal/connector/ledger_tasks.go +++ b/internal/connector/ledger_tasks.go @@ -1,58 +1,24 @@ package connector -import ( - "context" - "crypto/rand" - "database/sql" - "encoding/hex" - "errors" - "fmt" - "slices" - "strings" - "time" - - "github.com/basecamp/basecamp-cli/internal/connector/driver" -) - -// Tasks and attempts: the dispatcher's half of the ledger. -// -// A task is one conversation's work, bound to a token; an attempt is one -// worker run under it. The basecamp_connect domain (ledger_dispatch.go) is -// the worker's view of the same rows. +// Tasks and attempts: the dispatcher's half of the ledger, as an older build +// wrote it. // -// # Invariants +// A task was one conversation's work, bound to a token; an attempt was one +// worker run under it. Since #815 removed the worker side nothing writes +// either. Status reads what a ledger still holds (statusTasks, +// statusDispatches), a discard or an import reads the outcome an attempt +// settled (loadEventTask), and the schema below stays as it shipped. // -// Each is held by the database where SQL can say it, and by a test that fails -// without it (ledger_tasks_test.go). +// What the schema held an older build to, and still holds any write to: // -// 1. Exposure before hand-off. An attempt is written launching in the same -// transaction that writes its originating event exposed and moves the -// record to dispatched, and before the driver is asked to start anything. -// A follow-up is written exposed (ExposeEvent) before a prompt about it is -// sent. -// 2. One live task per conversation, one live attempt per task, and -// (migration 5's task_events_one_live_task) one live task per event. -// Unique partial indexes, so two dispatchers on one ledger cannot both -// win. There is no rule about directories: every task runs in the -// connector's own, and two workers in one directory are managed by the -// people running them until sandboxes are real. -// 3. An ended task has no valid token and no live events. Ending a task, -// superseding its token and retiring its events are one transaction, and -// a trigger refuses the end without the supersession, so a worker that -// outlives its task is refused by basecamp_connect. -// 4. Automatic retry is bounded and proven. An exposure is withdrawn — the -// record back to admitted — only when the attempt that wrote it ended with -// the driver's report that no worker process existed, and only for the -// event's first such withdrawal (withdrawn_at, kept on the retired row, -// is that budget); a second is blocked(spawn_failed), which -// waits for a person. Anything else that ends an exposed, unreported event -// makes it completed with outcome unknown. -// 5. Outcomes and stop reasons are separate. A stop reason is written on the -// attempt, an outcome on the task event; neither is computed from the -// other, and settlement never overwrites a reported outcome. -// 6. An adopted reply is a link, never an outcome: AdoptReply writes a reply -// id beside an unknown outcome and leaves the outcome unknown. -// 7. Attempt states move forward only: launching → running → ended, or +// 1. One live task per conversation, one live attempt per task, and +// (migration 5's task_events_one_live_task) one live task per event, by +// unique partial indexes. +// 2. An ended task has no valid token and no live events: a trigger refuses +// the end without the supersession. +// 3. Outcomes and stop reasons are separate. A stop reason is on the +// attempt, an outcome on the task event. +// 4. Attempt states move forward only: launching → running → ended, or // launching → ended. // // migrationTasksAndAttempts is migration 7 as it shipped. Its route and @@ -130,1243 +96,6 @@ BEGIN END; ` -// AttemptState is where an attempt is. -type AttemptState string - -const ( - // AttemptLaunching is written before the driver is asked to start a - // worker. Found after a crash it is treated as running: the worker may - // exist. - AttemptLaunching AttemptState = "launching" - // AttemptRunning has its process or session id. - AttemptRunning AttemptState = "running" - // AttemptEnded has a stop reason. - AttemptEnded AttemptState = "ended" -) - -// StopReason is why an attempt ended. It is not an outcome. -type StopReason string - -const ( - // StopFinished is a clean stop: the turn ended and the worker exited 0. - StopFinished StopReason = "finished" - // StopFailed is a refusal, a stop the connector did not ask for, a - // non-zero exit, or a worker that could not be started. - StopFailed StopReason = "failed" - // StopDeadline is the task's deadline. - StopDeadline StopReason = "deadline" - // StopShutdown is the connector shutting down. - StopShutdown StopReason = "shutdown" - // StopLost is a worker that went away with a turn in flight, or one a - // restarted connector found. - StopLost StopReason = "lost" -) - // OutcomeUnknown is an event that was exposed to a worker and never // reported: whatever ended the attempt, the worker may have acted on it. const OutcomeUnknown Outcome = "unknown" - -// ReasonSpawnFailed blocks an event whose worker could not be started a -// second time. It waits for a person's redispatch. -const ReasonSpawnFailed = "spawn_failed" - -// Errors from the task ledger. -var ( - // ErrNotStartable is a launch for a record that is not waiting for a - // worker: not admitted or queued, without its snapshot, in a project - // connect.json does not serve, or on a conversation that already has a - // live task. - ErrNotStartable = errors.New("the record is not waiting for a worker") - // ErrNoLiveAttempt is a write for an attempt that has ended or never was. - ErrNoLiveAttempt = errors.New("no live attempt by that id") -) - -// Tx is a ledger transaction a hook writes in, so what the hook writes (an -// outbox intent) commits or rolls back with the transition that called for -// it. -type Tx interface { - ExecContext(ctx context.Context, query string, args ...any) (sql.Result, error) - QueryRowContext(ctx context.Context, query string, args ...any) *sql.Row - QueryContext(ctx context.Context, query string, args ...any) (*sql.Rows, error) -} - -// Hooks run inside the transactions of the ledger's lifecycle transitions. -// A hook's error rolls the transition back. Set them once, before the ledger -// is used. -type Hooks struct { - // VerdictCommitted runs in admission's verdict transaction, after the - // verdict is written: where the guard acknowledgement and the holding - // reply are called for. - VerdictCommitted func(ctx context.Context, tx Tx, v CommittedVerdict) error - // TaskLaunched runs in LaunchTask's transaction. - TaskLaunched func(ctx context.Context, tx Tx, launch Launch) error - // AttemptEnded runs in EndAttempt's transaction, after every event is - // settled: where the attempt's completion message is called for. - AttemptEnded func(ctx context.Context, tx Tx, s Settlement) error - // StillRunning runs in StillRunning's transaction. - StillRunning func(ctx context.Context, tx Tx, tick StillRunningTick) error - // RecordDecided runs in the transaction of a person's decision on a - // record — a redispatch or a discard: where a notice already posted stops - // asking for anything, and is retracted. - RecordDecided func(ctx context.Context, tx Tx, d RecordDecision) error -} - -// RecordDecision is what RecordDecided is told: the record a person decided, -// how, and when. -type RecordDecision struct { - EventID int64 - Action DecisionAction - At time.Time -} - -// DecisionAction is a person's decision on a record, in the words the -// decisions table records it with. -type DecisionAction string - -const ( - // DecisionRedispatch authorized the record to run. - DecisionRedispatch DecisionAction = "redispatch" - // DecisionDiscard closed it without running it. - DecisionDiscard DecisionAction = "discard" -) - -// SetHooks installs hooks. Not safe concurrently with ledger use. -func (l *Ledger) SetHooks(h Hooks) { l.hooks = h } - -// CommittedVerdict is what VerdictCommitted is told. -type CommittedVerdict struct { - EventID int64 - State RecordState - Reason string - Trigger string - Acknowledge bool - ReplyKind string - // ReplyRecordingID is where a reply to the event goes. - ReplyRecordingID int64 -} - -// LaunchSpec asks for a task and its first attempt. -type LaunchSpec struct { - // EventID is the originating event: an admitted or queued record. - EventID int64 - // Served is the served projects the caller decided this launch from, - // already cut to the run's --project scope. For the dispatcher it is the - // one reading of connect.json that this pass took, handed down — not a - // fresh read, and not a promise about what the file says at the moment - // the transaction commits. Records on the originating record's - // conversation join its task only from its own project, and only while - // that project is among these. - Served []int64 - // Driver is the driver's name. - Driver string - // Deadline is how long the task may run; zero for none. - Deadline time.Duration -} - -// Launch is a task written launching. -type Launch struct { - TaskID int64 - // Token binds the worker to the task. It is returned once and stored - // only as a hash. - Token string - AttemptID string - // EventIDs are the task's events, originating first. Only the originating - // event is exposed; the rest wait at delivery admitted. - EventIDs []int64 - ConversationKey string - Driver string - LaunchedAt time.Time - // DeadlineAt is zero when the task has no deadline. - DeadlineAt time.Time -} - -// LaunchTask writes a task, its first attempt as launching, and its -// originating event exposed, in one transaction (invariant 1). Records on the -// same conversation that wait for a worker join the task at delivery -// admitted. -func (l *Ledger) LaunchTask(ctx context.Context, spec LaunchSpec) (Launch, error) { - if spec.Driver == "" { - return Launch{}, errors.New("connector: a launch needs a driver") - } - attemptID, err := newAttemptID() - if err != nil { - return Launch{}, err - } - var out Launch - err = retryBusy(func() error { - var err error - out, err = l.launchTask(ctx, spec, attemptID) - return err - }) - return out, err -} - -func (l *Ledger) launchTask(ctx context.Context, spec LaunchSpec, attemptID string) (Launch, error) { - tx, err := l.db.BeginTx(ctx, nil) - if err != nil { - return Launch{}, fmt.Errorf("connector: begin launch: %w", err) - } - defer func() { _ = tx.Rollback() }() - - record, err := loadRecord(ctx, tx, spec.EventID) - if err != nil { - return Launch{}, err - } - switch { - case record.State != StateAdmitted && record.State != StateQueued, - record.ContentDropped, len(record.Decision.Snapshot) == 0, - !record.Decision.Served, record.Decision.ConversationKey == "": - return Launch{}, fmt.Errorf("connector: launch event %d (%s): %w", spec.EventID, record.State, ErrNotStartable) - case !slices.Contains(spec.Served, record.BucketID): - // Two different questions, and neither is "what does the file say - // right now". Decision.Served is what admission wrote when it - // decided the record, so it says the project was served then; - // spec.Served is the set its caller decided this launch from. - // - // What this buys, said plainly, because the comment here used to - // claim more. It is a check on the set its caller named, not a read - // of the file: a launch that names a set not covering its record is - // refused rather than trusted, and one that names no set authorizes - // nothing rather than defaulting open. That is what makes a - // non-dispatcher caller fail closed. - // - // What closes the window between reading connect.json and - // committing this transaction is the caller's: it has to read the - // served set under connect.json's own lock and keep the lock until - // this commits, so an unserve lands wholly before the reading or - // wholly after the task exists. This check is what that reading is - // spent on. Dispatcher.launch was that caller, and nothing calls - // this since #815 removed the dispatcher. - return Launch{}, fmt.Errorf("connector: launch event %d in project %d, which is not served: %w", spec.EventID, record.BucketID, ErrNotStartable) - } - var busy bool - if err := tx.QueryRowContext(ctx, ` -SELECT EXISTS (SELECT 1 FROM tasks WHERE ended_at IS NULL AND conversation_key = ?) - OR EXISTS (SELECT 1 FROM task_events WHERE event_id = ? AND retired_at IS NULL)`, - record.Decision.ConversationKey, spec.EventID).Scan(&busy); err != nil { - return Launch{}, fmt.Errorf("connector: launch event %d: %w", spec.EventID, err) - } - if busy { - return Launch{}, fmt.Errorf("connector: launch event %d: a live task holds its conversation: %w", spec.EventID, ErrNotStartable) - } - - now := l.now() - nowStamp := stamp(now) - var deadline any - var deadlineAt time.Time - if spec.Deadline > 0 { - deadlineAt = now.Add(spec.Deadline) - deadline = stamp(deadlineAt) - } - // The originating event first, then every other record on the - // conversation that waits for a worker. createTask dispatches them all - // and refuses an event a live task already carries. - // In the originating record's own project, and in one this run serves - // now: the record was chosen from the served set, and what joins it is - // held to the same set rather than to what admission wrote on each row. - joinable, err := joinableOn(ctx, tx, record.Decision.ConversationKey, record.BucketID, spec.Served, spec.EventID) - if err != nil { - return Launch{}, err - } - grant, err := l.createTask(ctx, tx, append([]int64{spec.EventID}, joinable...)) - if err != nil { - return Launch{}, err - } - taskID := grant.ID - if _, err := tx.ExecContext(ctx, ` -UPDATE tasks SET conversation_key = ?, driver = ?, originating_event_id = ?, deadline_at = ? -WHERE id = ?`, record.Decision.ConversationKey, spec.Driver, spec.EventID, deadline, taskID); err != nil { - return Launch{}, fmt.Errorf("connector: create task for %d: %w", spec.EventID, err) - } - if _, err := tx.ExecContext(ctx, ` -INSERT INTO attempts (id, task_id, seq, driver, state, launched_at) VALUES (?, ?, 1, ?, 'launching', ?)`, - attemptID, taskID, spec.Driver, nowStamp); err != nil { - return Launch{}, fmt.Errorf("connector: write attempt for %d: %w", spec.EventID, err) - } - // The prompt names the originating event's recording, so it is exposed - // before the driver is asked for anything. - if _, err := tx.ExecContext(ctx, ` -UPDATE task_events SET delivery = 'exposed', exposed_at = ?, exposed_attempt_id = ? -WHERE task_id = ? AND event_id = ?`, nowStamp, attemptID, taskID, spec.EventID); err != nil { - return Launch{}, fmt.Errorf("connector: expose event %d: %w", spec.EventID, err) - } - joined := joinable - token := grant.Token - - out := Launch{ - TaskID: taskID, - Token: token, - AttemptID: attemptID, - EventIDs: append([]int64{spec.EventID}, joined...), - ConversationKey: record.Decision.ConversationKey, - Driver: spec.Driver, - LaunchedAt: now, - DeadlineAt: deadlineAt, - } - if l.hooks.TaskLaunched != nil { - if err := l.hooks.TaskLaunched(ctx, tx, out); err != nil { - return Launch{}, fmt.Errorf("connector: launch hook for %d: %w", spec.EventID, err) - } - } - if err := tx.Commit(); err != nil { - return Launch{}, fmt.Errorf("connector: commit launch of %d: %w", spec.EventID, err) - } - return out, nil -} - -func guardFor(acknowledge bool) string { - if acknowledge { - return "armed" - } - return "" -} - -// startableFrom is the SQL condition for a record waiting for a worker: it -// carries what a dispatch needs and no live task holds it. -const startableCondition = ` -e.state IN ('admitted', 'queued') AND e.content_dropped = 0 AND e.snapshot IS NOT NULL -AND e.served = 1 AND e.conversation_key <> '' -AND NOT EXISTS (SELECT 1 FROM task_events te WHERE te.event_id = e.id AND te.retired_at IS NULL)` - -// joinableOn lists the records on key, in bucket, other than except, that -// wait for a worker, oldest first. served narrows it further to the projects -// connect.json serves right now, already cut to the run's --project scope; -// empty serves nothing. -// -// The conversation is not the whole of it, and this is the reason (Copilot on -// #765). A conversation key is the recording's or the Campfire's, never the -// bucket's, so two records on one conversation can sit in two projects — a -// recording moved between them is the ordinary way, and connect.json serving -// only one of them is the ordinary case. A task is authorized against the -// project its originating record was in; handing its worker an event from -// another project would make the served list, which is the whole local answer -// to which projects may drive this agent, leak at the one seam it exists to -// hold. -// -// The record's own served bit is not that authorization either: admission -// wrote it when the record was decided, so it says the project was served -// then. served here is read at join time, so a project the operator has -// stopped serving stops feeding a task that is already running. -func joinableOn(ctx context.Context, tx *sql.Tx, key string, bucket int64, served []int64, except int64) ([]int64, error) { - if !slices.Contains(served, bucket) { - return nil, nil - } - rows, err := tx.QueryContext(ctx, `SELECT e.id FROM events e WHERE e.conversation_key = ? AND e.bucket_id = ? AND e.id <> ? AND `+startableCondition+` ORDER BY e.id`, key, bucket, except) - if err != nil { - return nil, fmt.Errorf("connector: find follow-ups on %s: %w", key, err) - } - defer func() { _ = rows.Close() }() - var ids []int64 - for rows.Next() { - var id int64 - if err := rows.Scan(&id); err != nil { - return nil, err - } - ids = append(ids, id) - } - return ids, rows.Err() -} - -// joinConversation puts every record on key, in bucket and in a project -// served now, that waits for a worker onto the live task taskID at delivery -// admitted, dispatched, as createTask would have, and returns their ids, -// oldest first. -func (l *Ledger) joinConversation(ctx context.Context, tx *sql.Tx, taskID int64, key string, bucket int64, served []int64) ([]int64, error) { - ids, err := joinableOn(ctx, tx, key, bucket, served, 0) - if err != nil { - return nil, err - } - for _, id := range ids { - var acknowledge bool - if err := tx.QueryRowContext(ctx, `SELECT acknowledge FROM events WHERE id = ?`, id).Scan(&acknowledge); err != nil { - return nil, fmt.Errorf("connector: join event %d to task %d: %w", id, taskID, err) - } - if _, err := tx.ExecContext(ctx, `INSERT INTO task_events (task_id, event_id, guard) VALUES (?, ?, ?)`, taskID, id, guardFor(acknowledge)); err != nil { - if isConstraint(err) { - return nil, fmt.Errorf("connector: join event %d to task %d: %w", id, taskID, ErrEventOnLiveTask) - } - return nil, fmt.Errorf("connector: join event %d to task %d: %w", id, taskID, err) - } - moved, err := l.move(ctx, tx, transition{id: id, state: StateDispatched, from: []RecordState{StateAdmitted, StateQueued}}) - if err != nil { - return nil, err - } - if !moved { - return nil, fmt.Errorf("connector: join event %d to task %d: %w", id, taskID, ErrNotStartable) - } - } - return ids, nil -} - -// JoinConversation puts the records on a live task's conversation, in the -// task's own project, that wait for a worker onto the task, at delivery -// admitted, and returns their ids. served is the projects connect.json serves -// now, already cut to the run's --project scope; a task whose project is not -// among them takes nothing. -// -// A task that has ended takes none: they start a task of their own. Nor does a -// task a redispatch superseded while it runs: its worker's token is refused, -// so what joined it could only end unknown. Nor does any task while the hold -// marker stands: joining is a hand-off to a worker (ledger_hold.go). -func (l *Ledger) JoinConversation(ctx context.Context, taskID int64, served []int64) ([]int64, error) { - var out []int64 - err := retryBusy(func() error { - tx, err := l.db.BeginTx(ctx, nil) - if err != nil { - return fmt.Errorf("connector: begin join: %w", err) - } - defer func() { _ = tx.Rollback() }() - var ( - key string - bucket int64 - ) - // The task's project is its originating record's, read here rather - // than trusted from the caller. That record is dispatched while the - // task is live, so retention has not cleared its bucket. - switch err := tx.QueryRowContext(ctx, `SELECT t.conversation_key, e.bucket_id -FROM tasks t JOIN events e ON e.id = t.originating_event_id -WHERE t.id = ? AND t.ended_at IS NULL AND t.superseded_at IS NULL - AND NOT EXISTS (SELECT 1 FROM hold_marker)`, taskID).Scan(&key, &bucket); { - case errors.Is(err, sql.ErrNoRows): - out = nil - return nil - case err != nil: - return fmt.Errorf("connector: join task %d: %w", taskID, err) - } - if key == "" || bucket == 0 { - out = nil - return nil - } - ids, err := l.joinConversation(ctx, tx, taskID, key, bucket, served) - if err != nil { - return err - } - if err := tx.Commit(); err != nil { - return fmt.Errorf("connector: commit join of task %d: %w", taskID, err) - } - out = ids - return nil - }) - return out, err -} - -// UnexposedEvents are the events on a task still at delivery admitted, oldest -// first: the follow-ups a live session has not been prompted with. -func (l *Ledger) UnexposedEvents(ctx context.Context, taskID int64) ([]int64, error) { - rows, err := l.db.QueryContext(ctx, ` -SELECT event_id FROM task_events WHERE task_id = ? AND delivery = 'admitted' AND retired_at IS NULL ORDER BY event_id`, taskID) - if err != nil { - return nil, fmt.Errorf("connector: unexposed events of task %d: %w", taskID, err) - } - defer func() { _ = rows.Close() }() - var ids []int64 - for rows.Next() { - var id int64 - if err := rows.Scan(&id); err != nil { - return nil, err - } - ids = append(ids, id) - } - return ids, rows.Err() -} - -// ExposeEvent writes a follow-up exposed by the live attempt, and moves its -// record to dispatched, before a prompt about it is sent (invariant 1). It -// reports false when the event was already exposed — by get_dispatch, say — -// which is not an error. -func (l *Ledger) ExposeEvent(ctx context.Context, attemptID string, eventID int64) (bool, error) { - var exposed bool - err := retryBusy(func() error { - tx, err := l.db.BeginTx(ctx, nil) - if err != nil { - return fmt.Errorf("connector: begin expose: %w", err) - } - defer func() { _ = tx.Rollback() }() - taskID, err := liveAttemptTask(ctx, tx, attemptID) - if err != nil { - return err - } - var delivery string - switch err := tx.QueryRowContext(ctx, `SELECT delivery FROM task_events WHERE task_id = ? AND event_id = ? AND retired_at IS NULL`, taskID, eventID).Scan(&delivery); { - case errors.Is(err, sql.ErrNoRows): - return fmt.Errorf("connector: expose event %d: %w", eventID, ErrNotOnTask) - case err != nil: - return fmt.Errorf("connector: expose event %d: %w", eventID, err) - } - if Delivery(delivery) != DeliveryAdmitted { - exposed = false - return nil - } - moved, err := l.move(ctx, tx, transition{id: eventID, state: StateDispatched, from: []RecordState{StateAdmitted, StateQueued, StateDispatched}}) - if err != nil { - return err - } - if !moved { - return fmt.Errorf("connector: expose event %d: %w", eventID, ErrNotDispatchable) - } - if _, err := tx.ExecContext(ctx, ` -UPDATE task_events SET delivery = 'exposed', exposed_at = ?, exposed_attempt_id = ? -WHERE task_id = ? AND event_id = ? AND delivery = 'admitted'`, l.timestamp(), attemptID, taskID, eventID); err != nil { - // A hold refuses a first hand-off (ledger_hold.go). That is not a - // failure of the task: nothing more is handed over until release. - if held, holdErr := isHeld(ctx, tx); holdErr == nil && held { - return fmt.Errorf("connector: expose event %d: %w", eventID, ErrHeld) - } - return fmt.Errorf("connector: expose event %d: %w", eventID, err) - } - if err := tx.Commit(); err != nil { - return fmt.Errorf("connector: commit exposure of %d: %w", eventID, err) - } - exposed = true - return nil - }) - return exposed, err -} - -func liveAttemptTask(ctx context.Context, tx *sql.Tx, attemptID string) (int64, error) { - var taskID int64 - switch err := tx.QueryRowContext(ctx, `SELECT task_id FROM attempts WHERE id = ? AND state <> 'ended'`, attemptID).Scan(&taskID); { - case errors.Is(err, sql.ErrNoRows): - return 0, fmt.Errorf("connector: attempt %s: %w", attemptID, ErrNoLiveAttempt) - case err != nil: - return 0, fmt.Errorf("connector: attempt %s: %w", attemptID, err) - } - return taskID, nil -} - -// AttemptProcess is what MarkRunning records: the worker's process, where -// there is one, and its session id. -// -// Only an identity the kernel gave is written. A start time the connector -// guessed cannot tell a pid from a later process that reused it, so it is -// left out of the record rather than written as if it could, and a record -// with a pid and no start time is one a later process signals nothing on -// (driver.ErrIdentityUnknown). What the ledger holds is therefore exact by -// construction, which is why Identity reads it back as exact. -type AttemptProcess struct { - PID int - PGID int - StartedAt time.Time - // StartedExact says StartedAt is the kernel's own start time for the pid - // (driver.Process.StartedExact). Only then is it written. - StartedExact bool - SessionID string -} - -// Identity is the process the record names, for the one-owner rule. A start -// time in the ledger is the kernel's, since nothing else is written. -func (p AttemptProcess) Identity() driver.Process { - return recordedIdentity(p.PID, p.PGID, p.StartedAt) -} - -// recordedIdentity is the one way a process read back out of the ledger -// becomes an identity the one-owner rule can act on (driver.OwnsWorker). -// -// The exactness bit is not stored beside the stamp and does not need to be: -// startedStamp writes a start time only when the kernel gave it, so a stamp -// that came back out of the ledger is the kernel's by construction and a -// record with no stamp has no identity at all. Every read-back path goes -// through here — the attempt's worker, a redispatch's live worker, status's -// worker and taker — because a path that rebuilds driver.Process by hand -// drops StartedExact, and a record without it is neither gone nor running -// (driver.ErrIdentityUnknown): status would call every live worker -// "unverified" and a redispatch would refuse to stop the worker it replaces. -func recordedIdentity(pid, pgid int, started time.Time) driver.Process { - return driver.Process{PID: pid, PGID: pgid, StartedAt: started, StartedExact: !started.IsZero()} -} - -// startedStamp is the start time as the ledger writes it: the kernel's, or -// nothing at all. -func (p AttemptProcess) startedStamp() any { - if !p.StartedExact || p.StartedAt.IsZero() { - return nil - } - return stamp(p.StartedAt) -} - -// RecordTaker records the process that took the attempt's task token — the -// worker's MCP server, which an agent may have started in a process group of -// its own. A restart ends it by this record, as it ends the worker by the -// worker's. -func (l *Ledger) RecordTaker(ctx context.Context, attemptID string, p AttemptProcess) error { - return retryBusy(func() error { - started := p.startedStamp() - res, err := l.db.ExecContext(ctx, ` -UPDATE attempts SET taker_pid = ?, taker_pgid = ?, taker_started = ? WHERE id = ? AND state <> 'ended'`, - nullableInt(p.PID), nullableInt(p.PGID), started, attemptID) - if err != nil { - return fmt.Errorf("connector: record the process that took the token of %s: %w", attemptID, err) - } - n, err := res.RowsAffected() - if err != nil { - return nil //nolint:nilerr // the write is committed - } - if n == 0 { - return fmt.Errorf("connector: record the process that took the token of %s: %w", attemptID, ErrNoLiveAttempt) - } - return nil - }) -} - -// MarkTakerUnaccounted records that the attempt's task token was delivered -// and the process holding it cannot be accounted for. It is the ledger's -// half of the same rule the socket holds in memory: a restart must not read -// an attempt with no taker recorded as an attempt whose token nobody took, -// and settle around a process that may still have it. -func (l *Ledger) MarkTakerUnaccounted(ctx context.Context, attemptID string) error { - return retryBusy(func() error { - res, err := l.db.ExecContext(ctx, - `UPDATE attempts SET taker_unaccounted = 1 WHERE id = ? AND state <> 'ended'`, attemptID) - if err != nil { - return fmt.Errorf("connector: record the unaccounted token holder of %s: %w", attemptID, err) - } - n, err := res.RowsAffected() - if err != nil { - return nil //nolint:nilerr // the write is committed - } - if n == 0 { - return fmt.Errorf("connector: record the unaccounted token holder of %s: %w", attemptID, ErrNoLiveAttempt) - } - return nil - }) -} - -// MarkRunning moves a launching attempt to running with its process and -// session. -func (l *Ledger) MarkRunning(ctx context.Context, attemptID string, p AttemptProcess) error { - return retryBusy(func() error { - started := p.startedStamp() - res, err := l.db.ExecContext(ctx, ` -UPDATE attempts SET state = 'running', running_at = ?, pid = ?, pgid = ?, process_started = ?, session_id = ? -WHERE id = ? AND state = 'launching'`, - l.timestamp(), nullableInt(p.PID), nullableInt(p.PGID), started, p.SessionID, attemptID) - if err != nil { - return fmt.Errorf("connector: mark attempt %s running: %w", attemptID, err) - } - n, err := res.RowsAffected() - if err != nil { - return err - } - if n == 0 { - return fmt.Errorf("connector: mark attempt %s running: %w", attemptID, ErrNoLiveAttempt) - } - return nil - }) -} - -func nullableInt(v int) any { - if v == 0 { - return nil - } - return v -} - -// AttemptEnd is how an attempt ended. -type AttemptEnd struct { - AttemptID string - Stop StopReason - // SpawnFailed is the driver's report that no worker process ever existed - // (driver.ErrNotStarted). Nothing else makes an exposure withdrawable. - SpawnFailed bool - // NoAutomaticRetry refuses the withdrawal even then: a task under the - // sandbox launcher is never retried automatically. - NoAutomaticRetry bool - // UnrecordedRefusals are refusals RecordRefusal could not write when they - // happened, settled here with the attempt. Refusals it did write are - // already on the attempt. - UnrecordedRefusals int -} - -// Settlement is what ending an attempt did to its task. -type Settlement struct { - TaskID int64 - AttemptID string - Stop StopReason - // SpawnFailed repeats AttemptEnd.SpawnFailed. - SpawnFailed bool - // OriginatingEventID is the task's originating event. - OriginatingEventID int64 - Events []SettledEvent -} - -// SettledEvent is one event's state after its task ended. -type SettledEvent struct { - EventID int64 - // Outcome is the reported outcome, or unknown for an event exposed and - // never reported. Empty for an event never exposed, or withdrawn. - Outcome Outcome - // Reported is whether the outcome is the worker's own report. - Reported bool - ReplyID *int64 - // Returned is an event never exposed: it waits for a task of its own. - Returned bool - // Withdrawn is an exposure withdrawn after a start that ran nothing; the - // record is admitted again, or blocked(spawn_failed) when it already was - // once. - Withdrawn bool - // Blocked is a withdrawal refused a second automatic retry. - Blocked bool - // Decided is a record a person has already redispatched or discarded, so - // the completion notice asks nothing of them. - Decided bool - // Pulled is whether a worker asked for the event's instruction. An event - // the launch exposed and no worker pulled is one the worker never picked - // up: it stopped before it began. - Pulled bool -} - -// EndAttempt ends a live attempt with its stop reason, supersedes the task's -// token, settles every event on the task, and ends the task, in one -// transaction (invariants 3 to 5). Ending an attempt that already ended is -// ErrNoLiveAttempt. -func (l *Ledger) EndAttempt(ctx context.Context, end AttemptEnd) (Settlement, error) { - switch end.Stop { - case StopFinished, StopFailed, StopDeadline, StopShutdown, StopLost: - default: - return Settlement{}, fmt.Errorf("connector: %q is not a stop reason", end.Stop) - } - if end.SpawnFailed && end.Stop != StopFailed { - return Settlement{}, errors.New("connector: a worker that was never started stops as failed") - } - var out Settlement - err := retryBusy(func() error { - var err error - out, err = l.endAttempt(ctx, end) - return err - }) - return out, err -} - -func (l *Ledger) endAttempt(ctx context.Context, end AttemptEnd) (Settlement, error) { - tx, err := l.db.BeginTx(ctx, nil) - if err != nil { - return Settlement{}, fmt.Errorf("connector: begin end of attempt: %w", err) - } - defer func() { _ = tx.Rollback() }() - taskID, err := liveAttemptTask(ctx, tx, end.AttemptID) - if err != nil { - return Settlement{}, err - } - now := l.timestamp() - if _, err := tx.ExecContext(ctx, ` -UPDATE attempts SET state = 'ended', ended_at = ?, stop_reason = ?, spawn_failed = ?, refusals = refusals + ? WHERE id = ?`, - now, string(end.Stop), end.SpawnFailed, end.UnrecordedRefusals, end.AttemptID); err != nil { - return Settlement{}, fmt.Errorf("connector: end attempt %s: %w", end.AttemptID, err) - } - - settlement := Settlement{TaskID: taskID, AttemptID: end.AttemptID, Stop: end.Stop, SpawnFailed: end.SpawnFailed} - var originating sql.NullInt64 - if err := tx.QueryRowContext(ctx, `SELECT originating_event_id FROM tasks WHERE id = ?`, taskID).Scan(&originating); err != nil { - return Settlement{}, fmt.Errorf("connector: settle task %d: %w", taskID, err) - } - settlement.OriginatingEventID = originating.Int64 - - type row struct { - eventID int64 - delivery Delivery - outcome string - replyID sql.NullInt64 - exposedBy sql.NullString - pulled bool - } - rows, err := tx.QueryContext(ctx, ` -SELECT event_id, delivery, outcome, reply_id, exposed_attempt_id, pulled_at IS NOT NULL FROM task_events -WHERE task_id = ? AND retired_at IS NULL ORDER BY event_id`, taskID) - if err != nil { - return Settlement{}, fmt.Errorf("connector: settle task %d: %w", taskID, err) - } - var events []row - for rows.Next() { - var r row - var delivery string - if err := rows.Scan(&r.eventID, &delivery, &r.outcome, &r.replyID, &r.exposedBy, &r.pulled); err != nil { - _ = rows.Close() - return Settlement{}, fmt.Errorf("connector: settle task %d: %w", taskID, err) - } - r.delivery = Delivery(delivery) - events = append(events, r) - } - if err := rows.Close(); err != nil { - return Settlement{}, err - } - - // Withdrawals wait for the supersession: #736's withdrawExposure takes an - // exposure only on a task already superseded. - var withdrawals []int - for _, r := range events { - se := SettledEvent{EventID: r.eventID, Pulled: r.pulled} - switch { - case r.delivery == DeliveryCompleted: - // A reported outcome stands (invariant 5). - se.Outcome, se.Reported = Outcome(r.outcome), r.outcome != string(OutcomeUnknown) - if r.replyID.Valid { - id := r.replyID.Int64 - se.ReplyID = &id - } - case r.delivery == DeliveryAdmitted: - // Never exposed: supersedeTask below returns it to admitted, to - // wait for a task of its own. - se.Returned = true - case end.SpawnFailed && r.exposedBy.Valid && r.exposedBy.String == end.AttemptID: - // Exposed by this attempt, whose driver proved nothing ran - // (invariant 4): withdrawn once the task is superseded, below. - withdrawals = append(withdrawals, len(settlement.Events)) - default: - moved, err := l.move(ctx, tx, transition{id: r.eventID, state: StateCompleted, from: []RecordState{StateDispatched}}) - if err != nil { - return Settlement{}, err - } - if !moved { - // #736's invariant 4: a record a worker was handed leaves - // dispatched only to completed, so nothing else can have moved - // it. Reaching here is a ledger someone wrote by hand. - return Settlement{}, fmt.Errorf("connector: settle event %d: %w", r.eventID, ErrNotDispatchable) - } - if _, err := tx.ExecContext(ctx, ` -UPDATE task_events SET delivery = 'completed', completed_at = ?, outcome = ? WHERE task_id = ? AND event_id = ?`, - now, string(OutcomeUnknown), taskID, r.eventID); err != nil { - return Settlement{}, fmt.Errorf("connector: settle event %d: %w", r.eventID, err) - } - se.Outcome = OutcomeUnknown - } - settlement.Events = append(settlement.Events, se) - } - - // #736's supersession: the token refused, every row retired, and the - // never-exposed events returned to admitted. Then the task ends; the - // trigger refuses an end the supersession did not precede. - if err := l.supersedeTask(ctx, tx, taskID); err != nil { - return Settlement{}, err - } - for _, i := range withdrawals { - if err := l.withdraw(ctx, tx, taskID, settlement.Events[i].EventID, end.NoAutomaticRetry, &settlement.Events[i]); err != nil { - return Settlement{}, err - } - } - if _, err := tx.ExecContext(ctx, `UPDATE tasks SET ended_at = ? WHERE id = ?`, now, taskID); err != nil { - return Settlement{}, fmt.Errorf("connector: end task %d: %w", taskID, err) - } - if l.hooks.AttemptEnded != nil { - if err := l.hooks.AttemptEnded(ctx, tx, settlement); err != nil { - return Settlement{}, fmt.Errorf("connector: attempt-ended hook for %s: %w", end.AttemptID, err) - } - } - if err := tx.Commit(); err != nil { - return Settlement{}, fmt.Errorf("connector: commit end of attempt %s: %w", end.AttemptID, err) - } - return settlement, nil -} - -// withdraw takes back an exposure whose worker never existed: once, the record -// returns to admitted; a second time, or with automatic retry refused, it is -// blocked(spawn_failed). -func (l *Ledger) withdraw(ctx context.Context, tx *sql.Tx, taskID, eventID int64, noRetry bool, se *SettledEvent) error { - var earlier int - if err := tx.QueryRowContext(ctx, `SELECT COUNT(*) FROM task_events WHERE event_id = ? AND withdrawn_at IS NOT NULL`, eventID).Scan(&earlier); err != nil { - return fmt.Errorf("connector: withdraw event %d: %w", eventID, err) - } - to, reason := StateAdmitted, "" - if earlier > 0 || noRetry { - to, reason = StateBlocked, ReasonSpawnFailed - se.Blocked = true - } - // #736's one withdrawal: the marker, then the record's move, refused by - // the database for anything but a launch exposure no worker pulled. - if err := l.withdrawExposure(ctx, tx, taskID, eventID, to, reason); err != nil { - return err - } - se.Withdrawn = true - return nil -} - -// LiveAttempt is an attempt that has not ended. -type LiveAttempt struct { - AttemptID string - TaskID int64 - State AttemptState - Driver string - ConversationKey string - Process AttemptProcess - // Taker is the process the task token went to, where one took it. Its - // PID is zero when none did. - Taker AttemptProcess - // TakerUnaccounted is a token that went out to a process this attempt - // could not account for. Its PID is zero too, and the difference - // matters: nothing of such an attempt is released. - TakerUnaccounted bool - LaunchedAt time.Time - // DeadlineAt is zero when the task has none. - DeadlineAt time.Time -} - -// LiveAttempts lists every attempt not ended, oldest first. On start they are -// all a previous process's: launching is read as running, because the worker -// may exist. -func (l *Ledger) LiveAttempts(ctx context.Context) ([]LiveAttempt, error) { - rows, err := l.db.QueryContext(ctx, ` -SELECT a.id, a.task_id, a.state, a.driver, t.conversation_key, - COALESCE(a.pid, 0), COALESCE(a.pgid, 0), a.process_started, a.session_id, a.launched_at, t.deadline_at, - COALESCE(a.taker_pid, 0), COALESCE(a.taker_pgid, 0), a.taker_started, a.taker_unaccounted -FROM attempts a JOIN tasks t ON t.id = a.task_id -WHERE a.state <> 'ended' ORDER BY a.launched_at, a.id`) - if err != nil { - return nil, fmt.Errorf("connector: live attempts: %w", err) - } - defer func() { _ = rows.Close() }() - var out []LiveAttempt - for rows.Next() { - var ( - a LiveAttempt - state, launched string - started, deadline, took sql.NullString - ) - if err := rows.Scan(&a.AttemptID, &a.TaskID, &state, &a.Driver, &a.ConversationKey, - &a.Process.PID, &a.Process.PGID, &started, &a.Process.SessionID, &launched, &deadline, - &a.Taker.PID, &a.Taker.PGID, &took, &a.TakerUnaccounted); err != nil { - return nil, fmt.Errorf("connector: live attempts: %w", err) - } - if took.Valid { - if a.Taker.StartedAt, err = parseStamp(took.String); err != nil { - return nil, err - } - } - a.State = AttemptState(state) - if a.LaunchedAt, err = parseStamp(launched); err != nil { - return nil, err - } - if started.Valid { - if a.Process.StartedAt, err = parseStamp(started.String); err != nil { - return nil, err - } - } - if deadline.Valid { - if a.DeadlineAt, err = parseStamp(deadline.String); err != nil { - return nil, err - } - } - out = append(out, a) - } - return out, rows.Err() -} - -// StartableRecords returns up to limit records waiting for a worker, the -// oldest per conversation, oldest first. While the hold -// marker stands there are none: the database would refuse their launch -// (ledger_hold.go). -func (l *Ledger) StartableRecords(ctx context.Context, limit int) ([]Record, error) { - return l.startable(ctx, "", nil, limit) -} - -// StartableFilter narrows StartableRecordsWhere to what the dispatcher can -// start now, in the query itself: a record it would skip must never take a -// place in the window, or a backlog it cannot start starves everything behind -// it. -type StartableFilter struct { - // Served are the served project ids the caller decided this query from, - // already narrowed to --project — one reading, not a promise about the - // file at any later moment. A record in any other project is not - // startable. Empty means nothing is. - Served []int64 - Limit int -} - -// StartableRecordsWhere is StartableRecords narrowed by f. -func (l *Ledger) StartableRecordsWhere(ctx context.Context, f StartableFilter) ([]Record, error) { - if len(f.Served) == 0 { - return nil, nil - } - buckets := slices.Clone(f.Served) - slices.Sort(buckets) - buckets = slices.Compact(buckets) - args := make([]any, 0, len(buckets)) - for _, bucket := range buckets { - args = append(args, bucket) - } - where := " AND e.bucket_id IN (" + placeholders(len(buckets)) + ")" - return l.startable(ctx, where, args, f.Limit) -} - -// startable runs the startable query with an extra condition. extra is built -// from this package's constants and placeholders only. -func (l *Ledger) startable(ctx context.Context, extra string, args []any, limit int) ([]Record, error) { - //nolint:gosec // G202: extra is this package's constants and placeholders, never a value - query := ` -SELECT MIN(e.id) FROM events e -WHERE ` + startableCondition + extra + ` - AND NOT EXISTS (SELECT 1 FROM tasks t WHERE t.ended_at IS NULL AND t.conversation_key = e.conversation_key) - AND NOT EXISTS (SELECT 1 FROM hold_marker) -GROUP BY e.conversation_key ORDER BY MIN(e.id) LIMIT ?` - rows, err := l.db.QueryContext(ctx, query, append(args, limit)...) - if err != nil { - return nil, fmt.Errorf("connector: startable records: %w", err) - } - var ids []int64 - for rows.Next() { - var id int64 - if err := rows.Scan(&id); err != nil { - _ = rows.Close() - return nil, err - } - ids = append(ids, id) - } - if err := rows.Close(); err != nil { - return nil, err - } - out := make([]Record, 0, len(ids)) - for _, id := range ids { - r, ok, err := l.Get(ctx, id) - if err != nil { - return nil, err - } - if ok { - out = append(out, r) - } - } - return out, nil -} - -// StrandedRecords counts the records waiting for a worker in a project -// connect.json no longer serves: work admitted while the project was served, -// which nothing will start until a person serves it again or discards the -// record. -// buckets is the run's --project scope: work in a project this run does not -// hear is another run's to dispatch, not stranded, so it is not counted. -func (l *Ledger) StrandedRecords(ctx context.Context, served []int64, buckets []int64) (int, error) { - var where strings.Builder - args := make([]any, 0, len(served)+len(buckets)) - for _, bucket := range served { - where.WriteString(" AND e.bucket_id <> ?") - args = append(args, bucket) - } - if len(buckets) > 0 { - where.WriteString(" AND e.bucket_id IN (" + placeholders(len(buckets)) + ")") - for _, bucket := range buckets { - args = append(args, bucket) - } - } - //nolint:gosec // G202: the condition is this package's constants and placeholders, never a value - query := `SELECT COUNT(*) FROM events e WHERE ` + startableCondition + where.String() - var n int - if err := l.db.QueryRowContext(ctx, query, args...).Scan(&n); err != nil { - return 0, fmt.Errorf("connector: count stranded records: %w", err) - } - return n, nil -} - -// RecordRefusal records one refusal on a live attempt, at the moment the -// driver made or observed it (driver's "Refusals"). An attempt that has ended -// is ErrNoLiveAttempt: its count was settled with it. -func (l *Ledger) RecordRefusal(ctx context.Context, attemptID string) error { - return retryBusy(func() error { - res, err := l.db.ExecContext(ctx, `UPDATE attempts SET refusals = refusals + 1 WHERE id = ? AND state <> 'ended'`, attemptID) - if err != nil { - return fmt.Errorf("connector: record refusal on %s: %w", attemptID, err) - } - n, err := res.RowsAffected() - if err != nil { - // The write is already committed; a driver that cannot say how - // many rows it touched is not a reason to count the refusal - // again at settlement. - return nil //nolint:nilerr // the write is committed, so the refusal is recorded - } - if n == 0 { - return fmt.Errorf("connector: record refusal on %s: %w", attemptID, ErrNoLiveAttempt) - } - return nil - }) -} - -// RecordProgress stamps the live attempt's last progress, which still-running -// reads. -func (l *Ledger) RecordProgress(ctx context.Context, attemptID string) error { - return retryBusy(func() error { - _, err := l.db.ExecContext(ctx, `UPDATE attempts SET progress_at = ? WHERE id = ? AND state <> 'ended'`, l.timestamp(), attemptID) - return err - }) -} - -// StillRunningTick is one still-running occurrence of a live attempt. -type StillRunningTick struct { - AttemptID string - TaskID int64 - // Occurrence counts from 1 per attempt. - Occurrence int - // ProgressAt is the attempt's last progress; zero when none was seen. - ProgressAt time.Time -} - -// StillRunning counts one more still-running occurrence for a live attempt, -// running the StillRunning hook in the same transaction. -func (l *Ledger) StillRunning(ctx context.Context, attemptID string) (StillRunningTick, error) { - var out StillRunningTick - err := retryBusy(func() error { - tx, err := l.db.BeginTx(ctx, nil) - if err != nil { - return fmt.Errorf("connector: begin still-running: %w", err) - } - defer func() { _ = tx.Rollback() }() - taskID, err := liveAttemptTask(ctx, tx, attemptID) - if err != nil { - return err - } - if _, err := tx.ExecContext(ctx, `UPDATE attempts SET still_running = still_running + 1 WHERE id = ?`, attemptID); err != nil { - return fmt.Errorf("connector: still-running %s: %w", attemptID, err) - } - tick := StillRunningTick{AttemptID: attemptID, TaskID: taskID} - var progress sql.NullString - if err := tx.QueryRowContext(ctx, `SELECT still_running, progress_at FROM attempts WHERE id = ?`, attemptID).Scan(&tick.Occurrence, &progress); err != nil { - return fmt.Errorf("connector: still-running %s: %w", attemptID, err) - } - if progress.Valid { - if tick.ProgressAt, err = parseStamp(progress.String); err != nil { - return err - } - } - if l.hooks.StillRunning != nil { - if err := l.hooks.StillRunning(ctx, tx, tick); err != nil { - return fmt.Errorf("connector: still-running hook for %s: %w", attemptID, err) - } - } - if err := tx.Commit(); err != nil { - return fmt.Errorf("connector: commit still-running %s: %w", attemptID, err) - } - out = tick - return nil - }) - return out, err -} - -// AdoptionCandidate is an event whose worker's report was lost after it -// acknowledged: settled unknown, delivered, and with no reply of its own. -type AdoptionCandidate struct { - TaskID int64 - EventID int64 - ReplyKind string - ReplyRecordingID int64 - // DeliveredAt is the event's ack_dispatch. - DeliveredAt time.Time - // NextAckAt is the first acknowledgement of a later instruction on the - // CONVERSATION, which may be on a task started after this one ended; - // zero when there is none. - NextAckAt time.Time - // AckID is the worker's own acknowledgement, which is never its reply - // however the clocks compare. - AckID int64 -} - -// AdoptionCandidates lists a settled task's events a reply could be adopted -// for. -func (l *Ledger) AdoptionCandidates(ctx context.Context, taskID int64) ([]AdoptionCandidate, error) { - rows, err := l.db.QueryContext(ctx, ` -SELECT te.event_id, e.reply_kind, e.reply_recording_id, te.delivered_at, te.ack_id, - -- The boundary is the conversation's, not this task's: settlement ends - -- the task and adoption runs after it, so the next instruction may - -- already be on a task of its own, and its reply is not this event's - -- (Copilot). - (SELECT MIN(later.delivered_at) FROM task_events later - JOIN tasks lt ON lt.id = later.task_id - WHERE lt.conversation_key = t.conversation_key - AND later.event_id > te.event_id AND later.delivered_at IS NOT NULL) -FROM task_events te JOIN events e ON e.id = te.event_id JOIN tasks t ON t.id = te.task_id -WHERE te.task_id = ? AND te.outcome = 'unknown' AND te.delivered_at IS NOT NULL - AND te.reply_id IS NULL AND te.adopted_reply_id IS NULL -ORDER BY te.event_id`, taskID) - if err != nil { - return nil, fmt.Errorf("connector: adoption candidates of task %d: %w", taskID, err) - } - defer func() { _ = rows.Close() }() - var out []AdoptionCandidate - for rows.Next() { - c := AdoptionCandidate{TaskID: taskID} - var delivered string - var next sql.NullString - var ackID sql.NullInt64 - if err := rows.Scan(&c.EventID, &c.ReplyKind, &c.ReplyRecordingID, &delivered, &ackID, &next); err != nil { - return nil, err - } - if c.DeliveredAt, err = parseStamp(delivered); err != nil { - return nil, err - } - if next.Valid { - if c.NextAckAt, err = parseStamp(next.String); err != nil { - return nil, err - } - } - if ackID.Valid { - c.AckID = ackID.Int64 - } - out = append(out, c) - } - return out, rows.Err() -} - -// AgentReply is a comment or chat line by the agent at a destination. -type AgentReply struct { - ID int64 - CreatedAt time.Time -} - -// AdoptableReply applies the adopted-reply rule: exactly one reply by the -// agent at the destination after the event's acknowledgement, not after a -// later instruction's acknowledgement, and not one of the connector's own -// lifecycle messages. -func AdoptableReply(c AdoptionCandidate, replies []AgentReply, lifecycle func(id int64) bool) (int64, bool) { - var found []int64 - for _, r := range replies { - if r.ID == c.AckID { - // The worker's acknowledgement is not the worker's reply, and - // the server's clock is not this machine's. - continue - } - if !r.CreatedAt.After(c.DeliveredAt) { - continue - } - if !c.NextAckAt.IsZero() && !r.CreatedAt.Before(c.NextAckAt) { - continue - } - if lifecycle != nil && lifecycle(r.ID) { - continue - } - found = append(found, r.ID) - } - if len(found) != 1 { - return 0, false - } - return found[0], true -} - -// AdoptReply links a reply to an event whose outcome is unknown. The outcome -// stays unknown (invariant 6). -func (l *Ledger) AdoptReply(ctx context.Context, taskID, eventID, replyID int64) error { - if replyID <= 0 { - return errors.New("connector: adopt a reply by its id") - } - return retryBusy(func() error { - res, err := l.db.ExecContext(ctx, ` -UPDATE task_events SET adopted_reply_id = ? -WHERE task_id = ? AND event_id = ? AND outcome = 'unknown' AND reply_id IS NULL AND adopted_reply_id IS NULL`, - replyID, taskID, eventID) - if err != nil { - return fmt.Errorf("connector: adopt reply for %d: %w", eventID, err) - } - if n, err := res.RowsAffected(); err != nil { - return err - } else if n == 0 { - return fmt.Errorf("connector: adopt reply for %d: the event is not unknown, or already has a reply", eventID) - } - return nil - }) -} - -// AttemptIDLength is how long an attempt id is: "att_" and 12 random bytes in -// hex. Anything that has to know whether a path built from one fits (a unix -// socket's 103 bytes) asks here rather than guessing. -const AttemptIDLength = 4 + 24 - -func newAttemptID() (string, error) { - raw := make([]byte, 12) - if _, err := rand.Read(raw); err != nil { - return "", fmt.Errorf("connector: attempt id: %w", err) - } - return "att_" + strings.ToLower(hex.EncodeToString(raw)), nil -} diff --git a/internal/connector/ledger_tasks_test.go b/internal/connector/ledger_tasks_test.go deleted file mode 100644 index 12ca6a8d7..000000000 --- a/internal/connector/ledger_tasks_test.go +++ /dev/null @@ -1,601 +0,0 @@ -package connector - -import ( - "context" - "errors" - "testing" - "time" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - "github.com/basecamp/basecamp-cli/internal/connector/admission" -) - -// admitOn writes an admitted record on a conversation key. -func admitOn(t *testing.T, ledger *Ledger, id int64, key string) { - t.Helper() - seenRecord(t, ledger, id) - _, err := ledger.Admission().Commit(context.Background(), admittedVerdict(id, 0, key)) - require.NoError(t, err) -} - -func launch(t *testing.T, ledger *Ledger, id int64) Launch { - t.Helper() - l, err := ledger.LaunchTask(context.Background(), LaunchSpec{EventID: id, Served: []int64{adapterBucketID}, Driver: "fake", Deadline: time.Hour}) - require.NoError(t, err) - return l -} - -type attemptRow struct { - State, StopReason string - SpawnFailed bool -} - -func readAttempt(t *testing.T, ledger *Ledger, id string) attemptRow { - t.Helper() - var r attemptRow - require.NoError(t, ledger.db.QueryRowContext(context.Background(), `SELECT state, stop_reason, spawn_failed FROM attempts WHERE id = ?`, id).Scan(&r.State, &r.StopReason, &r.SpawnFailed)) - return r -} - -type taskEventState struct { - Delivery, Outcome string - ExposedBy *string - Withdrawn *string - Adopted *int64 -} - -func readTaskEvent(t *testing.T, ledger *Ledger, taskID, eventID int64) taskEventState { - t.Helper() - var s taskEventState - require.NoError(t, ledger.db.QueryRowContext(context.Background(), `SELECT delivery, outcome, exposed_attempt_id, withdrawn_at, adopted_reply_id FROM task_events WHERE task_id = ? AND event_id = ?`, - taskID, eventID).Scan(&s.Delivery, &s.Outcome, &s.ExposedBy, &s.Withdrawn, &s.Adopted)) - return s -} - -// Ledger invariant 1: launching, the originating exposure and the record's -// move are one transaction. -func TestLaunchWritesLaunchingAndExposureTogether(t *testing.T) { - ledger := newTestLedger(t) - admitOn(t, ledger, 1, "recording:1") - admitOn(t, ledger, 2, "recording:1") - - l := launch(t, ledger, 1) - assert.Equal(t, []int64{1, 2}, l.EventIDs) - assert.Equal(t, "launching", readAttempt(t, ledger, l.AttemptID).State) - - origin := readTaskEvent(t, ledger, l.TaskID, 1) - assert.Equal(t, "exposed", origin.Delivery) - require.NotNil(t, origin.ExposedBy) - assert.Equal(t, l.AttemptID, *origin.ExposedBy) - assert.Equal(t, StateDispatched, getRecord(t, ledger, 1).State) - - follow := readTaskEvent(t, ledger, l.TaskID, 2) - assert.Equal(t, "admitted", follow.Delivery, "a joined follow-up is not exposed by the launch") - assert.Equal(t, StateDispatched, getRecord(t, ledger, 2).State, "a record on a task has left the queue") -} - -func TestALaunchHookFailureLeavesNothingWritten(t *testing.T) { - ledger := newTestLedger(t) - admitOn(t, ledger, 1, "recording:1") - ledger.SetHooks(Hooks{TaskLaunched: func(context.Context, Tx, Launch) error { return errors.New("outbox refused") }}) - - _, err := ledger.LaunchTask(context.Background(), LaunchSpec{EventID: 1, Served: []int64{adapterBucketID}, Driver: "fake"}) - require.Error(t, err) - assert.Equal(t, StateAdmitted, getRecord(t, ledger, 1).State) - var tasks, attempts int - require.NoError(t, ledger.db.QueryRowContext(context.Background(), `SELECT (SELECT COUNT(*) FROM tasks), (SELECT COUNT(*) FROM attempts)`).Scan(&tasks, &attempts)) - assert.Zero(t, tasks) - assert.Zero(t, attempts) -} - -// Ledger invariant 2. Two conversations run side by side now: nothing holds -// a directory, because there is no per-task directory to hold. -func TestOneLiveTaskPerConversationAndNoMore(t *testing.T) { - ledger := newTestLedger(t) - ctx := context.Background() - admitOn(t, ledger, 1, "recording:1") - launch(t, ledger, 1) - - admitOn(t, ledger, 3, "recording:3") - second, err := ledger.LaunchTask(ctx, LaunchSpec{EventID: 3, Served: []int64{adapterBucketID}, Driver: "fake"}) - require.NoError(t, err, "another conversation is another task, in the same directory") - assert.NotZero(t, second.TaskID) - - // The database holds the conversation rule too, whatever the code checks - // first. - _, err = ledger.db.ExecContext(ctx, `INSERT INTO tasks (token_sha256, created_at, conversation_key) VALUES ('y', 'now', 'recording:1')`) - require.Error(t, err) -} - -func TestAnEventIsOnAtMostOneLiveTask(t *testing.T) { - ledger := newTestLedger(t) - admitOn(t, ledger, 1, "recording:1") - l := launch(t, ledger, 1) - _, err := ledger.db.ExecContext(context.Background(), `INSERT INTO tasks (token_sha256, created_at) VALUES ('z', 'now')`) - require.NoError(t, err) - _, err = ledger.db.ExecContext(context.Background(), `INSERT INTO task_events (task_id, event_id) VALUES (?, 1)`, l.TaskID+1) - assert.ErrorContains(t, err, "UNIQUE constraint failed") -} - -// Ledger invariant 3. -func TestAnEndedTaskHasNoValidToken(t *testing.T) { - ledger := newTestLedger(t) - ctx := context.Background() - admitOn(t, ledger, 1, "recording:1") - l := launch(t, ledger, 1) - d, err := ledger.Dispatch(context.Background(), l.Token, adapterAgentID) - require.NoError(t, err) - _, ok, err := d.Get(ctx, 1) - require.NoError(t, err) - require.True(t, ok) - - _, err = ledger.EndAttempt(ctx, AttemptEnd{AttemptID: l.AttemptID, Stop: StopFinished}) - require.NoError(t, err) - _, _, err = d.Get(ctx, 1) - assert.ErrorIs(t, err, ErrTaskTokenRefused) - - admitOn(t, ledger, 2, "recording:2") - l2 := launch(t, ledger, 2) - _, err = ledger.db.ExecContext(context.Background(), `UPDATE tasks SET ended_at = 'now' WHERE id = ?`, l2.TaskID) - assert.ErrorContains(t, err, "superseded") -} - -// Ledger invariant 4: a proven spawn failure withdraws once. -func TestASpawnFailureIsRetriedOnceThenBlocked(t *testing.T) { - ledger := newTestLedger(t) - ctx := context.Background() - admitOn(t, ledger, 1, "recording:1") - - first := launch(t, ledger, 1) - s, err := ledger.EndAttempt(ctx, AttemptEnd{AttemptID: first.AttemptID, Stop: StopFailed, SpawnFailed: true}) - require.NoError(t, err) - require.Len(t, s.Events, 1) - assert.True(t, s.Events[0].Withdrawn) - assert.False(t, s.Events[0].Blocked) - assert.Equal(t, StateAdmitted, getRecord(t, ledger, 1).State) - assert.NotNil(t, readTaskEvent(t, ledger, first.TaskID, 1).Withdrawn) - - second := launch(t, ledger, 1) - s, err = ledger.EndAttempt(ctx, AttemptEnd{AttemptID: second.AttemptID, Stop: StopFailed, SpawnFailed: true}) - require.NoError(t, err) - assert.True(t, s.Events[0].Blocked) - record := getRecord(t, ledger, 1) - assert.Equal(t, StateBlocked, record.State) - assert.Equal(t, ReasonSpawnFailed, record.Reason) -} - -func TestNoAutomaticRetryBlocksTheFirstSpawnFailure(t *testing.T) { - ledger := newTestLedger(t) - admitOn(t, ledger, 1, "recording:1") - l := launch(t, ledger, 1) - _, err := ledger.EndAttempt(context.Background(), AttemptEnd{AttemptID: l.AttemptID, Stop: StopFailed, SpawnFailed: true, NoAutomaticRetry: true}) - require.NoError(t, err) - assert.Equal(t, StateBlocked, getRecord(t, ledger, 1).State) -} - -func TestAWorkerThatRanMakesItsExposedEventsUnknown(t *testing.T) { - ledger := newTestLedger(t) - ctx := context.Background() - admitOn(t, ledger, 1, "recording:1") - l := launch(t, ledger, 1) - require.NoError(t, ledger.MarkRunning(ctx, l.AttemptID, AttemptProcess{PID: 4242, PGID: 4242, SessionID: "s"})) - - s, err := ledger.EndAttempt(ctx, AttemptEnd{AttemptID: l.AttemptID, Stop: StopLost}) - require.NoError(t, err) - assert.Equal(t, OutcomeUnknown, s.Events[0].Outcome) - assert.False(t, s.Events[0].Withdrawn) - assert.Equal(t, StateCompleted, getRecord(t, ledger, 1).State) -} - -func TestASpawnFailureNeverWithdrawsAnExposureTheWorkerMade(t *testing.T) { - ledger := newTestLedger(t) - ctx := context.Background() - admitOn(t, ledger, 1, "recording:1") - admitOn(t, ledger, 2, "recording:1") - l := launch(t, ledger, 1) - d, err := ledger.Dispatch(context.Background(), l.Token, adapterAgentID) - require.NoError(t, err) - _, _, err = d.Get(ctx, 2) - require.NoError(t, err) - - s, err := ledger.EndAttempt(ctx, AttemptEnd{AttemptID: l.AttemptID, Stop: StopFailed, SpawnFailed: true}) - require.NoError(t, err) - byID := map[int64]SettledEvent{} - for _, e := range s.Events { - byID[e.EventID] = e - } - assert.True(t, byID[1].Withdrawn) - assert.Equal(t, OutcomeUnknown, byID[2].Outcome, "get_dispatch's exposure is not the launch's to withdraw") -} - -// Ledger invariant 5 and the sibling rule. -func TestSettlementKeepsReportsAndReturnsWhatWasNeverExposed(t *testing.T) { - ledger := newTestLedger(t) - ctx := context.Background() - for _, id := range []int64{1, 2, 3} { - admitOn(t, ledger, id, "recording:1") - } - l := launch(t, ledger, 1) - d, err := ledger.Dispatch(context.Background(), l.Token, adapterAgentID) - require.NoError(t, err) - reply := int64(99) - // A worker completes what it pulled (#736). - _, _, err = d.Get(ctx, 1) - require.NoError(t, err) - _, err = d.Complete(ctx, 1, Completion{Outcome: OutcomeFailed, ReplyID: &reply}) - require.NoError(t, err) - exposed, err := ledger.ExposeEvent(ctx, l.AttemptID, 2) - require.NoError(t, err) - require.True(t, exposed) - - s, err := ledger.EndAttempt(ctx, AttemptEnd{AttemptID: l.AttemptID, Stop: StopFinished}) - require.NoError(t, err) - byID := map[int64]SettledEvent{} - for _, e := range s.Events { - byID[e.EventID] = e - } - assert.Equal(t, OutcomeFailed, byID[1].Outcome, "a reported outcome stands, whatever the stop reason") - assert.True(t, byID[1].Reported) - assert.Equal(t, OutcomeUnknown, byID[2].Outcome) - assert.True(t, byID[3].Returned) - assert.Equal(t, StateAdmitted, getRecord(t, ledger, 3).State) - assert.Equal(t, "finished", readAttempt(t, ledger, l.AttemptID).StopReason) - - // A returned follow-up starts a task of its own. - startable, err := ledger.StartableRecords(ctx, 10) - require.NoError(t, err) - require.Len(t, startable, 1) - assert.Equal(t, int64(3), startable[0].ID) -} - -func TestExposeEventIsWrittenOnceAndOnlyForALiveAttempt(t *testing.T) { - ledger := newTestLedger(t) - ctx := context.Background() - admitOn(t, ledger, 1, "recording:1") - admitOn(t, ledger, 2, "recording:1") - l := launch(t, ledger, 1) - - exposed, err := ledger.ExposeEvent(ctx, l.AttemptID, 2) - require.NoError(t, err) - assert.True(t, exposed) - exposed, err = ledger.ExposeEvent(ctx, l.AttemptID, 2) - require.NoError(t, err) - assert.False(t, exposed) - - _, err = ledger.EndAttempt(ctx, AttemptEnd{AttemptID: l.AttemptID, Stop: StopShutdown}) - require.NoError(t, err) - _, err = ledger.ExposeEvent(ctx, l.AttemptID, 2) - assert.ErrorIs(t, err, ErrNoLiveAttempt) - _, err = ledger.EndAttempt(ctx, AttemptEnd{AttemptID: l.AttemptID, Stop: StopShutdown}) - assert.ErrorIs(t, err, ErrNoLiveAttempt) -} - -func TestJoinConversationTakesLaterFollowUpsOnlyWhileTheTaskIsLive(t *testing.T) { - ledger := newTestLedger(t) - ctx := context.Background() - admitOn(t, ledger, 1, "recording:1") - l := launch(t, ledger, 1) - admitOn(t, ledger, 2, "recording:1") - assert.Equal(t, StateQueued, getRecord(t, ledger, 2).State) - - joined, err := ledger.JoinConversation(ctx, l.TaskID, []int64{adapterBucketID}) - require.NoError(t, err) - assert.Equal(t, []int64{2}, joined) - pending, err := ledger.UnexposedEvents(ctx, l.TaskID) - require.NoError(t, err) - assert.Equal(t, []int64{2}, pending) - - _, err = ledger.EndAttempt(ctx, AttemptEnd{AttemptID: l.AttemptID, Stop: StopFinished}) - require.NoError(t, err) - admitOn(t, ledger, 3, "recording:1") - joined, err = ledger.JoinConversation(ctx, l.TaskID, []int64{adapterBucketID}) - require.NoError(t, err) - assert.Empty(t, joined) -} - -// Ledger invariant 7. -func TestAttemptStatesMoveForwardOnly(t *testing.T) { - ledger := newTestLedger(t) - ctx := context.Background() - admitOn(t, ledger, 1, "recording:1") - l := launch(t, ledger, 1) - require.NoError(t, ledger.MarkRunning(ctx, l.AttemptID, AttemptProcess{PID: 1234, PGID: 1234, SessionID: "s"})) - _, err := ledger.db.ExecContext(context.Background(), `UPDATE attempts SET state = 'launching' WHERE id = ?`, l.AttemptID) - assert.ErrorContains(t, err, "never goes back") - assert.ErrorIs(t, ledger.MarkRunning(ctx, l.AttemptID, AttemptProcess{}), ErrNoLiveAttempt) - _, err = ledger.EndAttempt(ctx, AttemptEnd{AttemptID: l.AttemptID, Stop: StopDeadline}) - require.NoError(t, err) - _, err = ledger.db.ExecContext(context.Background(), `UPDATE attempts SET stop_reason = 'finished', state = 'ended' WHERE id = ?`, l.AttemptID) - assert.Error(t, err, "an ended attempt's stop reason is not rewritten") -} - -func TestLiveAttemptsIncludesLaunching(t *testing.T) { - ledger := newTestLedger(t) - ctx := context.Background() - admitOn(t, ledger, 1, "recording:1") - l := launch(t, ledger, 1) - live, err := ledger.LiveAttempts(ctx) - require.NoError(t, err) - require.Len(t, live, 1) - assert.Equal(t, AttemptLaunching, live[0].State) - assert.Equal(t, l.AttemptID, live[0].AttemptID) -} - -func TestAHookFailureRollsTheTransitionBack(t *testing.T) { - t.Run("attempt ended", func(t *testing.T) { - ctx := context.Background() - ledger := newTestLedger(t) - admitOn(t, ledger, 1, "recording:1") - l := launch(t, ledger, 1) - ledger.SetHooks(Hooks{AttemptEnded: func(context.Context, Tx, Settlement) error { return errors.New("no") }}) - _, err := ledger.EndAttempt(ctx, AttemptEnd{AttemptID: l.AttemptID, Stop: StopFinished}) - require.Error(t, err) - assert.Equal(t, "launching", readAttempt(t, ledger, l.AttemptID).State) - assert.Equal(t, StateDispatched, getRecord(t, ledger, 1).State) - }) - t.Run("verdict", func(t *testing.T) { - ctx := context.Background() - ledger := newTestLedger(t) - seenRecord(t, ledger, 1) - ledger.SetHooks(Hooks{VerdictCommitted: func(context.Context, Tx, CommittedVerdict) error { return errors.New("no") }}) - _, err := ledger.Admission().Commit(ctx, admittedVerdict(1, 0, "recording:1")) - require.Error(t, err) - assert.Equal(t, StateSeen, getRecord(t, ledger, 1).State) - }) - t.Run("still running", func(t *testing.T) { - ctx := context.Background() - ledger := newTestLedger(t) - admitOn(t, ledger, 1, "recording:1") - l := launch(t, ledger, 1) - ledger.SetHooks(Hooks{StillRunning: func(context.Context, Tx, StillRunningTick) error { return errors.New("no") }}) - _, err := ledger.StillRunning(ctx, l.AttemptID) - require.Error(t, err) - ledger.SetHooks(Hooks{}) - tick, err := ledger.StillRunning(ctx, l.AttemptID) - require.NoError(t, err) - assert.Equal(t, 1, tick.Occurrence, "the refused occurrence was not counted") - }) -} - -// Ledger invariant 6. -func TestAnAdoptedReplyNeverMakesAnOutcome(t *testing.T) { - ledger := newTestLedger(t) - ctx := context.Background() - admitOn(t, ledger, 1, "recording:1") - l := launch(t, ledger, 1) - d, err := ledger.Dispatch(context.Background(), l.Token, adapterAgentID) - require.NoError(t, err) - // A worker acknowledges what it pulled (#736): get_dispatch first. - _, _, err = d.Get(ctx, 1) - require.NoError(t, err) - _, err = d.Ack(ctx, 1, nil) - require.NoError(t, err) - _, err = ledger.EndAttempt(ctx, AttemptEnd{AttemptID: l.AttemptID, Stop: StopLost}) - require.NoError(t, err) - - candidates, err := ledger.AdoptionCandidates(ctx, l.TaskID) - require.NoError(t, err) - require.Len(t, candidates, 1) - require.NoError(t, ledger.AdoptReply(ctx, l.TaskID, 1, 555)) - row := readTaskEvent(t, ledger, l.TaskID, 1) - assert.Equal(t, "unknown", row.Outcome) - require.NotNil(t, row.Adopted) - assert.Equal(t, int64(555), *row.Adopted) - assert.Error(t, ledger.AdoptReply(ctx, l.TaskID, 1, 556), "one adoption") -} - -func TestAdoptableReplyRule(t *testing.T) { - acked := time.Date(2026, 9, 17, 10, 0, 0, 0, time.UTC) - c := AdoptionCandidate{DeliveredAt: acked, NextAckAt: acked.Add(10 * time.Minute)} - at := func(m int) time.Time { return acked.Add(time.Duration(m) * time.Minute) } - - id, ok := AdoptableReply(c, []AgentReply{{ID: 1, CreatedAt: at(-1)}, {ID: 2, CreatedAt: at(1)}, {ID: 3, CreatedAt: at(11)}}, nil) - assert.True(t, ok) - assert.Equal(t, int64(2), id, "only a reply after the ack and before a later instruction's ack") - - _, ok = AdoptableReply(c, []AgentReply{{ID: 2, CreatedAt: at(1)}, {ID: 4, CreatedAt: at(2)}}, nil) - assert.False(t, ok, "two candidates adopt nothing") - - _, ok = AdoptableReply(c, []AgentReply{{ID: 2, CreatedAt: at(1)}}, func(id int64) bool { return id == 2 }) - assert.False(t, ok, "a lifecycle message is never adopted") -} - -// A follow-up on the task's conversation joins it. The conversation is the -// whole of the test now: there is no second thing for it to match. -func TestAFollowUpOnTheConversationJoinsTheTask(t *testing.T) { - ledger := newTestLedger(t) - ctx := context.Background() - admitOn(t, ledger, 1, "recording:1") - l := launch(t, ledger, 1) - admitOn(t, ledger, 2, "recording:1") - - joined, err := ledger.JoinConversation(ctx, l.TaskID, []int64{adapterBucketID}) - require.NoError(t, err) - assert.Equal(t, []int64{2}, joined) -} - -// Copilot on #765: the originating record is held to the served set too, not -// only the records that join it. -// -// The ledger launches only what the set it is given covers, and a launch -// naming no set authorizes nothing rather than defaulting open. For the -// dispatcher that is belt and braces — one pass takes a single reading and -// hands the same slice to the startable query and to LaunchSpec, so the -// record is always in it — and it is what makes any other caller fail closed. -// -// It is not a fix for the window between reading connect.json and committing -// the launch: an unserve landing there still starts this one task, and the -// comment here used to say otherwise. -func TestALaunchIsRefusedForAProjectTheSpecDoesNotServe(t *testing.T) { - ledger := newTestLedger(t) - ctx := context.Background() - admitOn(t, ledger, 1, "recording:1") - - _, err := ledger.LaunchTask(ctx, LaunchSpec{EventID: 1, Served: []int64{adapterBucketID + 1}, Driver: "fake"}) - assert.ErrorIs(t, err, ErrNotStartable, "another project's served set does not authorize this record") - - _, err = ledger.LaunchTask(ctx, LaunchSpec{EventID: 1, Driver: "fake"}) - assert.ErrorIs(t, err, ErrNotStartable, "and a launch that names no served project authorizes nothing") - - assert.Equal(t, StateAdmitted, getRecord(t, ledger, 1).State, "nothing was written either time") - - l, err := ledger.LaunchTask(ctx, LaunchSpec{EventID: 1, Served: []int64{adapterBucketID}, Driver: "fake"}) - require.NoError(t, err, "served, and it launches") - assert.NotZero(t, l.TaskID) -} - -// Copilot on #765: a task is authorized against one project, so nothing from -// another project joins it, however the conversation is shared. -// -// A conversation key is the recording's or the Campfire's, never the -// bucket's, so two records on one conversation can sit in two projects — a -// recording moved between them is the ordinary way. The record carries the -// `served` bit admission wrote, which says the project was served *then*; -// joining on that alone would expose an event through a task authorized -// against a different project, and would go on doing it after the operator -// stopped serving the second one. -func TestAFollowUpInAnotherProjectDoesNotJoinTheTask(t *testing.T) { - ledger := newTestLedger(t) - ctx := context.Background() - admitOn(t, ledger, 1, "recording:1") - l := launch(t, ledger, 1) - - // Event 2 is on the same conversation and in another project. - seenRecord(t, ledger, 2) - _, err := ledger.ledgerCommitWithBucket(admittedVerdict(2, 0, "recording:1"), adapterBucketID+1) - require.NoError(t, err) - - joined, err := ledger.JoinConversation(ctx, l.TaskID, []int64{adapterBucketID, adapterBucketID + 1}) - require.NoError(t, err) - assert.Empty(t, joined, "the task is the originating project's; another project's record is not handed to its worker") - assert.Equal(t, StateQueued, getRecord(t, ledger, 2).State, - "and it waits behind the live task on its conversation for a task of its own, rather than riding along in this one") -} - -// And the served set is read now, not as it was when the record was -// admitted: a project the operator has stopped serving stops feeding the -// live task on its conversation. -func TestAFollowUpInAProjectNoLongerServedDoesNotJoinTheTask(t *testing.T) { - ledger := newTestLedger(t) - ctx := context.Background() - admitOn(t, ledger, 1, "recording:1") - l := launch(t, ledger, 1) - admitOn(t, ledger, 2, "recording:1") - - joined, err := ledger.JoinConversation(ctx, l.TaskID, nil) - require.NoError(t, err) - assert.Empty(t, joined, "no project is served now") - - joined, err = ledger.JoinConversation(ctx, l.TaskID, []int64{adapterBucketID + 5}) - require.NoError(t, err) - assert.Empty(t, joined, "and the task's own project is not among those served") - - joined, err = ledger.JoinConversation(ctx, l.TaskID, []int64{adapterBucketID}) - require.NoError(t, err) - assert.Equal(t, []int64{2}, joined, "served again, and the follow-up joins") -} - -// Review r2: work in a project no longer served is counted, not silently -// stuck. -func TestStrandedRecordsCountsWorkInUnservedProjects(t *testing.T) { - ledger := newTestLedger(t) - ctx := context.Background() - admitOn(t, ledger, 1, "recording:1") - seenRecord(t, ledger, 2) - _, err := ledger.ledgerCommitWithBucket(admittedVerdict(2, 0, "recording:2"), adapterBucketID+1) - require.NoError(t, err) - - stranded, err := ledger.StrandedRecords(ctx, []int64{adapterBucketID}, nil) - require.NoError(t, err) - assert.Equal(t, 1, stranded, "the record in a project connect.json no longer serves") - - stranded, err = ledger.StrandedRecords(ctx, []int64{adapterBucketID, adapterBucketID + 1}, nil) - require.NoError(t, err) - assert.Zero(t, stranded, "both projects served") - - stranded, err = ledger.StrandedRecords(ctx, []int64{adapterBucketID + 5}, []int64{adapterBucketID + 5}) - require.NoError(t, err) - assert.Zero(t, stranded, "work in a project this run does not hear is another run's, not stranded") -} - -// Review r2: the worker's acknowledgement is never adopted as its reply. -func TestAnAcknowledgementIsNeverAdoptedAsTheReply(t *testing.T) { - acked := time.Date(2026, 9, 17, 10, 0, 0, 0, time.UTC) - c := AdoptionCandidate{DeliveredAt: acked, AckID: 7} - // The ack comment's server timestamp is after this machine's - // delivered_at, so time alone would adopt it. - _, ok := AdoptableReply(c, []AgentReply{{ID: 7, CreatedAt: acked.Add(time.Second)}}, nil) - assert.False(t, ok) -} - -// The refusal rule (driver's "Refusals"): a refusal is on the attempt's row -// the moment it is recorded, and settled with the attempt. -func TestARefusalIsRecordedOnTheLiveAttemptAndSettledWithIt(t *testing.T) { - ledger := newTestLedger(t) - admitOn(t, ledger, 1, "recording:1") - l := launch(t, ledger, 1) - refusals := func() int { - var n int - require.NoError(t, ledger.db.QueryRowContext(context.Background(), `SELECT refusals FROM attempts WHERE id = ?`, l.AttemptID).Scan(&n)) - return n - } - - require.NoError(t, ledger.RecordRefusal(context.Background(), l.AttemptID)) - require.NoError(t, ledger.RecordRefusal(context.Background(), l.AttemptID)) - assert.Equal(t, 2, refusals(), "written as they happen, not at the end") - - _, err := ledger.EndAttempt(context.Background(), AttemptEnd{AttemptID: l.AttemptID, Stop: StopLost, UnrecordedRefusals: 1}) - require.NoError(t, err) - assert.Equal(t, 3, refusals(), "what could not be written then is settled with the attempt") - - assert.ErrorIs(t, ledger.RecordRefusal(context.Background(), l.AttemptID), ErrNoLiveAttempt) - assert.Equal(t, 3, refusals(), "an ended attempt's count is final") -} - -// Copilot: settlement ends a task and adoption runs after it, so the next -// instruction on the conversation can already be on a task of its own. Its -// acknowledgement still bounds what the old event may adopt. -func TestTheAdoptionBoundaryIsTheConversationsNotTheTasks(t *testing.T) { - ledger := newTestLedger(t) - ctx := context.Background() - admitOn(t, ledger, 1, "recording:1") - first := launch(t, ledger, 1) - d, err := ledger.Dispatch(ctx, first.Token, adapterAgentID) - require.NoError(t, err) - _, _, err = d.Get(ctx, 1) - require.NoError(t, err) - _, err = d.Ack(ctx, 1, nil) - require.NoError(t, err) - _, err = ledger.EndAttempt(ctx, AttemptEnd{AttemptID: first.AttemptID, Stop: StopLost}) - require.NoError(t, err) - - // The next instruction on the same conversation, on a task of its own. - admitOn(t, ledger, 2, "recording:1") - second := launch(t, ledger, 2) - d2, err := ledger.Dispatch(ctx, second.Token, adapterAgentID) - require.NoError(t, err) - _, _, err = d2.Get(ctx, 2) - require.NoError(t, err) - _, err = d2.Ack(ctx, 2, nil) - require.NoError(t, err) - - candidates, err := ledger.AdoptionCandidates(ctx, first.TaskID) - require.NoError(t, err) - require.Len(t, candidates, 1) - assert.False(t, candidates[0].NextAckAt.IsZero(), - "the later task's acknowledgement bounds what the lost event may adopt") - assert.False(t, candidates[0].NextAckAt.Before(candidates[0].DeliveredAt)) -} - -// ledgerCommitWithBucket admits v and moves its record to another bucket, so -// tests can have several served projects. -func (l *Ledger) ledgerCommitWithBucket(v admission.Verdict, bucket int64) (admission.State, error) { - state, err := l.Admission().Commit(context.Background(), v) - if err != nil { - return state, err - } - _, err = l.db.ExecContext(context.Background(), `UPDATE events SET bucket_id = ? WHERE id = ?`, bucket, v.EventID) - return state, err -} diff --git a/internal/connector/ledger_test.go b/internal/connector/ledger_test.go index 50a1eb5d7..12cd99842 100644 --- a/internal/connector/ledger_test.go +++ b/internal/connector/ledger_test.go @@ -296,7 +296,7 @@ func TestASecondNameForALiveLedgerIsRefusedWithoutOpeningIt(t *testing.T) { link := filepath.Join(dir, "hard.db") require.NoError(t, os.Link(path, link)) - _, err = OpenExistingLedger(context.Background(), link) + _, err = OpenLedgerReadOnly(context.Background(), link) // Refused, because SQLite names its write-ahead log after the path and // one file under two names is two logs — and refused before the check diff --git a/internal/connector/operator_invariants_test.go b/internal/connector/operator_invariants_test.go index 94f938e4f..9b2fce483 100644 --- a/internal/connector/operator_invariants_test.go +++ b/internal/connector/operator_invariants_test.go @@ -3,7 +3,6 @@ package connector import ( "context" "path/filepath" - "strconv" "testing" "time" @@ -30,13 +29,6 @@ func opAdmit(t *testing.T, l *Ledger, id int64, key string) RecordState { return RecordState(state) } -func launchOf(t *testing.T, l *Ledger, id int64) Launch { - t.Helper() - launch, err := l.LaunchTask(context.Background(), LaunchSpec{EventID: id, Served: []int64{adapterBucketID}, Driver: "claude"}) - require.NoError(t, err) - return launch -} - func stateOf(t *testing.T, l *Ledger, id int64) RecordState { t.Helper() return getRecord(t, l, id).State @@ -49,268 +41,8 @@ func decisionsFor(t *testing.T, l *Ledger, id int64) int { return n } -// unknownOutcome takes a fresh record to completed(unknown): launched, running, -// lost. -func unknownOutcome(t *testing.T, l *Ledger, id int64) Launch { - t.Helper() - ctx := context.Background() - require.Equal(t, StateAdmitted, opAdmit(t, l, id, "recording:"+strconv.FormatInt(id, 10))) - launch := launchOf(t, l, id) - require.NoError(t, l.MarkRunning(ctx, launch.AttemptID, AttemptProcess{PID: 4242, PGID: 4242, StartedAt: time.Now()})) - _, err := l.EndAttempt(ctx, AttemptEnd{AttemptID: launch.AttemptID, Stop: StopLost}) - require.NoError(t, err) - require.Equal(t, StateCompleted, stateOf(t, l, id)) - return launch -} - -// Copilot on #765: a redispatch is authorized by the served projects its -// caller read from connect.json, not by the bit admission wrote when it -// decided the record. The caller reads that set under connect.json's lock -// and holds the lock until this write, so an unserve cannot land between -// the two (TestRedispatchReadsTheServedSetUnderTheLock, in commands). -// -// Without this the command reported success and admitted the record, and the -// dispatcher then refused to launch it — so the person was told their -// redispatch worked while the record sat stranded, with no holding reply and -// nothing else coming. -func TestRedispatchIsRefusedForAProjectNoLongerServed(t *testing.T) { - l := newTestLedger(t) - ctx := context.Background() - unknownOutcome(t, l, 1) - - _, err := l.Redispatch(ctx, 1, opBy, []int64{adapterBucketID + 1}) - require.ErrorIs(t, err, ErrDecisionRefused, "another project's served set does not authorize this record") - assert.Contains(t, err.Error(), "connect.json serves") - _, err = l.Redispatch(ctx, 1, opBy, nil) - require.ErrorIs(t, err, ErrDecisionRefused, "and serving nothing authorizes nothing") - - assert.Equal(t, StateCompleted, stateOf(t, l, 1), "the record is left where it was, both times") - assert.Zero(t, decisionsFor(t, l, 1), "and nothing was recorded as decided") - - got, err := l.Redispatch(ctx, 1, opBy, []int64{adapterBucketID}) - require.NoError(t, err, "served, and the redispatch lands") - assert.True(t, got.Admitted) -} - -// Done when: redispatch of completed(unknown) admits the record, supersedes -// the task's token and records who authorized it. -func TestRedispatchAdmitsAnUnknownOutcome(t *testing.T) { - l := newTestLedger(t) - ctx := context.Background() - launch := unknownOutcome(t, l, 1) - - got, err := l.Redispatch(ctx, 1, opBy, []int64{adapterBucketID}) - require.NoError(t, err) - assert.True(t, got.Admitted) - assert.Equal(t, StateAdmitted, got.State) - assert.Equal(t, OutcomeUnknown, got.FromOutcome) - assert.Equal(t, 1, decisionsFor(t, l, 1)) - - var by string - require.NoError(t, l.db.QueryRowContext(context.Background(), `SELECT authorized_by FROM events WHERE id = 1`).Scan(&by)) - assert.Equal(t, opBy, by) - _, err = l.Dispatch(ctx, launch.Token, adapterAgentID) - assert.ErrorIs(t, err, ErrTaskTokenRefused, "the replaced task's token is refused") - - startable, err := l.StartableRecords(ctx, 10) - require.NoError(t, err) - require.Len(t, startable, 1) - second := launchOf(t, l, 1) - assert.NotEqual(t, launch.TaskID, second.TaskID) -} - -// Done when: redispatch of completed(failed) admits it. -func TestRedispatchAdmitsAFailedOutcome(t *testing.T) { - l := newTestLedger(t) - ctx := context.Background() - require.Equal(t, StateAdmitted, opAdmit(t, l, 1, "recording:1")) - launch := launchOf(t, l, 1) - d, err := l.Dispatch(ctx, launch.Token, adapterAgentID) - require.NoError(t, err) - pulled(t, d, 1) - _, err = d.Complete(ctx, 1, Completion{Outcome: OutcomeFailed}) - require.NoError(t, err) - _, err = l.EndAttempt(ctx, AttemptEnd{AttemptID: launch.AttemptID, Stop: StopFinished}) - require.NoError(t, err) - - got, err := l.Redispatch(ctx, 1, opBy, []int64{adapterBucketID}) - require.NoError(t, err) - assert.True(t, got.Admitted) - assert.Equal(t, OutcomeFailed, got.FromOutcome) - assert.Equal(t, StateAdmitted, stateOf(t, l, 1)) -} - -// Invariant 5: an event whose task is still live is not admitted until the -// task ends, so two workers never run for it. -func TestRedispatchOnALiveTaskWaitsForItsEnd(t *testing.T) { - l := newTestLedger(t) - ctx := context.Background() - require.Equal(t, StateAdmitted, opAdmit(t, l, 1, "recording:9")) - require.Equal(t, StateQueued, opAdmit(t, l, 2, "recording:9")) - launch := launchOf(t, l, 1) - started := time.Now().Add(-time.Minute).UTC() - require.NoError(t, l.MarkRunning(ctx, launch.AttemptID, AttemptProcess{PID: 4242, PGID: 4242, StartedAt: started})) - d, err := l.Dispatch(ctx, launch.Token, adapterAgentID) - require.NoError(t, err) - pulled(t, d, 1) - _, err = d.Complete(ctx, 1, Completion{Outcome: OutcomeFailed}) - require.NoError(t, err) - - got, err := l.Redispatch(ctx, 1, opBy, []int64{adapterBucketID}) - require.NoError(t, err) - assert.True(t, got.Pending) - assert.False(t, got.Admitted) - assert.Equal(t, launch.TaskID, got.SupersededTaskID) - require.NotNil(t, got.Worker, "the live worker is handed back to be terminated") - assert.Equal(t, 4242, got.Worker.PGID) - assert.Equal(t, StateCompleted, stateOf(t, l, 1)) - - _, _, err = d.Get(ctx, 2) - assert.ErrorIs(t, err, ErrTaskTokenRefused, "the old worker is refused at once") - _, err = l.LaunchTask(ctx, LaunchSpec{EventID: 1, Served: []int64{adapterBucketID}, Driver: "claude"}) - assert.ErrorIs(t, err, ErrNotStartable, "no second task while the first is live") - startable, err := l.StartableRecords(ctx, 10) - require.NoError(t, err) - assert.Empty(t, startable) - - _, err = l.Redispatch(ctx, 1, opBy, []int64{adapterBucketID}) - assert.ErrorIs(t, err, ErrDecisionRefused, "a second redispatch while the first waits") - - _, err = l.EndAttempt(ctx, AttemptEnd{AttemptID: launch.AttemptID, Stop: StopLost}) - require.NoError(t, err) - assert.Equal(t, StateAdmitted, stateOf(t, l, 1), "admitted in the transaction that ended the task") - var consumed bool - require.NoError(t, l.db.QueryRowContext(context.Background(), `SELECT redispatch_decision IS NULL FROM events WHERE id = 1`).Scan(&consumed)) - assert.True(t, consumed, "the task's end consumed the redispatch") - second := launchOf(t, l, 1) - assert.NotEqual(t, launch.TaskID, second.TaskID) -} - -// Invariant 6: refused for succeeded, discarded and anything live, and a -// refusal writes nothing. -// -//nolint:contextcheck // subtests build their fixtures on background contexts -func TestRedispatchRefusesWhatItMustNotRun(t *testing.T) { - ctx := context.Background() - cases := map[string]func(t *testing.T, l *Ledger){ - "seen": func(t *testing.T, l *Ledger) { seenRecord(t, l, 1) }, - "admitted": func(t *testing.T, l *Ledger) { opAdmit(t, l, 1, "recording:1") }, - "queued": func(t *testing.T, l *Ledger) { - opAdmit(t, l, 7, "recording:1") - require.Equal(t, StateQueued, opAdmit(t, l, 1, "recording:1")) - }, - "dispatched": func(t *testing.T, l *Ledger) { - opAdmit(t, l, 1, "recording:1") - launchOf(t, l, 1) - }, - "discarded": func(t *testing.T, l *Ledger) { - seenRecord(t, l, 1) - require.NoError(t, l.SetState(ctx, 1, StateDiscarded, "untrusted_author")) - }, - "succeeded": func(t *testing.T, l *Ledger) { - opAdmit(t, l, 1, "recording:1") - launch := launchOf(t, l, 1) - d, err := l.Dispatch(ctx, launch.Token, adapterAgentID) - require.NoError(t, err) - pulled(t, d, 1) - _, err = d.Complete(ctx, 1, Completion{Outcome: OutcomeSucceeded}) - require.NoError(t, err) - _, err = l.EndAttempt(ctx, AttemptEnd{AttemptID: launch.AttemptID, Stop: StopFinished}) - require.NoError(t, err) - }, - } - for name, arrange := range cases { - t.Run(name, func(t *testing.T) { - l := newTestLedger(t) - arrange(t, l) - before := getRecord(t, l, 1) - - _, err := l.Redispatch(ctx, 1, opBy, []int64{adapterBucketID}) - require.ErrorIs(t, err, ErrDecisionRefused) - after := getRecord(t, l, 1) - assert.Equal(t, before.State, after.State) - assert.Equal(t, before.Revision, after.Revision) - assert.Zero(t, decisionsFor(t, l, 1)) - }) - } -} - -// Done when: a blocked record keeps its state with the authorization -// recorded, and is admitted the moment its prerequisite succeeds. -func TestRedispatchOfABlockedRecordRerunsItsPrerequisite(t *testing.T) { - l := newTestLedger(t) - ctx := context.Background() - seenRecord(t, l, 1) - _, err := l.Admission().Commit(ctx, blockedVerdict(1, 0, admission.ReasonReadFailed)) - require.NoError(t, err) - // A hold before the redispatch: the authorization is what lets it through. - _, err = l.SetHold(ctx, opBy, HoldByOperator) - require.NoError(t, err) - - got, err := l.Redispatch(ctx, 1, opBy, []int64{adapterBucketID}) - require.NoError(t, err) - assert.True(t, got.Rerun) - assert.True(t, got.Held) - assert.Equal(t, StateBlocked, got.State) - ids, err := l.AuthorizedBlocked(ctx, 10) - require.NoError(t, err) - assert.Equal(t, []int64{1}, ids) - - ev, ok, err := l.Admission().LoadUndecided(ctx, 1) - require.NoError(t, err) - require.True(t, ok) - v := admittedVerdict(1, ev.Revision, "recording:1") - written, err := l.Admission().Commit(ctx, v) - require.NoError(t, err) - assert.Equal(t, admission.StateAdmitted, written, "authorized, so admitted though tagged for review") - - // Under the hold it is authorized and not launched. - _, err = l.LaunchTask(ctx, LaunchSpec{EventID: 1, Served: []int64{adapterBucketID}, Driver: "claude"}) - require.Error(t, err) - assert.Contains(t, err.Error(), "held") - _, err = l.Release(ctx, opBy) - require.NoError(t, err) - launchOf(t, l, 1) -} - -// Done when: a held record with its snapshot, in a served project, is -// admitted at once. -func TestRedispatchAdmitsAHeldRecord(t *testing.T) { - l := newTestLedger(t) - ctx := context.Background() - opAdmit(t, l, 1, "recording:1") - res, err := l.SetHold(ctx, opBy, HoldByOperator) - require.NoError(t, err) - assert.Equal(t, 1, res.Held) - require.Equal(t, StateHeld, stateOf(t, l, 1)) - - got, err := l.Redispatch(ctx, 1, opBy, []int64{adapterBucketID}) - require.NoError(t, err) - assert.True(t, got.Admitted) - assert.Equal(t, StateAdmitted, stateOf(t, l, 1)) -} - -// A held record over a blocking reason runs what blocked it again. -func TestRedispatchOfARecordHeldOverAReasonRerunsIt(t *testing.T) { - l := newTestLedger(t) - ctx := context.Background() - opAdmit(t, l, 1, "recording:1") - _, err := l.SetHold(ctx, opBy, HoldByOperator) - require.NoError(t, err) - _, err = l.db.ExecContext(context.Background(), `UPDATE events SET reason = 'no_route' WHERE id = 1`) - require.NoError(t, err) - - got, err := l.Redispatch(ctx, 1, opBy, []int64{adapterBucketID}) - require.NoError(t, err) - assert.True(t, got.Rerun) - record := getRecord(t, l, 1) - assert.Equal(t, StateBlocked, record.State) - assert.Equal(t, "no_route", record.Reason) - assert.Nil(t, record.Decision.Snapshot, "a blocked record carries no content") -} - -// Done when: a review-tagged seen record becomes held, not dispatched — and -// the verdict says so, so no guard acknowledgement is called for. +// Done when: a review-tagged seen record becomes held, not admitted, and the +// verdict says so. func TestInvariant1AReviewTaggedSeenRecordIsHeldNotDispatched(t *testing.T) { l := newTestLedger(t) ctx := context.Background() @@ -321,23 +53,14 @@ func TestInvariant1AReviewTaggedSeenRecordIsHeldNotDispatched(t *testing.T) { state := opAdmit(t, l, 1, "recording:1") assert.Equal(t, StateHeld, state) assert.Equal(t, StateHeld, stateOf(t, l, 1)) - startable, err := l.StartableRecords(ctx, 10) - require.NoError(t, err) - assert.Empty(t, startable) - intents, err := l.Intents(ctx, IntentFilter{EventID: 1}) - require.NoError(t, err) - assert.Empty(t, intents, "a held record calls for no guard acknowledgement") _, err = l.Release(ctx, opBy) require.NoError(t, err) assert.Equal(t, StateHeld, stateOf(t, l, 1), "held records stay held on release") - startable, err = l.StartableRecords(ctx, 10) - require.NoError(t, err) - assert.Empty(t, startable) } -// Records of the generation a hold opened are not tagged, and dispatch once -// the hold is released. +// Records of the generation a hold opened are not tagged: they are admitted +// under the hold, and stay admitted once it is released. func TestANewGenerationIsNotTagged(t *testing.T) { l := newTestLedger(t) ctx := context.Background() @@ -346,26 +69,38 @@ func TestANewGenerationIsNotTagged(t *testing.T) { assert.Equal(t, StateAdmitted, opAdmit(t, l, 1, "recording:1")) _, err = l.Release(ctx, opBy) require.NoError(t, err) - launchOf(t, l, 1) + assert.Equal(t, StateAdmitted, stateOf(t, l, 1)) + var review bool + require.NoError(t, l.db.QueryRowContext(ctx, `SELECT review FROM events WHERE id = 1`).Scan(&review)) + assert.False(t, review) } -// Invariant 1, for every path to admitted: a tagged sibling a task's end -// returns is held. -func TestInvariant1ATaggedSiblingReturnedByATaskIsHeld(t *testing.T) { +// A record an older build's redispatch authorized is a person's decision the +// review tag does not override: admission writes it admitted though a later +// hold tagged it, because the trigger holds only a tagged record nobody +// authorized. +func TestAnAuthorizedRecordIsAdmittedThoughTagged(t *testing.T) { l := newTestLedger(t) ctx := context.Background() - opAdmit(t, l, 1, "recording:9") - require.Equal(t, StateQueued, opAdmit(t, l, 2, "recording:9")) - launch := launchOf(t, l, 1) - _, err := l.SetHold(ctx, opBy, HoldByOperator) + seenRecord(t, l, 1) + _, err := l.Admission().Commit(ctx, blockedVerdict(1, 0, admission.ReasonReadFailed)) + require.NoError(t, err) + _, err = l.SetHold(ctx, opBy, HoldByOperator) + require.NoError(t, err) + // As the removed Redispatch authorized a blocked record: never dated + // before its block, and due now. + now := l.timestamp() + _, err = l.db.ExecContext(ctx, ` +UPDATE events SET authorized_at = MAX(?, COALESCE(blocked_at, '')), authorized_by = ?, next_retry_at = ? +WHERE id = 1 AND state = 'blocked'`, now, opBy, now) require.NoError(t, err) - require.Equal(t, StateDispatched, stateOf(t, l, 2)) - settlement, err := l.EndAttempt(ctx, AttemptEnd{AttemptID: launch.AttemptID, Stop: StopShutdown}) + ev, ok, err := l.Admission().LoadUndecided(ctx, 1) + require.NoError(t, err) + require.True(t, ok) + written, err := l.Admission().Commit(ctx, admittedVerdict(1, ev.Revision, "recording:1")) require.NoError(t, err) - require.Len(t, settlement.Events, 2) - assert.True(t, settlement.Events[1].Returned) - assert.Equal(t, StateHeld, stateOf(t, l, 2)) + assert.Equal(t, admission.StateAdmitted, written) } // Invariant 1, at the database: any write of admitted onto a tagged, @@ -382,8 +117,7 @@ func TestInvariant1TheDatabaseHoldsATaggedRecord(t *testing.T) { assert.Equal(t, StateHeld, stateOf(t, l, 1)) } -// Done when: a held ledger survives restart until release, and the database -// refuses a launch while it stands. +// Done when: a held ledger survives restart until release. func TestInvariant2AHeldLedgerSurvivesRestartUntilRelease(t *testing.T) { path := filepath.Join(t.TempDir(), "state", LedgerFile) ctx := context.Background() @@ -396,26 +130,16 @@ func TestInvariant2AHeldLedgerSurvivesRestartUntilRelease(t *testing.T) { l, err = OpenLedger(path) require.NoError(t, err) t.Cleanup(func() { _ = l.Close() }) - held, err := l.Held(ctx) + _, held, err := l.HoldMarker(ctx) require.NoError(t, err) assert.True(t, held) - // A record of the new generation, which a person need not review, still - // does not launch while the marker stands. - assert.Equal(t, StateAdmitted, opAdmit(t, l, 1, "recording:1")) - startable, err := l.StartableRecords(ctx, 10) - require.NoError(t, err) - assert.Empty(t, startable, "the dispatcher is offered nothing while the hold stands") - _, err = l.LaunchTask(ctx, LaunchSpec{EventID: 1, Served: []int64{adapterBucketID}, Driver: "claude"}) - require.Error(t, err) - assert.Equal(t, StateAdmitted, stateOf(t, l, 1), "the refused launch rolled back") released, err := l.Release(ctx, opBy) require.NoError(t, err) assert.True(t, released.Released) - held, err = l.Held(ctx) + _, held, err = l.HoldMarker(ctx) require.NoError(t, err) assert.False(t, held) - launchOf(t, l, 1) } // Invariant 4: a terminal record leaves its state only with a decision @@ -426,14 +150,14 @@ func TestInvariant4TheDatabaseRefusesATerminalMoveWithoutADecision(t *testing.T) ctx := context.Background() t.Run("completed to admitted without a redispatch", func(t *testing.T) { l := newTestLedger(t) - unknownOutcome(t, l, 1) + olderUnknownOutcome(t, l, 1) _, err := l.db.ExecContext(ctx, `UPDATE events SET state = 'admitted' WHERE id = 1`) require.Error(t, err) assert.Contains(t, err.Error(), "terminal") }) t.Run("a redispatch of a success", func(t *testing.T) { l := newTestLedger(t) - unknownOutcome(t, l, 1) + olderUnknownOutcome(t, l, 1) decision := rawDecision(t, l, 1, "redispatch", "9999-01-01T00:00:00.000000000Z") _, err := l.db.ExecContext(ctx, `UPDATE task_events SET outcome = 'succeeded' WHERE event_id = 1`) require.NoError(t, err) @@ -444,7 +168,7 @@ func TestInvariant4TheDatabaseRefusesATerminalMoveWithoutADecision(t *testing.T) }) t.Run("a redispatch naming another event's decision", func(t *testing.T) { l := newTestLedger(t) - unknownOutcome(t, l, 1) + olderUnknownOutcome(t, l, 1) seenRecord(t, l, 2) decision := rawDecision(t, l, 2, "redispatch", "9999-01-01T00:00:00.000000000Z") _, err := l.db.ExecContext(ctx, `UPDATE events SET redispatch_decision = ? WHERE id = 1`, decision) @@ -454,7 +178,7 @@ func TestInvariant4TheDatabaseRefusesATerminalMoveWithoutADecision(t *testing.T) }) t.Run("a redispatch decided before the outcome settled", func(t *testing.T) { l := newTestLedger(t) - unknownOutcome(t, l, 1) + olderUnknownOutcome(t, l, 1) decision := rawDecision(t, l, 1, "redispatch", "2000-01-01T00:00:00.000000000Z") _, err := l.db.ExecContext(ctx, `UPDATE events SET redispatch_decision = ? WHERE id = 1`, decision) require.NoError(t, err) @@ -463,7 +187,7 @@ func TestInvariant4TheDatabaseRefusesATerminalMoveWithoutADecision(t *testing.T) }) t.Run("a discard with no decision", func(t *testing.T) { l := newTestLedger(t) - unknownOutcome(t, l, 1) + olderUnknownOutcome(t, l, 1) // Decisions that are not this record's discard do not stand in for one. seenRecord(t, l, 2) rawDecision(t, l, 2, "discard", "9999-01-01T00:00:00.000000000Z") @@ -473,7 +197,7 @@ func TestInvariant4TheDatabaseRefusesATerminalMoveWithoutADecision(t *testing.T) }) t.Run("a discard decided before the outcome settled", func(t *testing.T) { l := newTestLedger(t) - unknownOutcome(t, l, 1) + olderUnknownOutcome(t, l, 1) rawDecision(t, l, 1, "discard", "2000-01-01T00:00:00.000000000Z") _, err := l.db.ExecContext(ctx, `UPDATE events SET state = 'discarded', reason = 'by_operator' WHERE id = 1`) require.Error(t, err) @@ -490,7 +214,7 @@ func TestInvariant4TheDatabaseRefusesATerminalMoveWithoutADecision(t *testing.T) }) t.Run("an unknown outcome discarded for another reason", func(t *testing.T) { l := newTestLedger(t) - unknownOutcome(t, l, 1) + olderUnknownOutcome(t, l, 1) rawDecision(t, l, 1, "discard", "9999-01-01T00:00:00.000000000Z") _, err := l.db.ExecContext(ctx, `UPDATE events SET state = 'discarded', reason = 'untrusted_author' WHERE id = 1`) require.Error(t, err) @@ -514,7 +238,7 @@ func TestDiscard(t *testing.T) { _, err := l.Admission().Commit(ctx, blockedVerdict(1, 0, admission.ReasonReadFailed)) require.NoError(t, err) }, - "unknown": func(t *testing.T, l *Ledger) { unknownOutcome(t, l, 1) }, + "unknown": func(t *testing.T, l *Ledger) { olderUnknownOutcome(t, l, 1) }, } for name, arrange := range accepted { t.Run(name, func(t *testing.T) { @@ -531,8 +255,6 @@ func TestDiscard(t *testing.T) { again, err := l.Discard(ctx, 1, opBy) require.NoError(t, err) assert.True(t, again.Already) - _, err = l.Redispatch(ctx, 1, opBy, []int64{adapterBucketID}) - assert.ErrorIs(t, err, ErrDecisionRefused) }) } refused := map[string]func(t *testing.T, l *Ledger){ @@ -540,16 +262,11 @@ func TestDiscard(t *testing.T) { "admitted": func(t *testing.T, l *Ledger) { opAdmit(t, l, 1, "recording:1") }, "dispatched": func(t *testing.T, l *Ledger) { opAdmit(t, l, 1, "recording:1") - launchOf(t, l, 1) + olderLaunch(t, l, 1) }, "failed": func(t *testing.T, l *Ledger) { opAdmit(t, l, 1, "recording:1") - launch := launchOf(t, l, 1) - d, err := l.Dispatch(ctx, launch.Token, adapterAgentID) - require.NoError(t, err) - pulled(t, d, 1) - _, err = d.Complete(ctx, 1, Completion{Outcome: OutcomeFailed}) - require.NoError(t, err) + olderReport(t, l, olderLaunch(t, l, 1), 1, OutcomeFailed) }, "discarded by admission": func(t *testing.T, l *Ledger) { seenRecord(t, l, 1) @@ -580,77 +297,45 @@ func rawDecision(t *testing.T, l *Ledger, eventID int64, action, at string) int6 return id } -// pulled hands the worker its dispatch for eventID, as a real worker does -// before it acknowledges or completes: a record is exposed at launch, but the -// ledger refuses an acknowledgement or a completion until the worker has -// pulled it. -func pulled(t *testing.T, d *TaskDispatch, eventID int64) { - t.Helper() - _, ok, err := d.Get(context.Background(), eventID) - require.NoError(t, err) - require.True(t, ok, "the worker is handed event %d", eventID) -} - -// pendingRedispatch leaves event 1 completed(failed) on a live task with a -// redispatch waiting for the task to end, and returns the task's launch. -func pendingRedispatch(t *testing.T, l *Ledger) Launch { - t.Helper() - ctx := context.Background() - require.Equal(t, StateAdmitted, opAdmit(t, l, 1, "recording:9")) - launch := launchOf(t, l, 1) - d, err := l.Dispatch(ctx, launch.Token, adapterAgentID) - require.NoError(t, err) - pulled(t, d, 1) - _, err = d.Complete(ctx, 1, Completion{Outcome: OutcomeFailed}) - require.NoError(t, err) - got, err := l.Redispatch(ctx, 1, opBy, []int64{adapterBucketID}) - require.NoError(t, err) - require.True(t, got.Pending) - return launch -} - -// Retention never strands a waiting redispatch: the record keeps what its -// admission needs, and the task's end is never refused. +// Retention never strands a redispatch an older build left waiting: the +// record keeps what its admission needs. func TestRetentionKeepsARecordAWaitingRedispatchNeeds(t *testing.T) { l := newTestLedger(t) ctx := context.Background() - launch := pendingRedispatch(t, l) + olderPendingRedispatch(t, l) dropped, err := l.DropContent(ctx, time.Now().Add(24*time.Hour), time.Now().Add(24*time.Hour)) require.NoError(t, err) assert.Zero(t, dropped) - - _, err = l.EndAttempt(ctx, AttemptEnd{AttemptID: launch.AttemptID, Stop: StopLost}) - require.NoError(t, err) - assert.Equal(t, StateAdmitted, stateOf(t, l, 1)) + assert.False(t, getRecord(t, l, 1).ContentDropped) } -// A task's end never fails for a waiting redispatch whose record cannot be -// admitted: it stays completed. -func TestATaskEndIsNeverRefusedForAWaitingRedispatch(t *testing.T) { - l := newTestLedger(t) - ctx := context.Background() - launch := pendingRedispatch(t, l) - _, err := l.db.ExecContext(ctx, `UPDATE events SET content_dropped = 1, snapshot = NULL WHERE id = 1`) - require.NoError(t, err) - - _, err = l.EndAttempt(ctx, AttemptEnd{AttemptID: launch.AttemptID, Stop: StopLost}) - require.NoError(t, err) - assert.Equal(t, StateCompleted, stateOf(t, l, 1)) +// waitingRedispatch reports whether event 1 still names a redispatch, and who +// authorized it. +func waitingRedispatch(t *testing.T, l *Ledger) (bool, string) { + t.Helper() + var ( + waiting bool + by string + ) + require.NoError(t, l.db.QueryRowContext(context.Background(), `SELECT redispatch_decision IS NOT NULL, authorized_by FROM events WHERE id = 1`).Scan(&waiting, &by)) + return waiting, by } // Invariant 3: a hold withdraws a redispatch still waiting for its task. func TestInvariant3AHoldWithdrawsAWaitingRedispatch(t *testing.T) { l := newTestLedger(t) ctx := context.Background() - launch := pendingRedispatch(t, l) - _, err := l.SetHold(ctx, opBy, HoldByOperator) - require.NoError(t, err) + olderPendingRedispatch(t, l) + waiting, by := waitingRedispatch(t, l) + require.True(t, waiting) + require.Equal(t, opBy, by) - _, err = l.EndAttempt(ctx, AttemptEnd{AttemptID: launch.AttemptID, Stop: StopLost}) + _, err := l.SetHold(ctx, opBy, HoldByOperator) require.NoError(t, err) - assert.Equal(t, StateCompleted, stateOf(t, l, 1), "the authorization did not survive the hold") - _, err = l.Redispatch(ctx, 1, opBy, []int64{adapterBucketID}) - require.NoError(t, err, "a person can authorize it again") + waiting, by = waitingRedispatch(t, l) + assert.False(t, waiting, "the authorization did not survive the hold") + assert.Empty(t, by) + assert.Equal(t, StateCompleted, stateOf(t, l, 1)) } // An import withdraws a waiting redispatch, whether the file says the entry is @@ -666,55 +351,20 @@ func TestImportWithdrawsAWaitingRedispatch(t *testing.T) { t.Run(name, func(t *testing.T) { l := newTestLedger(t) ctx := context.Background() - launch := pendingRedispatch(t, l) + olderPendingRedispatch(t, l) _, err := l.Import(ctx, Reconciliation{Version: 1, Entries: c.entries}, opBy) require.NoError(t, err) - _, err = l.EndAttempt(ctx, AttemptEnd{AttemptID: launch.AttemptID, Stop: StopLost}) - require.NoError(t, err) - assert.Equal(t, c.want, stateOf(t, l, 1), "never admitted by the withdrawn redispatch") + waiting, _ := waitingRedispatch(t, l) + assert.False(t, waiting, "the redispatch is withdrawn") + assert.Equal(t, c.want, stateOf(t, l, 1)) }) } } -// A task a redispatch superseded while it runs takes no follow-up: the event -// waits for the task to end and starts its own. -func TestASupersededTaskTakesNoFollowUp(t *testing.T) { - l := newTestLedger(t) - ctx := context.Background() - launch := pendingRedispatch(t, l) - require.Equal(t, StateAdmitted, opAdmit(t, l, 2, "recording:9"), "the conversation's task is superseded, so a new event is admitted") - - joined, err := l.JoinConversation(ctx, launch.TaskID, []int64{adapterBucketID}) - require.NoError(t, err) - assert.Empty(t, joined) - _, err = l.EndAttempt(ctx, AttemptEnd{AttemptID: launch.AttemptID, Stop: StopLost}) - require.NoError(t, err) - startable, err := l.StartableRecords(ctx, 10) - require.NoError(t, err) - require.Len(t, startable, 1) - second := launchOf(t, l, 1) - assert.ElementsMatch(t, []int64{1, 2}, second.EventIDs) -} - -// A held record redispatched onto a live conversation is queued, as admission -// would write it. -func TestRedispatchQueuesAHeldRecordBehindALiveConversation(t *testing.T) { - l := newTestLedger(t) - ctx := context.Background() - opAdmit(t, l, 1, "recording:9") - _, err := l.SetHold(ctx, opBy, HoldByOperator) - require.NoError(t, err) - require.Equal(t, StateAdmitted, opAdmit(t, l, 2, "recording:9"), "a new generation's record on the same conversation") - - got, err := l.Redispatch(ctx, 1, opBy, []int64{adapterBucketID}) - require.NoError(t, err) - assert.Equal(t, StateQueued, got.State) - assert.False(t, got.Admitted) -} - -// An import that closes a record does not leave it a lifecycle message that -// asks for a redispatch the ledger would refuse. +// An import that closes a record cancels what an older build's outbox still +// had to post about it, so nothing asks for a decision the ledger would now +// refuse. func TestImportCancelsTheMessagesOfARecordItCloses(t *testing.T) { l := newTestLedger(t) ctx := context.Background() @@ -724,48 +374,29 @@ func TestImportCancelsTheMessagesOfARecordItCloses(t *testing.T) { v.Reply = &admission.ReplyDestination{Kind: admission.ReplyComment, RecordingID: 10304028989} _, err := l.Admission().Commit(ctx, v) require.NoError(t, err) - - _, err = l.Import(ctx, Reconciliation{Version: 1, Entries: []ReconciliationEntry{{EventID: 1, Decision: DecisionDone}}}, opBy) - require.NoError(t, err) - pending, err := l.Intents(ctx, IntentFilter{States: []IntentState{IntentPending}}) - require.NoError(t, err) - assert.Empty(t, pending, "nothing is posted about a record a person closed") -} - -// Invariant 2, at the database: a task takes no follow-up while the hold -// marker stands, however the hold arrived. -func TestInvariant2ATaskTakesNoFollowUpUnderTheHold(t *testing.T) { - l := newTestLedger(t) - ctx := context.Background() - opAdmit(t, l, 1, "recording:9") - launch := launchOf(t, l, 1) - _, err := l.SetHold(ctx, opBy, HoldByOperator) + now := l.timestamp() + _, err = l.db.ExecContext(ctx, ` +INSERT INTO outbox (intent_key, kind, event_id, bucket_id, message_kind, recording_id, body, created_at, not_before) +VALUES ('guard_ack:1', 'guard_ack', 1, ?, 'comment', 10304028989, 'On it.', ?, ?)`, adapterBucketID, now, now) require.NoError(t, err) - require.Equal(t, StateQueued, opAdmit(t, l, 2, "recording:9"), "a new generation's record on the live conversation") - joined, err := l.JoinConversation(ctx, launch.TaskID, []int64{adapterBucketID}) + _, err = l.Import(ctx, Reconciliation{Version: 1, Entries: []ReconciliationEntry{{EventID: 1, Decision: DecisionDone}}}, opBy) require.NoError(t, err) - assert.Empty(t, joined) - assert.Equal(t, StateQueued, stateOf(t, l, 2)) + var state string + require.NoError(t, l.db.QueryRowContext(ctx, `SELECT state FROM outbox WHERE event_id = 1`).Scan(&state)) + assert.Equal(t, "canceled", state, "nothing is posted about a record a person closed") } -// An import's done decision closes an unknown or failed outcome for good: no -// redispatch is accepted for it and no notice asks for one. A success stays -// as it was. +// An import's done decision closes an unknown or failed outcome for good. A +// success stays as it was. func TestImportDoneClosesAnOutcomeThatWaitedForAPerson(t *testing.T) { l := newTestLedger(t) ctx := context.Background() - unknownOutcome(t, l, 1) + olderUnknownOutcome(t, l, 1) opAdmit(t, l, 2, "recording:2") - launch := launchOf(t, l, 2) - d, err := l.Dispatch(ctx, launch.Token, adapterAgentID) - require.NoError(t, err) - pulled(t, d, 2) - reply := int64(77) - _, err = d.Complete(ctx, 2, Completion{Outcome: OutcomeSucceeded, ReplyID: &reply}) - require.NoError(t, err) - _, err = l.EndAttempt(ctx, AttemptEnd{AttemptID: launch.AttemptID, Stop: StopFinished}) - require.NoError(t, err) + task := olderLaunch(t, l, 2) + olderReport(t, l, task, 2, OutcomeSucceeded) + olderEnd(t, l, task, "finished") got, err := l.Import(ctx, Reconciliation{Version: 1, Entries: []ReconciliationEntry{ {EventID: 1, Decision: DecisionDone}, {EventID: 2, Decision: DecisionDone}, @@ -781,133 +412,23 @@ func TestImportDoneClosesAnOutcomeThatWaitedForAPerson(t *testing.T) { var recordedAs string require.NoError(t, l.db.QueryRowContext(ctx, `SELECT to_state FROM decisions WHERE event_id = 2 AND action = 'import'`).Scan(&recordedAs)) assert.Equal(t, string(StateCompleted), recordedAs, "the audit says what happened, not what would have") - _, err = l.Redispatch(ctx, 1, opBy, []int64{adapterBucketID}) - assert.ErrorIs(t, err, ErrDecisionRefused) } -// A record held over a blocking reason and redispatched is authorized for the -// block that redispatch put it in, though the move stamps the block after the -// authorization's own time was taken. -func TestARedispatchOntoBlockedIsAuthorizedForThatBlock(t *testing.T) { - l := newTestLedger(t) - ctx := context.Background() - opAdmit(t, l, 1, "recording:1") - _, err := l.SetHold(ctx, opBy, HoldByOperator) - require.NoError(t, err) - _, err = l.db.ExecContext(ctx, `UPDATE events SET reason = 'no_route' WHERE id = 1`) - require.NoError(t, err) - base := time.Now() - calls := 0 - l.now = func() time.Time { calls++; return base.Add(time.Duration(calls) * time.Second) } // each stamp later than the last - - _, err = l.Redispatch(ctx, 1, opBy, []int64{adapterBucketID}) - require.NoError(t, err) - ids, err := l.AuthorizedBlocked(ctx, 10) - require.NoError(t, err) - assert.Equal(t, []int64{1}, ids) -} - -// A person who redispatches and then discards the same record has decided -// twice: the discard stands, and the redispatch waiting for the task's end is +// A record an older build's redispatch left waiting for its task, discarded by +// a person, has been decided twice: the discard stands, and the redispatch is // withdrawn with it. func TestADiscardWithdrawsARedispatchWaitingForItsTask(t *testing.T) { l := newTestLedger(t) ctx := context.Background() - launch := pendingRedispatch(t, l) + olderPendingRedispatch(t, l) _, err := l.db.ExecContext(ctx, `UPDATE task_events SET outcome = 'unknown' WHERE event_id = 1`) require.NoError(t, err) _, err = l.Discard(ctx, 1, opBy) require.NoError(t, err) - _, err = l.EndAttempt(ctx, AttemptEnd{AttemptID: launch.AttemptID, Stop: StopLost}) - require.NoError(t, err) record := getRecord(t, l, 1) - assert.Equal(t, StateDiscarded, record.State, "the task's end does not reopen what a person closed") + assert.Equal(t, StateDiscarded, record.State) assert.Equal(t, ReasonByOperator, record.Reason) - var waiting bool - require.NoError(t, l.db.QueryRowContext(ctx, `SELECT redispatch_decision IS NOT NULL FROM events WHERE id = 1`).Scan(&waiting)) + waiting, _ := waitingRedispatch(t, l) assert.False(t, waiting, "the authorization went with the record") } - -// Invariant 2, for the worker a crashed connector left running: while the hold -// stands, get_dispatch hands out nothing it had not already handed out. -func TestInvariant2AWorkerIsHandedNothingNewUnderTheHold(t *testing.T) { - l := newTestLedger(t) - ctx := context.Background() - opAdmit(t, l, 1, "recording:9") - require.Equal(t, StateQueued, opAdmit(t, l, 2, "recording:9")) - launch := launchOf(t, l, 1) - d, err := l.Dispatch(ctx, launch.Token, adapterAgentID) - require.NoError(t, err) - first, ok, err := d.Get(ctx, 1) - require.NoError(t, err) - require.True(t, ok) - - _, err = l.SetHold(ctx, opBy, HoldByOperator) - require.NoError(t, err) - - // The instruction it already holds is answered again; the sibling it never - // pulled is not handed over. - repeat, ok, err := d.Get(ctx, 1) - require.NoError(t, err) - require.True(t, ok) - assert.Equal(t, first.EventID, repeat.EventID) - _, _, err = d.Get(ctx, 2) - require.Error(t, err) - assert.Contains(t, err.Error(), "held") - _, err = l.ExposeEvent(ctx, launch.AttemptID, 2) - require.Error(t, err) - assert.Contains(t, err.Error(), "held") - - _, err = l.Release(ctx, opBy) - require.NoError(t, err) - _, ok, err = d.Get(ctx, 2) - require.NoError(t, err) - assert.True(t, ok, "released, the follow-up is handed over") -} - -// A hold arriving mid-task is not a failure of that task: the exposure is -// refused as held, and the dispatcher's follow-up loop stops asking. -func TestAHoldRefusesAnExposureAsHeldNotAsAFailure(t *testing.T) { - l := newTestLedger(t) - ctx := context.Background() - opAdmit(t, l, 1, "recording:9") - require.Equal(t, StateQueued, opAdmit(t, l, 2, "recording:9")) - launch := launchOf(t, l, 1) - _, err := l.SetHold(ctx, opBy, HoldByOperator) - require.NoError(t, err) - - _, err = l.ExposeEvent(ctx, launch.AttemptID, 2) - require.ErrorIs(t, err, ErrHeld) -} - -// Invariant 2: an event the launch exposed carries only a pointer until a -// worker pulls its instruction, so the hold refuses that first pull too — -// the case a crash during launch and a restart under --hold leaves behind. -func TestInvariant2AFirstPullIsRefusedUnderTheHold(t *testing.T) { - l := newTestLedger(t) - ctx := context.Background() - opAdmit(t, l, 1, "recording:1") - launch := launchOf(t, l, 1) - d, err := l.Dispatch(ctx, launch.Token, adapterAgentID) - require.NoError(t, err) - _, err = l.SetHold(ctx, opBy, HoldByOperator) - require.NoError(t, err) - - _, _, err = d.Get(ctx, 1) - require.ErrorIs(t, err, ErrHeld, "the instruction is not handed over under the hold") - - _, err = l.Release(ctx, opBy) - require.NoError(t, err) - first, ok, err := d.Get(ctx, 1) - require.NoError(t, err) - require.True(t, ok) - - // Pulled once, a repeat is answered even if a hold lands after it. - _, err = l.SetHold(ctx, opBy, HoldByOperator) - require.NoError(t, err) - repeat, ok, err := d.Get(ctx, 1) - require.NoError(t, err) - require.True(t, ok) - assert.Equal(t, first.EventID, repeat.EventID) -} diff --git a/internal/connector/operator_migration_test.go b/internal/connector/operator_migration_test.go index 2ccb12b22..8ea6cc81d 100644 --- a/internal/connector/operator_migration_test.go +++ b/internal/connector/operator_migration_test.go @@ -75,7 +75,7 @@ func TestShadowPromoteYieldsAHeldLedger(t *testing.T) { l, err := OpenLedger(filepath.Join(stateDir, LedgerFile)) require.NoError(t, err) t.Cleanup(func() { _ = l.Close() }) - held, err := l.Held(ctx) + _, held, err := l.HoldMarker(ctx) require.NoError(t, err) assert.True(t, held) assert.Equal(t, StateHeld, stateOf(t, l, 1)) @@ -134,7 +134,7 @@ func assertUntouchedShadow(t *testing.T, shadowDir string) { l, err := OpenLedgerReadOnly(context.Background(), filepath.Join(shadowDir, LedgerFile)) require.NoError(t, err) defer func() { _ = l.Close() }() - held, err := l.Held(context.Background()) + _, held, err := l.HoldMarker(context.Background()) require.NoError(t, err) assert.False(t, held) assert.Equal(t, StateAdmitted, stateOf(t, l, 1)) @@ -254,7 +254,7 @@ func ledgerIsHeld(t *testing.T, path string) bool { l, err := OpenLedgerReadOnly(context.Background(), path) require.NoError(t, err) defer func() { _ = l.Close() }() - held, err := l.Held(context.Background()) + _, held, err := l.HoldMarker(context.Background()) require.NoError(t, err) return held } @@ -264,7 +264,7 @@ func assertHeld(t *testing.T, path string) { l, err := OpenLedger(path) require.NoError(t, err) defer func() { _ = l.Close() }() - held, err := l.Held(context.Background()) + _, held, err := l.HoldMarker(context.Background()) require.NoError(t, err) require.True(t, held, "%s is held", path) assert.Equal(t, StateHeld, stateOf(t, l, 1), "the waiting record is held") @@ -321,7 +321,7 @@ func TestImportRefusesAFileItCannotApplyWhole(t *testing.T) { t.Run(name, func(t *testing.T) { l := newTestLedger(t) opAdmit(t, l, 1, "recording:1") - launchOf(t, l, 1) + olderLaunch(t, l, 1) seenRecord(t, l, 2) _, err := l.Import(ctx, Reconciliation{Version: 1, Entries: entries}, opBy) diff --git a/internal/connector/operator_status_test.go b/internal/connector/operator_status_test.go index 315dbd7d1..e534e167c 100644 --- a/internal/connector/operator_status_test.go +++ b/internal/connector/operator_status_test.go @@ -26,10 +26,10 @@ func TestInvariant8StatusReadsBesideAWriterAndShowsNoSecrets(t *testing.T) { require.NoError(t, l.NotePollServed(ctx, testKey(), 41)) require.NoError(t, l.NoteConnection(ctx, ConnectionRunning, "")) - unknownOutcome(t, l, 2) + olderUnknownOutcome(t, l, 2) opAdmit(t, l, 1, "recording:1") - launch := launchOf(t, l, 1) - require.NoError(t, l.MarkRunning(ctx, launch.AttemptID, AttemptProcess{PID: 4242, PGID: 4242, StartedAt: time.Now()})) + launch := olderLaunch(t, l, 1) + olderRunning(t, l, launch.AttemptID, 4242, time.Time{}) seenRecord(t, l, 5) _, err = l.Admission().Commit(ctx, blockedVerdict(5, 0, "read_failed")) require.NoError(t, err) @@ -38,6 +38,9 @@ func TestInvariant8StatusReadsBesideAWriterAndShowsNoSecrets(t *testing.T) { _, err = l.SetHold(ctx, opBy, HoldByOperator) require.NoError(t, err) + var hash string + require.NoError(t, l.db.QueryRowContext(ctx, `SELECT token_sha256 FROM tasks WHERE id = ?`, launch.TaskID).Scan(&hash)) + // A writer holds the write lock while status reads. writer, err := l.db.BeginTx(ctx, nil) require.NoError(t, err) @@ -74,8 +77,7 @@ func TestInvariant8StatusReadsBesideAWriterAndShowsNoSecrets(t *testing.T) { out := string(raw) assert.NotContains(t, out, position) assert.NotContains(t, out, "please look", "no snapshot content") - assert.NotContains(t, out, tokenHash(launch.Token)) - assert.NotContains(t, out, launch.Token) + assert.NotContains(t, out, hash, "no token hash") } func TestOpenLedgerReadOnlyCreatesNothing(t *testing.T) { @@ -132,22 +134,18 @@ func TestOpenLedgerRefusesANewerSchema(t *testing.T) { } // Every process read back out of the ledger is an identity the one-owner -// rule will answer about. The stamp stored is the kernel's — startedStamp -// writes no other — so each read-back path marks it exact. A path that -// rebuilds driver.Process by hand loses that bit, and the rule then calls a -// live worker neither gone nor running (driver.ErrIdentityUnknown): status -// reported every worker "unverified" and a redispatch refused to stop the -// worker it replaced. +// rule will answer about. The stamp stored is the kernel's, since an older +// build wrote no other, so each read-back path marks it exact. A path that +// rebuilds procid.Process by hand loses that bit, and the rule then calls a +// live worker neither gone nor running (procid.ErrIdentityUnknown): status +// reported every worker "unverified". func TestEveryProcessReadBackOutOfTheLedgerIsAnIdentity(t *testing.T) { at := time.Now().UTC() - assert.True(t, AttemptProcess{PID: 7, PGID: 7, StartedAt: at, StartedExact: true}.Identity().StartedExact) - assert.True(t, LiveWorker{PID: 7, PGID: 7, StartedAt: at}.Identity().StartedExact) assert.True(t, TaskStatus{PID: 7, PGID: 7, ProcessStartedAt: &at}.WorkerIdentity().StartedExact) assert.True(t, TaskStatus{TakerPID: 7, TakerPGID: 7, TakerStartedAt: &at}.TakerIdentity().StartedExact) // And a record with no stamp is no identity at all, rather than one the // rule would answer about. - assert.False(t, AttemptProcess{PID: 7, PGID: 7}.Identity().StartedExact) assert.False(t, TaskStatus{PID: 7, PGID: 7}.WorkerIdentity().StartedExact) assert.False(t, TaskStatus{TakerPID: 7, TakerPGID: 7}.TakerIdentity().StartedExact) } @@ -155,21 +153,21 @@ func TestEveryProcessReadBackOutOfTheLedgerIsAnIdentity(t *testing.T) { // A token whose holder could not be accounted for is held, and status is // where the person who must settle it reads that. The zero taker cannot say // it: "nothing took the token" and "the token is out and nobody can name who -// has it" are opposite facts, and the ledger carries the difference the -// release point acts on (TokenHolder.Unaccounted) all the way out. +// has it" are opposite facts, and status carries the difference an older +// build recorded all the way out. func TestStatusCarriesATokenHolderThatCannotBeAccountedFor(t *testing.T) { ctx := context.Background() l := newTestLedger(t) opAdmit(t, l, 1, "recording:1") - launch := launchOf(t, l, 1) - require.NoError(t, l.MarkRunning(ctx, launch.AttemptID, AttemptProcess{PID: 4242, PGID: 4242, StartedAt: time.Now(), StartedExact: true})) + launch := olderLaunch(t, l, 1) + olderRunning(t, l, launch.AttemptID, 4242, time.Now()) before, err := l.Status(ctx) require.NoError(t, err) require.Len(t, before.Tasks, 1) assert.False(t, before.Tasks[0].TakerUnaccounted, "nothing has taken the token yet") - require.NoError(t, l.MarkTakerUnaccounted(ctx, launch.AttemptID)) + olderTakerUnaccounted(t, l, launch.AttemptID) after, err := l.Status(ctx) require.NoError(t, err) require.Len(t, after.Tasks, 1) diff --git a/internal/connector/outbox.go b/internal/connector/outbox.go index 13f467e0c..86d4ab2a7 100644 --- a/internal/connector/outbox.go +++ b/internal/connector/outbox.go @@ -1,87 +1,40 @@ package connector import ( - "context" "database/sql" - "errors" "fmt" "strings" "time" ) -// The outbox: every message the connector itself posts to Basecamp — the -// guard acknowledgement, the holding reply, still-running, the completion -// notice and the retraction that answers an ask in one of them — goes through -// one table with one rule. +// The outbox: every message an older build's connector posted to Basecamp +// itself (the guard acknowledgement, the holding reply, still-running, the +// completion notice, and the retraction that answered an ask in one of them) +// went through this one table. // -// # Invariants +// Since #815 removed the worker side, this build writes no intent and sends +// none: the table is written only by its migrations and by the triggers below. +// What an older build left in it is still read and still decided about. +// Status lists the intents waiting for a person (statusIntents), and a +// discard or an import's done decision cancels the guard acknowledgement and +// holding reply still pending for the record it closes. // -// Each is held by the database where SQL can say it, and by a test that fails -// without it (outbox_invariants_test.go). +// What the schema held those writes to, and still holds any write to: // -// 1. An intent is written in the transaction of the transition that calls -// for it, through the ledger's hooks, so the two commit or roll back -// together. -// 2. One intent per thing answered for: the key is the guard or holding -// reply per event, the completion per attempt, still-running per attempt -// and occurrence, the retraction per posted message and the event whose -// ask it answers. A second write for a key writes nothing. -// 3. Nothing is sent without a durable sending row. The only path to a -// request claims the intent — pending to sending, committed — first. -// 4. Nothing sending is sent again automatically. A request is made only for -// an intent this process just claimed from pending. A sending intent is -// reconciled by listing the destination, never by posting. -// 5. Reconciliation adopts only an unambiguous candidate: exactly one of the -// agent's messages at the destination since the intent went sending -// matches its body, the message is not a worker's own acknowledgement or -// reply, no other intent owns it, and no other intent at the destination -// whose own message may exist unreceipted — pending, sending, -// indeterminate, or abandoned by a person who could not prove it absent — -// has the same body. Anything else is indeterminate, for a person. -// 6. A receipt belongs to exactly one intent, and once written it never -// changes. A unique index and a trigger. -// 7. States move along the lifecycle's edges only: pending → sending | -// canceled; sending → sent | indeterminate, or canceled when Basecamp -// answered the request by refusing it, which creates nothing; and -// indeterminate → sent | abandoned | pending, those three only by a -// person, as is refused → pending once a person has fixed the cause. -// 8. get_dispatch cancels the guard: a trigger moves the guard intent from +// 1. One intent per thing answered for: intent_key is unique. +// 2. A receipt belongs to exactly one intent, and once written it never +// changes (outbox_receipt, outbox_receipt_is_final). +// 3. States move along the lifecycle's edges only (outbox_state_edges): +// pending to sending or canceled; sending to sent, indeterminate, or +// canceled when Basecamp refused the request; indeterminate to sent, +// abandoned or pending, those three only by a person. +// 4. get_dispatch canceled the guard: a trigger moved the guard intent from // pending to canceled in get_dispatch's own transaction, and a guard that -// already went out marks every task event it answers for as fired, so a -// worker is told the connector acknowledged. -// 9. A guard is reported fired from the moment it is claimed, and that is -// final: #736's task_events_guard_settles_once lets a guard move only -// from armed. The claim marks its task events fired in the claim's own -// transaction, so no worker asking while the request is in flight -// acknowledges a second time. If Basecamp then refuses the request, the -// intent is canceled — nothing was created — but its task events stay -// fired, so that task's workers do not acknowledge either: the -// acknowledgement is missing, never doubled, the spec's own preference. -// A later task for the event (a person's redispatch) has its guard armed, -// and the worker acknowledges itself: the intent is canceled, so nothing -// remains to fire that guard, and get_dispatch cancels it. This is the spec's rule: a -// missing acknowledgement costs less than a double one. A refusal here is -// Basecamp refusing the connector's own lifecycle request, recorded on -// the outbox row; a worker's permission refusals are another matter. -// 10. Reconciliation never holds up sending for long. A running connector -// sends a batch, then lists at most one due destination; each listing is -// bounded in time; each failure backs its intent off, doubling, and the -// intent is indeterminate after MaxReconcileFailures, with that count -// recorded. Start is the exception by design: it reconciles everything -// due before it sends, within the bound its caller sets, and stops -// sending at the first send that may not have landed. -// 11. A notice is retracted by another message, never by editing or deleting -// the one that went out. Only an ask is retracted — a line telling a -// person to run something — and both halves of what the retraction claims -// are asked again at its claim, because a retraction that is wrong is -// worse than none: a reader trusts the later message, so a line saying an -// ask is answered under an ask that is still live means nobody acts on -// it. The message it answers must be known to exist — one whose notice -// was refused, or left indeterminate for a person, waits for that person -// rather than being posted where there may be nothing to answer, and is -// canceled once they say there is nothing (abandoned) — and the record -// must not be waiting for that ask any more, which is a person's decision -// having actually settled it and not merely having been made. +// had already gone out marked every task event it answered for as fired +// (outbox_guard_canceled_by_get_dispatch, outbox_guard_fired_before_task). +// 5. A held record's pending guard is canceled, and nothing moves to sending +// while the hold marker stands (events_held_cancels_guard, +// outbox_refused_under_hold). const migrationOutbox = ` CREATE TABLE outbox ( id INTEGER PRIMARY KEY AUTOINCREMENT, @@ -380,17 +333,6 @@ type Intent struct { ReconcileAt *time.Time } -// Errors from the outbox. -var ( - // ErrNoSuchIntent is an intent id the ledger does not hold. - ErrNoSuchIntent = errors.New("no such outbox intent") - // ErrNotIndeterminate is a person's resolution for an intent that is not - // indeterminate. - ErrNotIndeterminate = errors.New("the intent is not indeterminate") - // ErrReceiptOwned is a receipt another intent already owns. - ErrReceiptOwned = errors.New("the receipt belongs to another intent") -) - func nullableID64(id int64) any { if id == 0 { return nil @@ -458,176 +400,6 @@ func parseNullStamp(s sql.NullString) (*time.Time, error) { return &t, nil } -// IntentFilter selects intents. Zero values select everything. -type IntentFilter struct { - States []IntentState - Kinds []IntentKind - EventID int64 - // Limit is the most returned, newest first; zero for all. - Limit int -} - -// Intents lists outbox intents, newest first. It only reads. -func (l *Ledger) Intents(ctx context.Context, f IntentFilter) ([]Intent, error) { - var out []Intent - err := retryBusy(func() error { - var err error - out, err = l.intents(ctx, f) - return err - }) - return out, err -} - -func (l *Ledger) intents(ctx context.Context, f IntentFilter) ([]Intent, error) { - var ( - where []string - args []any - ) - if len(f.States) > 0 { - where = append(where, "state IN ("+placeholders(len(f.States))+")") - for _, s := range f.States { - args = append(args, string(s)) - } - } - if len(f.Kinds) > 0 { - where = append(where, "kind IN ("+placeholders(len(f.Kinds))+")") - for _, k := range f.Kinds { - args = append(args, string(k)) - } - } - if f.EventID != 0 { - where = append(where, "event_id = ?") - args = append(args, f.EventID) - } - query := selectIntents - if len(where) > 0 { - query += " WHERE " + strings.Join(where, " AND ") - } - query += " ORDER BY id DESC" - if f.Limit > 0 { - query += " LIMIT ?" - args = append(args, f.Limit) - } - rows, err := l.db.QueryContext(ctx, query, args...) - if err != nil { - return nil, fmt.Errorf("connector: list outbox: %w", err) - } - return scanIntents(rows) -} - -// Intent reads one intent by id. -func (l *Ledger) Intent(ctx context.Context, id int64) (Intent, error) { - var intents []Intent - err := retryBusy(func() error { - rows, err := l.db.QueryContext(ctx, selectIntents+` WHERE id = ?`, id) - if err != nil { - return fmt.Errorf("connector: read outbox intent %d: %w", id, err) - } - intents, err = scanIntents(rows) - return err - }) - if err != nil { - return Intent{}, err - } - if len(intents) == 0 { - return Intent{}, fmt.Errorf("connector: outbox intent %d: %w", id, ErrNoSuchIntent) - } - return intents[0], nil -} - func placeholders(n int) string { return strings.TrimSuffix(strings.Repeat("?, ", n), ", ") } - -// IsLifecycleReceipt reports whether a message id is the receipt of one of the -// connector's own lifecycle messages of that kind. -func (l *Ledger) IsLifecycleReceipt(ctx context.Context, kind MessageKind, id int64) (bool, error) { - var found bool - err := retryBusy(func() error { - return l.db.QueryRowContext(ctx, `SELECT EXISTS (SELECT 1 FROM outbox WHERE message_kind = ? AND receipt_id = ?)`, string(kind), id).Scan(&found) - }) - if err != nil { - return false, fmt.Errorf("connector: lifecycle receipt %d: %w", id, err) - } - return found, nil -} - -// Resolution is a person's decision on an indeterminate intent. -type Resolution string - -const ( - // ResolveSent says the message is in Basecamp: ReceiptID names it. - ResolveSent Resolution = "sent" - // ResolveAbandon says it is not to be sent. - ResolveAbandon Resolution = "abandon" - // ResolveResend authorizes sending it again: the intent returns to - // pending. Only a person may choose this; nothing automatic does. - ResolveResend Resolution = "resend" -) - -// IntentResolution is a person's decision and who made it. -type IntentResolution struct { - Resolution Resolution - // ReceiptID is the message a ResolveSent names. - ReceiptID int64 - // By names who decided, for the record. Required. - By string -} - -// ResolveIntent applies a person's decision to an indeterminate intent. -func (l *Ledger) ResolveIntent(ctx context.Context, id int64, r IntentResolution) error { - if strings.TrimSpace(r.By) == "" { - return errors.New("connector: a resolution records who decided") - } - now := l.timestamp() - var ( - query string - args []any - ) - switch r.Resolution { - case ResolveSent: - if r.ReceiptID <= 0 { - return errors.New("connector: a sent resolution names the message") - } - query = `UPDATE outbox SET state = 'sent', receipt_id = ?, finished_at = ?, resolved_by = ?, note = ? WHERE id = ? AND state = 'indeterminate'` - args = []any{r.ReceiptID, now} - case ResolveAbandon: - query = `UPDATE outbox SET state = 'abandoned', finished_at = ?, resolved_by = ?, note = ? WHERE id = ? AND state = 'indeterminate'` - args = []any{now} - case ResolveResend: - // A refused request created nothing, so a person may send it again - // once the cause is fixed, as they may an indeterminate one. - query = `UPDATE outbox SET state = 'pending', sending_at = NULL, finished_at = NULL, reconcile_failures = 0, reconcile_at = NULL, not_before = ?, resolved_by = ?, note = ? -WHERE id = ? AND (state = 'indeterminate' OR (state = 'canceled' AND note = '` + RefusedNote + `'))` - args = []any{now} - default: - return fmt.Errorf("connector: %q is not a resolution", r.Resolution) - } - return retryBusy(func() error { - res, err := l.db.ExecContext(ctx, query, append(args, r.By, "resolved: "+string(r.Resolution), id)...) - if err != nil { - if isUniqueViolation(err) { - return fmt.Errorf("connector: resolve intent %d: %w", id, ErrReceiptOwned) - } - return fmt.Errorf("connector: resolve intent %d: %w", id, err) - } - n, err := res.RowsAffected() - if err != nil { - return err - } - if n == 0 { - if _, err := l.Intent(ctx, id); err != nil { - return err - } - return fmt.Errorf("connector: resolve intent %d: %w", id, ErrNotIndeterminate) - } - return nil - }) -} - -// RefusedNote is the note on an intent Basecamp refused. -const RefusedNote = "the request was refused; no message was created" - -func isUniqueViolation(err error) bool { - return err != nil && strings.Contains(err.Error(), "UNIQUE constraint failed") -} diff --git a/internal/connector/procid/procid.go b/internal/connector/procid/procid.go new file mode 100644 index 000000000..fc211ce1e --- /dev/null +++ b/internal/connector/procid/procid.go @@ -0,0 +1,50 @@ +// Package procid tells a process the ledger or a lock file recorded from a +// later process the kernel gave the same pid. +// +// A pid is not an identity: the kernel reuses them. An identity is the pid +// AND the kernel's own start time for it, compared exactly, and the questions +// here answer only on that: +// +// - LookupProcess takes the identity of a live process, such as the one +// named in the instance lock's metadata. +// - OwnsWorker answers whether a worker an older build recorded in the +// ledger is still that worker, and whether anything of its process group +// is left. Status reads a worker-era task's worker and taker through it. +// +// Where the kernel will not give a start time, or a record carries none, +// nothing here guesses: the answer is ErrIdentityUnknown, never "gone". +package procid + +import ( + "errors" + "time" +) + +// Process is a process the connector recorded. +type Process struct { + // PID is the process's id; zero when there is none. + PID int + // PGID is its process group. A worker led a group of its own, so for a + // worker PGID == PID. + PGID int + // StartedAt is when the process started, to tell it from a later one + // that reused its id. + StartedAt time.Time + // StartedExact is true when StartedAt is the kernel's own start time for + // the pid, which is what an identity is compared by. False is a + // wall-clock stamp, or no stamp at all: readable, but not an identity, + // and OwnsWorker refuses to answer on one (ErrIdentityUnknown). + StartedExact bool +} + +// ErrGroupOutlivedLeader is a recorded process group whose leader is gone, +// or is a pid the kernel has since reused, while the group still has +// members. They may be the worker's own children, so the caller must not +// treat the worker as finished. +var ErrGroupOutlivedLeader = errors.New("procid: the recorded process group outlived its leader") + +// ErrIdentityUnknown is a record the connector cannot tell from a later +// process that reused its pid, because no kernel start time was ever +// recorded for it. It is not "gone" and it is not "still running": it is +// unanswerable. +var ErrIdentityUnknown = errors.New("procid: the recorded process has no kernel start time, so it cannot be told from a later process that reused its pid") diff --git a/internal/connector/procid/procid_other.go b/internal/connector/procid/procid_other.go new file mode 100644 index 000000000..5be57c3f2 --- /dev/null +++ b/internal/connector/procid/procid_other.go @@ -0,0 +1,14 @@ +//go:build !unix + +package procid + +import "errors" + +var errUnsupported = errors.New("procid: process identities are read on Unix only") + +// OwnsWorker cannot answer off Unix, and an identity that cannot be +// established is never acted on. +func OwnsWorker(Process) (bool, error) { return false, errUnsupported } + +// LookupProcess cannot answer off Unix. +func LookupProcess(int) (Process, error) { return Process{}, errUnsupported } diff --git a/internal/connector/procid/procid_unix.go b/internal/connector/procid/procid_unix.go new file mode 100644 index 000000000..c33e87fac --- /dev/null +++ b/internal/connector/procid/procid_unix.go @@ -0,0 +1,157 @@ +//go:build unix + +package procid + +import ( + "errors" + "fmt" + "os" + "runtime" + "syscall" +) + +// OwnsWorker answers whether the process a record names is still the worker +// the record was written for. Everything that asks about a recorded worker +// asks this rather than testing a pid of its own: +// +// - (true, nil): the process is still that worker. +// - (false, nil): it is gone, and its group has no members left. +// - (false, ErrGroupOutlivedLeader): the leader is gone or is now some other +// process, and the recorded group still has members, which may be the +// worker's children. +// - (false, err): the identity cannot be established here (an unreadable +// process table, a record with no kernel start time, a platform that +// cannot say). +// +// It signals nothing: the only signal it sends is the zero signal, which asks +// whether a group exists. +func OwnsWorker(p Process) (bool, error) { + if p.PID <= 0 || p.PGID <= 0 { + // There is no process here to own. + return false, nil + } + gone, err := processGone(p) + if err != nil { + return false, err + } + if gone { + // The leader is gone, or its pid is somebody else's now: what is left + // of the group decides whether anything of this worker remains. + return false, groupGone(p.PGID) + } + return true, nil +} + +// processGone reports whether the process a record names is gone: no process +// by that pid, a zombie, or a later process the kernel gave the same pid. It +// asks only about that process and says nothing about its group. +// +// The comparison is exact. A kernel start time is read the same way every +// time it is read, so the process that was recorded answers with the value +// recorded for it and anything else is another process. A record whose start +// time the kernel never gave (StartedExact false) is ErrIdentityUnknown +// rather than a comparison against a tolerance: under fast pid reuse a window +// wide enough to cover a wall-clock stamp is wide enough to accept a +// stranger. +func processGone(p Process) (bool, error) { + if p.PID <= 0 { + return true, nil + } + started, err := processStartTime(p.PID) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + // No process by that pid at all: nothing of it is left, whatever + // the record says about when it started. + return true, nil + } + return false, err + } + if !p.StartedExact { + return false, fmt.Errorf("%w: pid %d", ErrIdentityUnknown, p.PID) + } + if !started.Equal(p.StartedAt) { + return true, nil + } + return false, nil +} + +// LookupProcess is a live process's identity: its pid, the process group it +// leads or belongs to, and the start time that tells it from a later process +// the kernel gave the same pid. A process that is gone, or a zombie, which +// runs nothing, is os.ErrNotExist. +func LookupProcess(pid int) (Process, error) { + if pid <= 0 { + return Process{}, os.ErrNotExist + } + started, err := processStartTime(pid) + if err != nil { + return Process{}, err + } + pgid, err := syscall.Getpgid(pid) + if err != nil { + return Process{}, err + } + return Process{PID: pid, PGID: pgid, StartedAt: started, StartedExact: true}, nil +} + +// groupGone reports nil only when the kernel says there is no such process +// group, or when every member it still lists is a zombie. Anything else, a +// member that runs, a listing that could not be read, or a probe that was +// refused, is not absence. +// +// A zombie answers a zero signal like a live process, and one stays a member +// until its parent waits for it, so a probe that counted zombies would see a +// finished worker as still there for as long as nobody reaped it. +func groupGone(pgid int) error { + err := zeroSignalGroup(pgid) + if err == nil { + running, listErr := groupRunning(pgid) + switch { + case listErr != nil: + return fmt.Errorf("%w: %d: %w", ErrGroupOutlivedLeader, pgid, listErr) + case !running: + return nil + } + } + return groupProbe(pgid, err) +} + +// groupProbe reads what a zero signal to a process group said. Only ESRCH, +// "no such process group", is proof of absence; a refusal (EPERM, from a +// group this process may not signal) is a group that is probably there and +// certainly not proven gone. +func groupProbe(pgid int, err error) error { + switch { + case err == nil: + return fmt.Errorf("%w: %d", ErrGroupOutlivedLeader, pgid) + case errors.Is(err, syscall.ESRCH): + return nil + default: + return fmt.Errorf("%w: %d: %w", ErrGroupOutlivedLeader, pgid, err) + } +} + +// zeroSignalGroup sends the zero signal to every process in the group, which +// delivers nothing and says whether the group exists. A non-positive pgid is +// refused, since kill(0) and kill(-1) mean this group and every process, and +// so is the connector's own group: a worker always led a group of its own, +// so a recorded group that is this process's own is a mistake. +func zeroSignalGroup(pgid int) error { + if pgid <= 1 || pgid == syscall.Getpgrp() { + return syscall.EINVAL + } + err := syscall.Kill(-pgid, 0) + // macOS refuses any signal to a group whose only members are zombies, + // where Linux delivers it; nothing in such a group runs. When the + // kernel's own listing says so, it is the absent group it is everywhere + // else. A group that runs but may not be signaled (another user's) still + // lists its members, and keeps the refusal. On Linux EPERM is a group + // this process may not signal, and with /proc mounted hidepid its + // listing cannot see that group's members to say otherwise. + if runtime.GOOS == "darwin" && errors.Is(err, syscall.EPERM) { + if running, listErr := groupRunning(pgid); listErr == nil && !running { + return syscall.ESRCH + } + } + return err +} diff --git a/internal/connector/procid/procid_unix_test.go b/internal/connector/procid/procid_unix_test.go new file mode 100644 index 000000000..14504d6f2 --- /dev/null +++ b/internal/connector/procid/procid_unix_test.go @@ -0,0 +1,153 @@ +//go:build unix + +package procid + +import ( + "context" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "syscall" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// startWithChild starts a process that leads a group of its own and leaves a +// child in it, the shape of a worker that started tools of its own. It gives +// back the leader's command, its identity, and the child's pid. +func startWithChild(t *testing.T) (*exec.Cmd, Process, int) { + t.Helper() + pidFile := filepath.Join(t.TempDir(), "child") + cmd := exec.CommandContext(context.Background(), "/bin/sh", "-c", "sleep 300 & echo $! > "+pidFile+"; exec sleep 300") + cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} + require.NoError(t, cmd.Start()) + t.Cleanup(func() { + _ = syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL) + _ = cmd.Wait() + }) + var child int + require.Eventually(t, func() bool { + data, err := os.ReadFile(pidFile) + if err != nil { + return false + } + child, err = strconv.Atoi(strings.TrimSpace(string(data))) + return err == nil + }, 5*time.Second, 10*time.Millisecond) + p, err := LookupProcess(cmd.Process.Pid) + require.NoError(t, err) + return cmd, p, child +} + +func alive(pid int) bool { return syscall.Kill(pid, 0) == nil } + +// Copilot r3: a process group can outlive its leader, and its members may be +// the worker's own children. +func TestAGroupThatOutlivedItsLeaderIsNotSilenceAbsence(t *testing.T) { + cmd, leader, child := startWithChild(t) + + // The leader alone goes, and is reaped; its child keeps the group. + require.NoError(t, cmd.Process.Kill()) + _ = cmd.Wait() + + owns, err := OwnsWorker(leader) + assert.False(t, owns) + assert.ErrorIs(t, err, ErrGroupOutlivedLeader) + assert.True(t, alive(child), "and the child is left alone for a person to decide about") +} + +// A pid is not an identity. +func TestOwnsWorkerAnswersWhetherThisIsStillTheWorker(t *testing.T) { + _, p, _ := startWithChild(t) + + owns, err := OwnsWorker(p) + require.NoError(t, err) + assert.True(t, owns, "the process that was recorded") + + reused := p + reused.StartedAt = p.StartedAt.Add(-time.Hour) + owns, err = OwnsWorker(reused) + assert.False(t, owns, "the same pid with another start time is another process") + assert.ErrorIs(t, err, ErrGroupOutlivedLeader, "and its group still has members") + + owns, err = OwnsWorker(Process{PID: 1 << 30, PGID: 1 << 30, StartedAt: time.Now()}) + assert.False(t, owns) + assert.NoError(t, err, "a pid that names nothing, in a group with no members, is simply gone") + + owns, err = OwnsWorker(Process{}) + assert.False(t, owns) + assert.NoError(t, err, "a record with no process is nothing to own") +} + +// Copilot r4: only "no such process group" proves a group is gone; a probe +// that was refused is not absence. +func TestOnlyNoSuchProcessGroupProvesAbsence(t *testing.T) { + assert.NoError(t, groupProbe(4242, syscall.ESRCH), "no such group: gone") + assert.ErrorIs(t, groupProbe(4242, nil), ErrGroupOutlivedLeader, "answered: members remain") + assert.ErrorIs(t, groupProbe(4242, syscall.EPERM), ErrGroupOutlivedLeader, "refused: not proven gone") + assert.ErrorIs(t, groupProbe(4242, syscall.EINVAL), ErrGroupOutlivedLeader, "any other answer: not proven gone") +} + +// sleepInItsOwnGroup starts a process that leads a group of its own, and +// gives back the kernel's identity for it. It stands in for whatever holds a +// pid now: a worker of a later attempt, or any process of this user the +// kernel gave a recycled id to. +func sleepInItsOwnGroup(t *testing.T) Process { + t.Helper() + cmd := exec.CommandContext(context.Background(), "/bin/sleep", "300") + cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} + require.NoError(t, cmd.Start()) + t.Cleanup(func() { + _ = cmd.Process.Kill() + _ = cmd.Wait() + }) + p, err := LookupProcess(cmd.Process.Pid) + require.NoError(t, err) + require.True(t, p.StartedExact, "the kernel's own start time is the identity") + return p +} + +// Copilot on #738: the kernel's start time was recorded when it could be +// read, but the comparison accepted anything within three seconds of it, +// so under fast pid reuse a stranger started just after the record was +// written passed as the worker. +func TestAProcessStartedJustAfterTheRecordIsNotTheRecordedOne(t *testing.T) { + p := sleepInItsOwnGroup(t) + + stranger := p + stranger.StartedAt = p.StartedAt.Add(-time.Second) + gone, err := processGone(stranger) + require.NoError(t, err) + assert.True(t, gone, "a second between the record and the kernel is another process, not this one") + + owns, err := OwnsWorker(stranger) + assert.False(t, owns) + assert.ErrorIs(t, err, ErrGroupOutlivedLeader, "and its group is not settled around either") +} + +// The wall-clock fallback is not an identity: where the kernel would not say +// when a process started, the rule refuses to answer rather than compare +// against a stamp taken around a fork. +func TestARecordWithNoKernelStartTimeIsRefused(t *testing.T) { + p := sleepInItsOwnGroup(t) + + stamped := Process{PID: p.PID, PGID: p.PGID, StartedAt: time.Now()} + _, err := processGone(stamped) + assert.ErrorIs(t, err, ErrIdentityUnknown) + + owns, err := OwnsWorker(stamped) + assert.False(t, owns) + assert.ErrorIs(t, err, ErrIdentityUnknown, "neither owned nor gone: unanswerable") + + // And a record with a pid but no start time at all, a ledger row written + // where the kernel could not be asked, is the same answer, not + // "gone, settle it". + owns, err = OwnsWorker(Process{PID: p.PID, PGID: p.PGID}) + assert.False(t, owns) + assert.ErrorIs(t, err, ErrIdentityUnknown) +} diff --git a/internal/connector/driver/proctime_darwin.go b/internal/connector/procid/proctime_darwin.go similarity index 98% rename from internal/connector/driver/proctime_darwin.go rename to internal/connector/procid/proctime_darwin.go index 6c88ddb9b..8e95615ff 100644 --- a/internal/connector/driver/proctime_darwin.go +++ b/internal/connector/procid/proctime_darwin.go @@ -1,4 +1,4 @@ -package driver +package procid import ( "errors" diff --git a/internal/connector/driver/proctime_linux.go b/internal/connector/procid/proctime_linux.go similarity index 91% rename from internal/connector/driver/proctime_linux.go rename to internal/connector/procid/proctime_linux.go index d3f0fdb9a..f36557279 100644 --- a/internal/connector/driver/proctime_linux.go +++ b/internal/connector/procid/proctime_linux.go @@ -1,4 +1,4 @@ -package driver +package procid import ( "bufio" @@ -31,21 +31,21 @@ func readProcStat(pid int) (procStat, error) { // the fields after the last ')' are fixed. end := strings.LastIndexByte(string(raw), ')') if end < 0 { - return procStat{}, errors.New("driver: unreadable /proc stat") + return procStat{}, errors.New("procid: unreadable /proc stat") } fields := strings.Fields(string(raw)[end+1:]) // fields[0] is the state (field 3), fields[2] the process group (field // 5), fields[19] the start time (field 22). if len(fields) < 20 || len(fields[0]) != 1 { - return procStat{}, errors.New("driver: short /proc stat") + return procStat{}, errors.New("procid: short /proc stat") } pgrp, err := strconv.Atoi(fields[2]) if err != nil { - return procStat{}, fmt.Errorf("driver: /proc stat pgrp: %w", err) + return procStat{}, fmt.Errorf("procid: /proc stat pgrp: %w", err) } ticks, err := strconv.ParseInt(fields[19], 10, 64) if err != nil { - return procStat{}, fmt.Errorf("driver: /proc stat starttime: %w", err) + return procStat{}, fmt.Errorf("procid: /proc stat starttime: %w", err) } return procStat{state: fields[0][0], pgrp: pgrp, ticks: ticks}, nil } @@ -127,5 +127,5 @@ func readBootTime() (time.Time, error) { return time.Unix(secs, 0), nil } } - return time.Time{}, errors.New("driver: no btime in /proc/stat") + return time.Time{}, errors.New("procid: no btime in /proc/stat") } diff --git a/internal/connector/driver/proctime_other.go b/internal/connector/procid/proctime_other.go similarity index 76% rename from internal/connector/driver/proctime_other.go rename to internal/connector/procid/proctime_other.go index 0d425c799..bf1c241a3 100644 --- a/internal/connector/driver/proctime_other.go +++ b/internal/connector/procid/proctime_other.go @@ -1,6 +1,6 @@ //go:build unix && !linux && !darwin -package driver +package procid import ( "errors" @@ -10,11 +10,11 @@ import ( // processStartTime is unknown here, so a recorded worker is never signaled: // a pid cannot be told from a later process that reused it. func processStartTime(int) (time.Time, error) { - return time.Time{}, errors.New("driver: process start times are not readable on this platform") + return time.Time{}, errors.New("procid: process start times are not readable on this platform") } // groupRunning cannot list a group here, so a group the kernel still has is // never proven to hold only zombies. func groupRunning(int) (bool, error) { - return false, errors.New("driver: process groups are not listable on this platform") + return false, errors.New("procid: process groups are not listable on this platform") } diff --git a/internal/connector/driver/zombie_darwin_test.go b/internal/connector/procid/zombie_darwin_test.go similarity index 80% rename from internal/connector/driver/zombie_darwin_test.go rename to internal/connector/procid/zombie_darwin_test.go index c31d6c5c5..0e128dcf6 100644 --- a/internal/connector/driver/zombie_darwin_test.go +++ b/internal/connector/procid/zombie_darwin_test.go @@ -1,6 +1,6 @@ //go:build darwin -package driver +package procid import ( "os/exec" @@ -13,7 +13,7 @@ import ( ) // macOS refuses any signal to a process group whose only members are -// zombies. That refusal is the absent group it is on Linux: signalGroup +// zombies. That refusal is the absent group it is on Linux: zeroSignalGroup // answers ESRCH, so the group can be confirmed gone. func TestAZombieOnlyGroupIsNoSuchGroupOnMacOS(t *testing.T) { leader := exec.CommandContext(t.Context(), "/bin/sleep", "300") @@ -26,21 +26,21 @@ func TestAZombieOnlyGroupIsNoSuchGroupOnMacOS(t *testing.T) { require.Eventually(t, func() bool { _, err := processStartTime(pgid); return err != nil }, 5*time.Second, 10*time.Millisecond) require.ErrorIs(t, syscall.Kill(-pgid, 0), syscall.EPERM, "macOS refuses a zombie-only group") - assert.ErrorIs(t, signalGroup(pgid, 0), syscall.ESRCH) + assert.ErrorIs(t, zeroSignalGroup(pgid), syscall.ESRCH) assert.NoError(t, groupGone(pgid)) } // A group whose leader is a zombie but whose child still runs is not gone: -// the signal reaches the child. -func TestAGroupWithALiveMemberIsStillSignaledOnMacOS(t *testing.T) { +// the zero signal reaches the child. +func TestAGroupWithALiveMemberIsStillThereOnMacOS(t *testing.T) { leader := exec.CommandContext(t.Context(), "/bin/sh", "-c", "/bin/sleep 300 & exit 0") leader.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} require.NoError(t, leader.Start()) pgid := leader.Process.Pid - t.Cleanup(func() { _ = signalGroup(pgid, syscall.SIGKILL); _ = leader.Wait() }) + t.Cleanup(func() { _ = syscall.Kill(-pgid, syscall.SIGKILL); _ = leader.Wait() }) require.Eventually(t, func() bool { _, err := processStartTime(pgid); return err != nil }, 5*time.Second, 10*time.Millisecond, "the leader has exited") - err := signalGroup(pgid, 0) + err := zeroSignalGroup(pgid) assert.NoError(t, err, "the live child receives it") assert.Error(t, groupGone(pgid), "a group with a live member is not gone") } diff --git a/internal/connector/driver/zombie_linux_test.go b/internal/connector/procid/zombie_linux_test.go similarity index 80% rename from internal/connector/driver/zombie_linux_test.go rename to internal/connector/procid/zombie_linux_test.go index 6cdbab269..50a6b9145 100644 --- a/internal/connector/driver/zombie_linux_test.go +++ b/internal/connector/procid/zombie_linux_test.go @@ -1,4 +1,4 @@ -package driver +package procid import ( "context" @@ -16,8 +16,8 @@ import ( ) // startUnreaped starts script as the leader of its own group and never waits -// for it until the test ends, the way the connector's own worker sits between -// its exit and the Wait that reaps it. The script runs once stdin closes. +// for it until the test ends, the way a worker sat between its exit and the +// Wait that reaped it. The script runs once stdin closes. func startUnreaped(t *testing.T, script string) (*exec.Cmd, Process) { t.Helper() cmd := exec.CommandContext(context.Background(), "/bin/sh", "-c", "read _; "+script) @@ -45,18 +45,11 @@ func startUnreaped(t *testing.T, script string) (*exec.Cmd, Process) { func TestAGroupOfOnlyAnUnreapedLeaderIsGone(t *testing.T) { _, p := startUnreaped(t, "exit 0") - begin := time.Now() - require.NoError(t, ConfirmGroupGone(p, 2*time.Second)) - assert.Less(t, time.Since(begin), time.Second, "not held for the grace") - assert.False(t, GroupMembersRemain(p)) + assert.NoError(t, groupGone(p.PGID)) owns, err := OwnsWorker(p) assert.False(t, owns, "a zombie is not the worker") assert.NoError(t, err) - - signaled, err := TerminateRecorded(p, 2*time.Second) - assert.False(t, signaled) - assert.NoError(t, err) } // A zombie leader does not make a live member absent. @@ -73,7 +66,7 @@ func TestAnUnreapedLeaderWithALiveChildIsStillHeld(t *testing.T) { return err == nil }, 5*time.Second, 10*time.Millisecond) - assert.True(t, GroupMembersRemain(p)) + assert.ErrorIs(t, groupGone(p.PGID), ErrGroupOutlivedLeader) owns, err := OwnsWorker(p) assert.False(t, owns) assert.ErrorIs(t, err, ErrGroupOutlivedLeader) diff --git a/internal/connector/round7_test.go b/internal/connector/round7_test.go index d294ede2e..69a22d2b5 100644 --- a/internal/connector/round7_test.go +++ b/internal/connector/round7_test.go @@ -40,7 +40,7 @@ func TestDropContentKeepsARecordUpdatedJustAfterAWholeSecondCutoff(t *testing.T) _, err := ledger.RecordSeen(ctx, testEvent(43), LanePoll) require.NoError(t, err) // The lifecycle has no shortcut to completed, so the record walks there. - dispatchForTest(t, ledger, 43) + olderDispatch(t, ledger, 43) require.NoError(t, ledger.SetState(ctx, 43, StateCompleted, "")) dropped, err := ledger.DropContent(ctx, cutoff, cutoff) diff --git a/internal/connector/setup/file.go b/internal/connector/setup/file.go index 9d70c732c..a34d0c3f7 100644 --- a/internal/connector/setup/file.go +++ b/internal/connector/setup/file.go @@ -41,7 +41,6 @@ import ( "regexp" "slices" "strconv" - "time" "github.com/basecamp/basecamp-cli/internal/auth" "github.com/basecamp/basecamp-cli/internal/connector/admission" @@ -119,28 +118,6 @@ type Agent struct { IdentityID int64 `json:"identity_id,omitempty"` } -// Duration is a time.Duration written as Go's duration string ("45m0s"). -type Duration time.Duration - -// MarshalJSON implements json.Marshaler. -func (d Duration) MarshalJSON() ([]byte, error) { - return json.Marshal(time.Duration(d).String()) -} - -// UnmarshalJSON implements json.Unmarshaler. -func (d *Duration) UnmarshalJSON(b []byte) error { - var s string - if err := json.Unmarshal(b, &s); err != nil { - return fmt.Errorf("deadline must be a duration string such as \"45m\": %w", err) - } - v, err := time.ParseDuration(s) - if err != nil { - return fmt.Errorf("deadline: %w", err) - } - *d = Duration(v) - return nil -} - // New is an empty connect.json for a profile, with the spec's defaults. func New(profile string) File { return File{ diff --git a/internal/connector/setup/private.go b/internal/connector/setup/private.go index 7b8c1d956..bbbf7807a 100644 --- a/internal/connector/setup/private.go +++ b/internal/connector/setup/private.go @@ -70,8 +70,7 @@ var ErrLockUnavailable = errors.New("this host cannot lock the connector's polic // // Until #815 removed them, `connect redispatch` and the running connector's // dispatcher took it too, each across one file read and one ledger -// transaction; TryLock was the dispatcher's form. The connector reads -// connect.json without it now. +// transaction. The connector reads connect.json without it now. // // Other locks are taken UNDER this one and never the other way about: the // credential key's lock on setup's final write. That one order is what keeps @@ -111,19 +110,6 @@ const lockPoll = 20 * time.Millisecond // turn, which is what the lock is for. A second setup whose predecessor is // still going after LockWait is refused. func Lock(ctx context.Context, path string) (unlock func(), err error) { - return lock(ctx, path, LockWait) -} - -// TryLock takes the lock if it is free and reports ErrSetupRunning if it is -// not, without waiting. It was the dispatcher's form, so a pass never parked -// behind a `connect setup`'s network checks; since #815 removed the -// dispatcher only this package's tests call it, to probe the lock. -// It takes no context because it does not wait: there is nothing to cancel. -func TryLock(path string) (unlock func(), err error) { - return lock(context.Background(), path, 0) -} - -func lock(ctx context.Context, path string, wait time.Duration) (unlock func(), err error) { if err := ensurePrivateDirs(path); err != nil { return nil, err } @@ -140,7 +126,7 @@ func lock(ctx context.Context, path string, wait time.Duration) (unlock func(), return nil, fmt.Errorf("%w: %s: %w", ErrLockUnavailable, lockPath, err) } lock := flock.New(lockPath, flock.SetPermissions(0o600)) - deadline := time.Now().Add(wait) + deadline := time.Now().Add(LockWait) for { // Before the attempt, not only after it: a context that has already // ended must not come away holding the lock and go on to act under diff --git a/internal/connector/setup/private_unix_test.go b/internal/connector/setup/private_unix_test.go index 233b038da..819eb1f01 100644 --- a/internal/connector/setup/private_unix_test.go +++ b/internal/connector/setup/private_unix_test.go @@ -200,33 +200,12 @@ func TestLockRefusesASecondSetup(t *testing.T) { unlockAgain() } -// The dispatcher's form: it never waits, so a pass that overlaps a setup -// gives up its turn instead of parking on a file another process writes. -func TestTryLockDoesNotWaitForAHolder(t *testing.T) { - path, err := Path(configDir(t), "agent") - require.NoError(t, err) - unlock, err := TryLock(path) - require.NoError(t, err) - - // LockWait is left long on purpose: if TryLock ever waited, this would - // take it rather than return inside the window asserted below. - start := time.Now() - _, err = TryLock(path) - assert.ErrorIs(t, err, ErrSetupRunning) - assert.Less(t, time.Since(start), LockWait/2, "TryLock returns rather than waits") - - unlock() - again, err := TryLock(path) - require.NoError(t, err) - again() -} - -// Lock waits for a holder that finishes inside LockWait, so a `connect -// setup` does not fail because a dispatcher pass happened to overlap it. +// Lock waits for a holder that finishes inside LockWait, so a second +// `connect setup` runs after the first rather than failing. func TestLockWaitsForAHolderThatFinishes(t *testing.T) { path, err := Path(configDir(t), "agent") require.NoError(t, err) - unlock, err := TryLock(path) + unlock, err := Lock(context.Background(), path) require.NoError(t, err) released := make(chan struct{}) @@ -275,7 +254,7 @@ func TestCheckPrivateFileCreatesNothingAndHoldsTheRules(t *testing.T) { func TestLockStopsWaitingWhenTheContextEnds(t *testing.T) { path, err := Path(configDir(t), "agent") require.NoError(t, err) - unlock, err := TryLock(path) + unlock, err := Lock(context.Background(), path) require.NoError(t, err) t.Cleanup(unlock) @@ -307,7 +286,7 @@ func TestLockRefusesAnAlreadyEndedContext(t *testing.T) { assert.Nil(t, held) // Proof that it refused rather than failed to acquire: the lock is free. - free, err := TryLock(path) + free, err := Lock(context.Background(), path) require.NoError(t, err) free() } @@ -344,7 +323,7 @@ func TestLockGivesBackALockItAcquiredForAnEndedContext(t *testing.T) { assert.Nil(t, held) require.Equal(t, 2, ctx.looks, "the wait looked before it tried and again once it held") - free, err := TryLock(path) + free, err := Lock(context.Background(), path) require.NoError(t, err, "and gave back what it had taken: nothing is left holding it") free() } @@ -361,7 +340,7 @@ func TestLockRefusesAnEndedContextWithoutTakingTheLock(t *testing.T) { assert.ErrorIs(t, err, context.Canceled) assert.Nil(t, held) - free, err := TryLock(path) + free, err := Lock(context.Background(), path) require.NoError(t, err) free() } @@ -376,7 +355,7 @@ func TestAnExpiredWaitOnAnEndedContextReportsTheInterruption(t *testing.T) { path, err := Path(configDir(t), "agent") require.NoError(t, err) - unlock, err := TryLock(path) + unlock, err := Lock(context.Background(), path) require.NoError(t, err) t.Cleanup(unlock) diff --git a/internal/connector/worker_era_test.go b/internal/connector/worker_era_test.go new file mode 100644 index 000000000..0b454bfd0 --- /dev/null +++ b/internal/connector/worker_era_test.go @@ -0,0 +1,288 @@ +package connector + +import ( + "context" + "crypto/rand" + "database/sql" + "encoding/hex" + "fmt" + "strconv" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +// What an older build's worker side left in a ledger. +// +// This build writes no task, attempt or task event: the dispatcher that wrote +// them went with #815. The ledgers people have still hold those rows, and this +// build still reads them (status), decides about them (discard, import, a +// hold) and moves records out of the states they left (retention, the +// lifecycle's own guards). The helpers here write the rows the removed +// dispatcher wrote, with the statements it used and in its order, so the +// schema's triggers hold them to the same rules and those paths are tested +// against the ledgers that exist. + +// olderTask is a task an older build launched, and its first attempt. +type olderTask struct { + TaskID int64 + AttemptID string +} + +// olderDispatch puts records on a new task as the removed CreateTask did: at +// delivery admitted, the acknowledgement guard armed where the verdict asked +// for one, each record moved to dispatched in the same transaction. A record a +// test walked to admitted by hand has no instruction, so one is attached +// first; a seen record is admitted first. +func olderDispatch(t *testing.T, l *Ledger, ids ...int64) int64 { + t.Helper() + ctx := context.Background() + for _, id := range ids { + _, err := l.db.ExecContext(ctx, `UPDATE events +SET snapshot = COALESCE(snapshot, CAST('{"content":"do it"}' AS BLOB)), + conversation_key = CASE WHEN conversation_key = '' THEN 'recording:' || id ELSE conversation_key END +WHERE id = ?`, id) + require.NoError(t, err) + if getRecord(t, l, id).State == StateSeen { + require.NoError(t, l.SetState(ctx, id, StateAdmitted, "")) + } + } + var taskID int64 + inTx(t, l, func(ctx context.Context, tx *sql.Tx) { taskID = writeOlderTask(ctx, t, l, tx, ids) }) + return taskID +} + +// olderLaunch is the removed LaunchTask: a task for the records, its first +// attempt launching, and the first record exposed to the worker about to +// start. The records are admitted or queued, with their verdicts. +func olderLaunch(t *testing.T, l *Ledger, ids ...int64) olderTask { + t.Helper() + var out olderTask + inTx(t, l, func(ctx context.Context, tx *sql.Tx) { + out.TaskID = writeOlderTask(ctx, t, l, tx, ids) + var key string + require.NoError(t, tx.QueryRowContext(ctx, `SELECT conversation_key FROM events WHERE id = ?`, ids[0]).Scan(&key)) + now := l.timestamp() + _, err := tx.ExecContext(ctx, ` +UPDATE tasks SET conversation_key = ?, driver = 'claude', originating_event_id = ? WHERE id = ?`, key, ids[0], out.TaskID) + require.NoError(t, err) + out.AttemptID = fmt.Sprintf("att_%024x", out.TaskID) + _, err = tx.ExecContext(ctx, ` +INSERT INTO attempts (id, task_id, seq, driver, state, launched_at) VALUES (?, ?, 1, 'claude', 'launching', ?)`, + out.AttemptID, out.TaskID, now) + require.NoError(t, err) + _, err = tx.ExecContext(ctx, ` +UPDATE task_events SET delivery = 'exposed', exposed_at = ?, exposed_attempt_id = ? +WHERE task_id = ? AND event_id = ?`, now, out.AttemptID, out.TaskID, ids[0]) + require.NoError(t, err) + }) + return out +} + +// writeOlderTask is the removed createTask's writes, in its order. +func writeOlderTask(ctx context.Context, t *testing.T, l *Ledger, tx *sql.Tx, ids []int64) int64 { + t.Helper() + raw := make([]byte, 32) + _, err := rand.Read(raw) + require.NoError(t, err) + res, err := tx.ExecContext(ctx, `INSERT INTO tasks (token_sha256, created_at) VALUES (?, ?)`, hex.EncodeToString(raw), l.timestamp()) + require.NoError(t, err) + taskID, err := res.LastInsertId() + require.NoError(t, err) + for _, id := range ids { + var ( + acknowledge bool + conversation string + ) + require.NoError(t, tx.QueryRowContext(ctx, `SELECT acknowledge, conversation_key FROM events WHERE id = ?`, id).Scan(&acknowledge, &conversation)) + guard := "" + if acknowledge { + guard = "armed" + } + _, err := tx.ExecContext(ctx, `INSERT INTO task_events (task_id, event_id, guard, conversation_key) VALUES (?, ?, ?, ?)`, taskID, id, guard, conversation) + require.NoError(t, err) + } + for _, id := range ids { + moved, err := l.move(ctx, tx, transition{id: id, state: StateDispatched, from: []RecordState{StateAdmitted, StateQueued, StateDispatched}}) + require.NoError(t, err) + require.True(t, moved, "event %d joins the task", id) + } + return taskID +} + +// olderRunning is the removed MarkRunning: the attempt running, with its +// worker's process. A zero started records the pid with no start time, as a +// build that could not read the kernel's did. +func olderRunning(t *testing.T, l *Ledger, attemptID string, pid int, started time.Time) { + t.Helper() + var stamped any + if !started.IsZero() { + stamped = stamp(started) + } + res, err := l.db.ExecContext(context.Background(), ` +UPDATE attempts SET state = 'running', running_at = ?, pid = ?, pgid = ?, process_started = ? +WHERE id = ? AND state = 'launching'`, l.timestamp(), pid, pid, stamped, attemptID) + require.NoError(t, err) + n, err := res.RowsAffected() + require.NoError(t, err) + require.Equal(t, int64(1), n, "attempt %s was launching", attemptID) +} + +// olderTakerUnaccounted is the removed MarkTakerUnaccounted: the task token +// went out and the process holding it could not be named. +func olderTakerUnaccounted(t *testing.T, l *Ledger, attemptID string) { + t.Helper() + _, err := l.db.ExecContext(context.Background(), `UPDATE attempts SET taker_unaccounted = 1 WHERE id = ? AND state <> 'ended'`, attemptID) + require.NoError(t, err) +} + +// olderReport is a worker pulling an event and completing it with outcome, as +// the removed get_dispatch and complete_dispatch wrote it: the delivery +// first, the record after. +func olderReport(t *testing.T, l *Ledger, task olderTask, eventID int64, outcome Outcome) { + t.Helper() + inTx(t, l, func(ctx context.Context, tx *sql.Tx) { + now := l.timestamp() + _, err := tx.ExecContext(ctx, `UPDATE task_events SET delivery = 'exposed', exposed_at = ? WHERE task_id = ? AND event_id = ? AND delivery = 'admitted'`, now, task.TaskID, eventID) + require.NoError(t, err) + _, err = tx.ExecContext(ctx, `UPDATE task_events SET pulled_at = ? WHERE task_id = ? AND event_id = ? AND pulled_at IS NULL`, now, task.TaskID, eventID) + require.NoError(t, err) + _, err = tx.ExecContext(ctx, `UPDATE task_events SET guard = 'canceled' WHERE task_id = ? AND event_id = ? AND guard = 'armed'`, task.TaskID, eventID) + require.NoError(t, err) + _, err = tx.ExecContext(ctx, ` +UPDATE task_events +SET delivery = 'completed', delivered_at = COALESCE(delivered_at, ?), completed_at = ?, outcome = ? +WHERE task_id = ? AND event_id = ?`, now, now, string(outcome), task.TaskID, eventID) + require.NoError(t, err) + moved, err := l.move(ctx, tx, transition{id: eventID, state: StateCompleted, from: []RecordState{StateDispatched}}) + require.NoError(t, err) + require.True(t, moved) + }) +} + +// olderEnd is the removed EndAttempt, for a worker that started: the attempt +// ended with stop, every exposed event the worker did not report settled +// completed(unknown), the task superseded, the events it never exposed +// returned to admitted, and the task ended. +func olderEnd(t *testing.T, l *Ledger, task olderTask, stop string) { + t.Helper() + inTx(t, l, func(ctx context.Context, tx *sql.Tx) { + now := l.timestamp() + _, err := tx.ExecContext(ctx, `UPDATE attempts SET state = 'ended', ended_at = ?, stop_reason = ? WHERE id = ?`, now, stop, task.AttemptID) + require.NoError(t, err) + rows, err := tx.QueryContext(ctx, `SELECT event_id, delivery FROM task_events WHERE task_id = ? AND retired_at IS NULL ORDER BY event_id`, task.TaskID) + require.NoError(t, err) + deliveries := map[int64]string{} + var order []int64 + for rows.Next() { + var ( + id int64 + delivery string + ) + require.NoError(t, rows.Scan(&id, &delivery)) + deliveries[id] = delivery + order = append(order, id) + } + require.NoError(t, rows.Close()) + var unexposed []int64 + for _, id := range order { + switch deliveries[id] { + case "completed": + case "admitted": + unexposed = append(unexposed, id) + default: + moved, err := l.move(ctx, tx, transition{id: id, state: StateCompleted, from: []RecordState{StateDispatched}}) + require.NoError(t, err) + require.True(t, moved) + _, err = tx.ExecContext(ctx, ` +UPDATE task_events SET delivery = 'completed', completed_at = ?, outcome = ? WHERE task_id = ? AND event_id = ?`, + now, string(OutcomeUnknown), task.TaskID, id) + require.NoError(t, err) + } + } + supersedeOlderTask(ctx, t, l, tx, task.TaskID, unexposed) + _, err = tx.ExecContext(ctx, `UPDATE tasks SET ended_at = ? WHERE id = ?`, now, task.TaskID) + require.NoError(t, err) + }) +} + +// supersedeOlderTask is the removed supersedeTask: the token refused, every +// row retired by the trigger, and what the task never exposed returned to +// admitted where nothing else still holds it dispatched. +func supersedeOlderTask(ctx context.Context, t *testing.T, l *Ledger, tx *sql.Tx, taskID int64, unexposed []int64) { + t.Helper() + _, err := tx.ExecContext(ctx, `UPDATE tasks SET superseded_at = COALESCE(superseded_at, ?) WHERE id = ?`, l.timestamp(), taskID) + require.NoError(t, err) + for _, id := range unexposed { + _, err := l.move(ctx, tx, transition{id: id, state: StateAdmitted, from: []RecordState{StateDispatched}}) + require.NoError(t, err) + } +} + +// olderSupersede retires a live task on its own, as the removed SupersedeTask +// did for a task whose worker never started. +func olderSupersede(t *testing.T, l *Ledger, taskID int64) { + t.Helper() + inTx(t, l, func(ctx context.Context, tx *sql.Tx) { + rows, err := tx.QueryContext(ctx, `SELECT event_id FROM task_events WHERE task_id = ? AND retired_at IS NULL AND delivery = 'admitted'`, taskID) + require.NoError(t, err) + var unexposed []int64 + for rows.Next() { + var id int64 + require.NoError(t, rows.Scan(&id)) + unexposed = append(unexposed, id) + } + require.NoError(t, rows.Close()) + supersedeOlderTask(ctx, t, l, tx, taskID, unexposed) + }) +} + +// olderUnknownOutcome takes a fresh record to completed(unknown) the way an +// older build did: admitted, launched, running, lost. +func olderUnknownOutcome(t *testing.T, l *Ledger, id int64) olderTask { + t.Helper() + require.Equal(t, StateAdmitted, opAdmit(t, l, id, "recording:"+strconv.FormatInt(id, 10))) + task := olderLaunch(t, l, id) + olderRunning(t, l, task.AttemptID, 4242, time.Time{}) + olderEnd(t, l, task, "lost") + require.Equal(t, StateCompleted, stateOf(t, l, id)) + return task +} + +// olderPendingRedispatch leaves event 1 completed(failed) on a live task with +// a person's redispatch waiting for that task to end, as the removed +// Redispatch left it: the decision recorded after the outcome settled, the +// record authorized and naming it, and the task's token superseded. +func olderPendingRedispatch(t *testing.T, l *Ledger) olderTask { + t.Helper() + require.Equal(t, StateAdmitted, opAdmit(t, l, 1, "recording:9")) + task := olderLaunch(t, l, 1) + olderReport(t, l, task, 1, OutcomeFailed) + inTx(t, l, func(ctx context.Context, tx *sql.Tx) { + supersedeOlderTask(ctx, t, l, tx, task.TaskID, nil) + var settled string + require.NoError(t, tx.QueryRowContext(ctx, `SELECT completed_at FROM task_events WHERE task_id = ? AND event_id = 1`, task.TaskID).Scan(&settled)) + now := notBefore(l.timestamp(), settled) + res, err := tx.ExecContext(ctx, ` +INSERT INTO decisions (action, event_id, decided_by, decided_at, from_state, from_outcome, to_state, superseded_task_id, note) +VALUES ('redispatch', 1, ?, ?, 'completed', 'failed', 'completed', ?, ?)`, opBy, now, task.TaskID, fmt.Sprintf("waits for task %d to end", task.TaskID)) + require.NoError(t, err) + decision, err := res.LastInsertId() + require.NoError(t, err) + _, err = tx.ExecContext(ctx, `UPDATE events SET authorized_at = ?, authorized_by = ?, redispatch_decision = ? WHERE id = 1`, now, opBy, decision) + require.NoError(t, err) + }) + return task +} + +// inTx runs write in one ledger transaction, as each removed call did. +func inTx(t *testing.T, l *Ledger, write func(ctx context.Context, tx *sql.Tx)) { + t.Helper() + ctx := context.Background() + tx, err := l.db.BeginTx(ctx, nil) + require.NoError(t, err) + defer func() { _ = tx.Rollback() }() + write(ctx, tx) + require.NoError(t, tx.Commit()) +}