diff --git a/.changeset/standby-singleton-readiness.md b/.changeset/standby-singleton-readiness.md new file mode 100644 index 0000000..a6d3240 --- /dev/null +++ b/.changeset/standby-singleton-readiness.md @@ -0,0 +1,7 @@ +--- +"@systemfsoftware/effect-daemon-spec": patch +"@systemfsoftware/effect-daemon-conformance": patch +"@systemfsoftware/effect-daemon-cluster": patch +--- + +Standby singletons no longer report ready before shard acquisition. A cluster singleton whose shard another runner owns now registers as a `standby`: it is not ready, children declared after it do not start until it owns its shard and its run signals ready, and `startTimeoutMillis` is measured from shard acquisition rather than registration. Losing the shard is a handoff, not a failure: the child returns to standby without a termination, restart, or intensity charge. The supervisor starts children in declared order, each gated on the readiness of the ones before it, and status and the trace expose `standby`, `acquired`, `ready` and `released` transitions for every child. diff --git a/packages/effect-daemon-cluster/README.md b/packages/effect-daemon-cluster/README.md index ffa82a4..39eca9e 100644 --- a/packages/effect-daemon-cluster/README.md +++ b/packages/effect-daemon-cluster/README.md @@ -21,9 +21,12 @@ Its peer dependencies, `effect@^4.1.2` and `typescript@>=7.0.0`, come from your an inferred failure. - **The declared loss**: the medium declares `{ reporting: 'inferred', groupStop: 'eventual' }`, so the conformance kit holds it only to what it can observe and honour. -- **Standby and handoff**: a singleton whose shard another runner owns is announced ready at once - and waits; it starts when the shard arrives. A run interrupted because its shard moved away is a - handoff, not a termination: the child stays registered and nothing is reported. +- **Standby and handoff**: a singleton whose shard another runner owns registers as a standby: it is + not ready, it is not bound by the start deadline, and children declared after it do not start until + it owns its shard and its run signals ready. The start deadline is armed when the shard arrives, so + `startTimeoutMillis` bounds the real post-acquisition initialization. A run interrupted because its + shard moved away is a handoff, not a termination: the child returns to standby, stays registered, + and nothing is reported, restarted or charged against the restart intensity. - **Singletons across processes over Postgres**: `ClusterMedium.sqlRunnerLayer` runs `Sharding` over `SqlRunnerStorage` with no-op runner transport, health and message storage. The oldest live runner owns every shard, so a joining process waits as a standby, and a runner whose sessions are diff --git a/packages/effect-daemon-cluster/src/ClusterMedium/medium.ts b/packages/effect-daemon-cluster/src/ClusterMedium/medium.ts index 17329b5..894916a 100644 --- a/packages/effect-daemon-cluster/src/ClusterMedium/medium.ts +++ b/packages/effect-daemon-cluster/src/ClusterMedium/medium.ts @@ -1,6 +1,20 @@ import { Supervisor } from '@systemfsoftware/effect-daemon-spec' -import { Cause, Deferred, Duration, Effect, Exit, Layer, Match, Option, Ref, Result, Scope } from 'effect' -import { EntityId, Sharding } from 'effect/cluster' +import { + Cause, + Deferred, + Duration, + Effect, + Exit, + Layer, + Match, + Option, + Queue, + Ref, + Result, + Scope, + Stream, +} from 'effect' +import { Sharding } from 'effect/cluster' import { absurd } from 'effect/Function' import type { ClusterProgram, ClusterProgramRequirements, EntityChild, SingletonChild } from './ClusterProgram.js' @@ -22,14 +36,25 @@ interface ClusterStarted extends Supervisor.Medium.Started { readonly ended: Deferred.Deferred readonly stopping: Ref.Ref readonly probe: Effect.Effect + readonly activityQueue?: Queue.Queue | undefined } +const activityExtras = ( + activity?: Supervisor.Medium.StartedActivity, + activityQueue?: Queue.Queue, +): { + readonly activity?: Supervisor.Medium.StartedActivity + readonly activityQueue?: Queue.Queue | undefined +} => activity === undefined ? {} : { activity, activityQueue } + const clusterStarted = ( ready: Effect.Effect, registration: Scope.Closeable, ended: Deferred.Deferred, stopping: Ref.Ref, probe: Effect.Effect, + activity?: Supervisor.Medium.StartedActivity, + activityQueue?: Queue.Queue, ): ClusterStarted => ({ ...Supervisor.Medium.started(ready), [ClusterStartedTypeId]: ClusterStartedTypeId, @@ -37,6 +62,7 @@ const clusterStarted = ( ended, stopping, probe, + ...activityExtras(activity, activityQueue), }) const isClusterStarted = (evidence: Supervisor.Medium.Started): evidence is ClusterStarted => @@ -78,11 +104,39 @@ const reportedOnExit = ( lostItsShard(stopped, exit) ? Effect.void : Effect.asVoid(Deferred.succeed(ended, terminationOf(stopped, exit))), ) -const ownsSingleton = (sharding: Sharding.Sharding['Service'], name: string, shardGroup: string | undefined) => - sharding.hasShardId(sharding.getShardId(EntityId.make(name), shardGroup ?? 'default')) +const announceAcquire = (changes: Queue.Queue): Effect.Effect => + Queue.offer(changes, { _tag: 'Acquired' }).pipe(Effect.asVoid) -const readyWhen = (owned: boolean, signalled: Deferred.Deferred): Effect.Effect => - owned ? Deferred.await(signalled) : Effect.void +const singletonReady = ( + signalled: Deferred.Deferred, + changes: Queue.Queue, +): Effect.Effect => + Effect.flatMap( + Deferred.isDone(signalled), + (alreadySignalled) => alreadySignalled ? Effect.void : Effect.asVoid(Deferred.succeed(signalled, void 0)), + ).pipe( + Effect.andThen(Queue.offer(changes, { _tag: 'Ready' })), + Effect.asVoid, + ) + +const singletonRun = ( + program: SingletonChild, + signalled: Deferred.Deferred, + ended: Deferred.Deferred, + stopping: Ref.Ref, + changes: Queue.Queue, +): Effect.Effect => + Effect.gen(function*() { + yield* announceAcquire(changes) + yield* program.run(singletonReady(signalled, changes)) + }).pipe( + Effect.onExit((exit) => + Effect.flatMap(Ref.get(stopping), (stopped) => + lostItsShard(stopped, exit) + ? Effect.asVoid(Queue.offer(changes, { _tag: 'Released' })) + : reportedOnExit(ended, stopping)(exit)) + ), + ) const startSingleton = ( program: SingletonChild, @@ -95,15 +149,21 @@ const startSingleton = ( const ended = yield* Deferred.make() const stopping = yield* Ref.make(false) const sharding = yield* Sharding.Sharding - const run = program.run(Deferred.succeed(signalled, void 0)).pipe( - Effect.onExit(reportedOnExit(ended, stopping)), - ) + const changes = yield* Queue.unbounded() + const run = singletonRun(program, signalled, ended, stopping, changes) yield* sharding.registerSingleton(program.name, run, { shardGroup }).pipe( Effect.provideService(Scope.Scope, registration), ) - const ready = readyWhen(ownsSingleton(sharding, program.name, shardGroup), signalled) const probe = Deferred.isDone(ended).pipe(Effect.map((done) => done === false)) - return clusterStarted(ready, registration, ended, stopping, probe) + return clusterStarted( + Deferred.await(signalled), + registration, + ended, + stopping, + probe, + { initiallyStandby: true, changes: Stream.fromQueue(changes) }, + changes, + ) }) const startEntity = ( @@ -155,6 +215,9 @@ const probeOf = ( onSome: (self) => self.probe, }) +const shutdownActivityOf = (self: ClusterStarted): Effect.Effect => + self.activityQueue === undefined ? Effect.void : Queue.shutdown(self.activityQueue).pipe(Effect.ignore) + const stopOf = ( evidence: Supervisor.Medium.Started, ): Effect.Effect => @@ -163,6 +226,7 @@ const stopOf = ( onSome: (self) => Ref.set(self.stopping, true).pipe( Effect.andThen(Scope.close(self.registration, Exit.void)), + Effect.andThen(shutdownActivityOf(self)), Effect.andThen(Deferred.succeed(self.ended, shutdownTermination)), Effect.as(Supervisor.Medium.stopped), ), diff --git a/packages/effect-daemon-cluster/tests/cluster-medium.conformance.test.ts b/packages/effect-daemon-cluster/tests/cluster-medium.conformance.test.ts index 6f2d050..f6836a1 100644 --- a/packages/effect-daemon-cluster/tests/cluster-medium.conformance.test.ts +++ b/packages/effect-daemon-cluster/tests/cluster-medium.conformance.test.ts @@ -114,13 +114,14 @@ const standbyThenOwned = Effect.gen(function*() { const shards = yield* ShardScript const { runs, program } = yield* countedSingleton const evidence = yield* medium.start(program) - yield* evidence.ready + const readyWhileStandby = yield* Effect.timeoutOption(evidence.ready, Duration.millis(100)) const aliveAsStandby = yield* medium.probe(evidence) const runsAsStandby = yield* Ref.get(runs) yield* shards.grant(SINGLETON) + yield* evidence.ready const startedOnceOwned = yield* runsReach(runs, 1) const reported = yield* reportedWithin(medium.report(evidence)) - return { aliveAsStandby, runsAsStandby, startedOnceOwned, reported } + return { readyWhileStandby, aliveAsStandby, runsAsStandby, startedOnceOwned, reported } }) const handedOffAndBack = Effect.gen(function*() { @@ -141,7 +142,6 @@ const standbyStopped = Effect.gen(function*() { const { medium } = yield* ClusterMedium.port const { program } = yield* countedSingleton const evidence = yield* medium.start(program) - yield* evidence.ready yield* medium.stop(evidence, { _tag: 'Brutal' }) const reported = yield* reportedWithin(medium.report(evidence)) yield* nothingLeftRegistered @@ -163,13 +163,15 @@ Feature('Supervising children hosted by the cluster', { timeout: 120_000 }) 'observed', (s) => Effect.provide(standbyThenOwned, s.environment), ), - Then('it was ready and alive without running, ran once its shard arrived, and nothing ended')((s, expect) => - expect(s.observed).toEqual({ - aliveAsStandby: true, - runsAsStandby: 0, - startedOnceOwned: true, - reported: Option.none(), - }) + Then('it was not ready yet stayed alive without running, ran once its shard arrived, and nothing ended')( + (s, expect) => + expect(s.observed).toEqual({ + readyWhileStandby: Option.none(), + aliveAsStandby: true, + runsAsStandby: 0, + startedOnceOwned: true, + reported: Option.none(), + }), ), ), ) diff --git a/packages/effect-daemon-cluster/tests/cluster-standby-lifecycle.integration.test.ts b/packages/effect-daemon-cluster/tests/cluster-standby-lifecycle.integration.test.ts new file mode 100644 index 0000000..229d91e --- /dev/null +++ b/packages/effect-daemon-cluster/tests/cluster-standby-lifecycle.integration.test.ts @@ -0,0 +1,599 @@ +import { ClusterMedium } from '@systemfsoftware/effect-daemon-cluster' +import { Supervisor } from '@systemfsoftware/effect-daemon-spec' +import { Gherkin, Given, it, makeFeature, Then, When } from '@systemfsoftware/effect-gherkin-spec' +import { Array as Arr, Duration, Effect, Layer, Match, Queue, Ref, Stream } from 'effect' +import { RegistrationLedger, scriptedShardingOwning, ShardScript } from './__fixtures__/scripted-sharding.fixture.js' + +const Feature = makeFeature({ it }) + +const SINGLETON = 'orders/settlement' +const DEPENDENT = 'orders/receipts' + +type Statuses = ReadonlyArray + +const statusesOf = (handle: Supervisor.RunningSupervisor): Effect.Effect => + Effect.map(Supervisor.statusOf(handle), (state) => + Match.value(state).pipe( + Match.tag('Terminated', () => Arr.empty()), + Match.orElse((running) => + Arr.map(running.core.children, (child) => [child.childId, child.generation, child.status] as const) + ), + )) + +const heldNames: Effect.Effect, never, RegistrationLedger> = Effect.flatten( + Effect.map(Effect.service(RegistrationLedger), (ledger) => ledger.held), +) + +const readyAfter = (millis: Duration.Input, ready: Effect.Effect): Effect.Effect => + Effect.sleep(millis).pipe(Effect.andThen(ready), Effect.asVoid) + +const traceCollector = (handle: Supervisor.RunningSupervisor) => + Effect.map(Ref.make>([]), (trace) => ({ + trace, + collect: Stream.runForEach(Supervisor.traceOf(handle), (entry) => Ref.update(trace, (seen) => [...seen, entry])) + .pipe(Effect.forkScoped, Effect.asVoid), + })) + +const eventLabel = (entry: Supervisor.TraceEntry): string => + Match.value(entry.event).pipe( + Match.tag('ChildStarted', (started) => `started:${started.childId}:${started.generation}`), + Match.tag('ChildStandby', (standby) => `standby:${standby.childId}:${standby.generation}`), + Match.tag('ChildAcquired', (acquired) => `acquired:${acquired.childId}:${acquired.generation}`), + Match.tag('ChildReady', (ready) => `ready:${ready.childId}:${ready.generation}`), + Match.tag('ChildReleased', (released) => `released:${released.childId}:${released.generation}`), + Match.tag('ChildTerminated', (terminated) => `terminated:${terminated.childId}:${terminated.generation}`), + Match.orElse(() => 'other'), + ) + +const countOf = (labels: ReadonlyArray, prefix: string): number => + labels.filter((label) => label.startsWith(prefix)).length + +const firstIndexOf = (labels: ReadonlyArray, wanted: string): number => labels.indexOf(wanted) + +const reacquiredTimerIsNotKilled = Effect.gen(function*() { + const shards = yield* ShardScript + const singleton = ClusterMedium.singletonChild({ + name: 'orders/epoch', + run: (ready) => + Effect.gen(function*() { + yield* readyAfter('250 millis', ready) + return yield* Effect.never + }), + }) + const handle = yield* Supervisor.make('epoch-tree').pipe( + Supervisor.strategy('one_for_one'), + Supervisor.intensity(3, 60_000), + Supervisor.children([ + Supervisor.ChildSpecs.on(ClusterMedium.port)('epoch', singleton, { startTimeoutMillis: 400 }), + ]), + ).scoped + const collector = yield* traceCollector(handle) + yield* collector.collect + yield* Effect.sleep('50 millis') + yield* shards.grant('orders/epoch') + yield* Effect.sleep('300 millis') + yield* shards.revoke('orders/epoch') + yield* Effect.sleep('50 millis') + yield* shards.grant('orders/epoch') + yield* Effect.sleep('500 millis') + const statuses = yield* statusesOf(handle) + const labels = yield* Effect.map(Ref.get(collector.trace), (entries) => Arr.map(entries, eventLabel)) + return { statuses, terminated: countOf(labels, 'terminated'), readyEvents: countOf(labels, 'ready:epoch') } +}) + +const interruptedBeforeReadyStaysStandby = Effect.gen(function*() { + const shards = yield* ShardScript + const slow = ClusterMedium.singletonChild({ + name: 'orders/slow', + run: (ready) => + Effect.gen(function*() { + yield* readyAfter('100 millis', ready) + return yield* Effect.never + }), + }) + const watchdog = ClusterMedium.singletonChild({ + name: 'orders/watchdog', + run: (ready) => Effect.andThen(ready, Effect.never), + }) + const handle = yield* Supervisor.make('interrupted-tree').pipe( + Supervisor.strategy('rest_for_one'), + Supervisor.intensity(3, 60_000), + Supervisor.children([ + Supervisor.ChildSpecs.on(ClusterMedium.port)('slow', slow, { startTimeoutMillis: 2_000 }), + Supervisor.ChildSpecs.on(ClusterMedium.port)('watchdog', watchdog, { startTimeoutMillis: 2_000 }), + ]), + ).scoped + const collector = yield* traceCollector(handle) + yield* collector.collect + yield* Effect.sleep('50 millis') + yield* shards.grant('orders/slow') + yield* Effect.sleep('20 millis') + yield* shards.revoke('orders/slow') + yield* Effect.sleep('150 millis') + const statuses = yield* statusesOf(handle) + const held = yield* heldNames + const labels = yield* Effect.map(Ref.get(collector.trace), (entries) => Arr.map(entries, eventLabel)) + return { + statuses, + held, + slowReadyEvents: countOf(labels, 'ready:slow'), + watchdogStarts: countOf(labels, 'started:watchdog'), + terminated: countOf(labels, 'terminated'), + } +}) + +const oneForOneRestartIsUngated = Effect.gen(function*() { + const crashes = yield* Queue.unbounded() + const shards = yield* ShardScript + const gate = ClusterMedium.singletonChild({ + name: 'orders/gate', + run: (ready) => Effect.andThen(ready, Effect.never), + }) + const worker = ClusterMedium.singletonChild({ + name: 'orders/worker', + run: (ready) => + Effect.gen(function*() { + yield* ready + yield* Queue.take(crashes) + return yield* Effect.die('crashed') + }), + }) + const handle = yield* Supervisor.make('ungated-tree').pipe( + Supervisor.strategy('one_for_one'), + Supervisor.intensity(3, 60_000), + Supervisor.children([ + Supervisor.ChildSpecs.on(ClusterMedium.port)('gate', gate, { startTimeoutMillis: 2_000 }), + Supervisor.ChildSpecs.on(ClusterMedium.port)('worker', worker, { startTimeoutMillis: 2_000 }), + ]), + ).scoped + const collector = yield* traceCollector(handle) + yield* collector.collect + yield* shards.grant('orders/gate') + yield* Effect.sleep('50 millis') + yield* shards.grant('orders/worker') + yield* Effect.sleep('100 millis') + yield* shards.revoke('orders/gate') + yield* Effect.sleep('50 millis') + const standbyStatuses = yield* statusesOf(handle) + yield* Queue.offer(crashes, void 0) + yield* Effect.sleep('100 millis') + const restartedStatuses = yield* statusesOf(handle) + const labels = yield* Effect.map(Ref.get(collector.trace), (entries) => Arr.map(entries, eventLabel)) + return { + standbyStatuses, + restartedStatuses, + workerStarts: countOf(labels, 'started:worker'), + terminated: countOf(labels, 'terminated'), + } +}) + +const acquisitionDuringBackoffIsKept = Effect.gen(function*() { + const crashes = yield* Queue.unbounded() + const shards = yield* ShardScript + const leader = ClusterMedium.singletonChild({ + name: 'orders/leader', + run: (ready) => + Effect.gen(function*() { + yield* readyAfter('150 millis', ready) + return yield* Effect.never + }), + }) + const worker = ClusterMedium.singletonChild({ + name: 'orders/worker', + run: (ready) => + Effect.gen(function*() { + yield* ready + yield* Queue.take(crashes) + return yield* Effect.die('crashed') + }), + }) + const handle = yield* Supervisor.make('backoff-tree').pipe( + Supervisor.strategy('one_for_all'), + Supervisor.intensity(3, 60_000), + Supervisor.backoff({ baseMillis: 100, multiplier: 1, capMillis: 100 }), + Supervisor.children([ + Supervisor.ChildSpecs.on(ClusterMedium.port)('leader', leader, { startTimeoutMillis: 2_000 }), + Supervisor.ChildSpecs.on(ClusterMedium.port)('worker', worker, { startTimeoutMillis: 2_000 }), + ]), + ).scoped + const collector = yield* traceCollector(handle) + yield* collector.collect + yield* shards.grant('orders/leader') + yield* shards.grant('orders/worker') + yield* Effect.sleep('300 millis') + yield* Queue.offer(crashes, void 0) + yield* Effect.sleep('500 millis') + const statuses = (yield* statusesOf(handle)).slice().sort((left, right) => left[0] < right[0] ? -1 : 1) + const labels = yield* Effect.map(Ref.get(collector.trace), (entries) => Arr.map(entries, eventLabel)) + return { + statuses, + acquisitions: countOf(labels, 'acquired:leader'), + terminated: countOf(labels, 'terminated'), + } +}) + +const ownedAtRegistrationStillAnnouncesStandby = Effect.gen(function*() { + const { medium } = yield* ClusterMedium.port + const probe = ClusterMedium.singletonChild({ + name: 'orders/probe', + run: (ready) => Effect.andThen(ready, Effect.never), + }) + const evidence = yield* medium.start(probe) + const declaredStandby = evidence.activity?.initiallyStandby === true + yield* evidence.ready + yield* medium.stop(evidence, { _tag: 'Brutal' }) + const singleton = ClusterMedium.singletonChild({ + name: 'orders/owned', + run: (ready) => Effect.andThen(ready, Effect.never), + }) + const handle = yield* Supervisor.make('owned-tree').pipe( + Supervisor.strategy('one_for_one'), + Supervisor.intensity(3, 60_000), + Supervisor.children([ + Supervisor.ChildSpecs.on(ClusterMedium.port)('owned', singleton, { startTimeoutMillis: 2_000 }), + ]), + ).scoped + yield* Effect.sleep('200 millis') + const statuses = yield* statusesOf(handle) + return { declaredStandby, statuses } +}) + +const dynamicStartWaitsForDeclaredReadiness = Effect.gen(function*() { + const shards = yield* ShardScript + const gate = ClusterMedium.singletonChild({ + name: 'orders/dyn-gate', + run: (ready) => Effect.andThen(ready, Effect.never), + }) + const handle = yield* Supervisor.make('dynamic-tree').pipe( + Supervisor.strategy('one_for_one'), + Supervisor.intensity(3, 60_000), + Supervisor.dynamic({ ceiling: 2, restartType: 'temporary' }), + Supervisor.children([ + Supervisor.ChildSpecs.on(ClusterMedium.port)('gate', gate, { startTimeoutMillis: 2_000 }), + ]), + ).scoped + const collector = yield* traceCollector(handle) + yield* collector.collect + yield* Effect.sleep('50 millis') + const reply = yield* Supervisor.startChild(handle, Supervisor.readyOnStart(Effect.never)) + const heldWhileStandby = yield* heldNames + yield* Effect.sleep('50 millis') + yield* shards.grant('orders/dyn-gate') + yield* Effect.sleep('100 millis') + const statuses = yield* statusesOf(handle) + const heldAfterGrant = yield* heldNames + return { reply, heldWhileStandby, statuses, heldAfterGrant } +}) + +interface LifecycleState { + readonly handle: Supervisor.RunningSupervisor + readonly trace: Ref.Ref> + readonly runs: Ref.Ref + readonly initialized: Ref.Ref + readonly dependentSaw: Ref.Ref +} + +const lifecycleTree = Effect.gen(function*() { + const runs = yield* Ref.make(0) + const initialized = yield* Ref.make(false) + const dependentSaw = yield* Ref.make(null) + const singleton = ClusterMedium.singletonChild({ + name: SINGLETON, + run: (ready) => + Effect.gen(function*() { + yield* Ref.update(runs, (count) => count + 1) + yield* Ref.set(initialized, true) + yield* ready + return yield* Effect.never + }), + }) + const dependent = ClusterMedium.singletonChild({ + name: DEPENDENT, + run: (ready) => + Effect.gen(function*() { + const saw = yield* Ref.get(initialized) + yield* Ref.set(dependentSaw, saw) + yield* ready + return yield* Effect.never + }), + }) + const handle = yield* Supervisor.make('standby-tree').pipe( + Supervisor.strategy('rest_for_one'), + Supervisor.intensity(1, 60_000), + Supervisor.children([ + Supervisor.ChildSpecs.on(ClusterMedium.port)('settlement', singleton, { startTimeoutMillis: 2_000 }), + Supervisor.ChildSpecs.on(ClusterMedium.port)('dependent', dependent, { startTimeoutMillis: 2_000 }), + ]), + ).scoped + const collector = yield* traceCollector(handle) + yield* collector.collect + return { + handle, + trace: collector.trace, + runs, + initialized, + dependentSaw, + } satisfies LifecycleState +}) + +const traceLabelsOf = (state: LifecycleState): Effect.Effect> => + Effect.map(Ref.get(state.trace), (entries) => Arr.map(entries, eventLabel)) + +const standbyToHandoffAndBack = Effect.gen(function*() { + const state = yield* lifecycleTree + const shards = yield* ShardScript + yield* Effect.sleep('50 millis') + const bootStatuses = yield* statusesOf(state.handle) + const bootHeld = yield* heldNames + yield* shards.grant(SINGLETON) + yield* shards.grant(DEPENDENT) + yield* Effect.sleep('50 millis') + const activeStatuses = yield* statusesOf(state.handle) + const activeHeld = yield* heldNames + const runsAfterPromotion = yield* Ref.get(state.runs) + const dependentSawInitialized = yield* Ref.get(state.dependentSaw) + const activeLabels = yield* traceLabelsOf(state) + const dependentStartedAfterSingletonReady = firstIndexOf(activeLabels, 'ready:settlement:0') < firstIndexOf( + activeLabels, + 'started:dependent:0', + ) + yield* shards.revoke(SINGLETON) + yield* Effect.sleep('50 millis') + const handedOffStatuses = yield* statusesOf(state.handle) + const handedOffLabels = yield* traceLabelsOf(state) + const handoffLeftDependentRunning = countOf(handedOffLabels, 'terminated') === 0 && + countOf(handedOffLabels, 'started:dependent') === 1 && + countOf(handedOffLabels, 'released:settlement') === 1 + yield* shards.grant(SINGLETON) + yield* Effect.sleep('50 millis') + const repromotedStatuses = yield* statusesOf(state.handle) + const repromotedLabels = yield* traceLabelsOf(state) + const runsAfterRepromotion = yield* Ref.get(state.runs) + const supervisorRunning = Arr.length(repromotedStatuses) > 0 + yield* Supervisor.shutdown(state.handle) + return { + bootStatuses, + bootHeld, + activeStatuses, + activeHeld, + runsAfterPromotion, + dependentSawInitialized, + dependentStartedAfterSingletonReady, + handedOffStatuses, + handoffLeftDependentRunning, + repromotedStatuses, + supervisorRunning, + runsAfterRepromotion, + acquisitions: countOf(repromotedLabels, 'acquired:settlement'), + dependentStarts: countOf(repromotedLabels, 'started:dependent'), + terminated: countOf(repromotedLabels, 'terminated'), + } +}) + +const timeoutEventsOf = (entries: ReadonlyArray): ReadonlyArray => + Arr.map(entries, (entry) => + Match.value(entry.event).pipe( + Match.tag('SupervisorStarted', () => 'supervisor-started'), + Match.tag('ChildStarted', () => 'started'), + Match.tag('ChildStandby', () => 'standby'), + Match.tag('ChildAcquired', () => 'acquired'), + Match.tag('ChildTerminated', () => 'terminated'), + Match.tag('TimerElapsed', () => 'timer'), + Match.orElse(() => 'other'), + )) + +const postAcquisitionTimeout = Effect.gen(function*() { + const neverReady = ClusterMedium.singletonChild({ + name: 'orders/hanging', + run: () => Effect.never, + }) + const handle = yield* Supervisor.make('timeout-tree').pipe( + Supervisor.strategy('one_for_one'), + Supervisor.intensity(0, 60_000), + Supervisor.children([ + Supervisor.ChildSpecs.on(ClusterMedium.port)('hanging', neverReady, { startTimeoutMillis: 200 }), + ]), + ).scoped + const collector = yield* traceCollector(handle) + yield* collector.collect + yield* Effect.sleep('400 millis') + const standbyStatuses = yield* statusesOf(handle) + const shards = yield* ShardScript + yield* shards.grant('orders/hanging') + yield* Effect.sleep('2 seconds') + const afterTimeoutStatuses = yield* statusesOf(handle) + const events = yield* Effect.map(Ref.get(collector.trace), timeoutEventsOf) + const acquiredAt = firstIndexOf(events, 'acquired') + const fromAcquisition = acquiredAt === -1 ? events : Arr.drop(events, acquiredAt) + return { standbyStatuses, afterTimeoutStatuses, fromAcquisition } +}) + +Feature('Standby singletons under shard movement', { timeout: 120_000 }) + .withLayer(Layer.empty) + .live('each scenario drives the simulation kernel itself, and a conformance check cannot run inside a kernel run') + .body(({ scenario }) => { + scenario( + 'A rest-for-one dependent starts only after the standby singleton owns its shard and runs ready, and handoff is not a failure', + Gherkin.Do.pipe( + Given('a cluster runner that owns no shard yet')( + 'environment', + () => Layer.build(Layer.merge(ClusterMedium.layer(), scriptedShardingOwning(false))), + ), + When('the singleton acquires its shard, hands off, and is promoted again')( + 'observed', + (s) => Effect.provide(standbyToHandoffAndBack, s.environment), + ), + Then('standby, acquisition, ordering, handoff and re-promotion are all visible with no termination')( + (s, expect) => + expect(s.observed).toEqual({ + bootStatuses: [['settlement', 0, 'standby'], ['dependent', 0, 'starting']], + bootHeld: [SINGLETON], + activeStatuses: [['settlement', 0, 'ready'], ['dependent', 0, 'ready']], + activeHeld: [SINGLETON, DEPENDENT], + runsAfterPromotion: 1, + dependentSawInitialized: true, + dependentStartedAfterSingletonReady: true, + handedOffStatuses: [['settlement', 0, 'standby'], ['dependent', 0, 'ready']], + handoffLeftDependentRunning: true, + repromotedStatuses: [['settlement', 0, 'ready'], ['dependent', 0, 'ready']], + supervisorRunning: true, + runsAfterRepromotion: 2, + acquisitions: 2, + dependentStarts: 1, + terminated: 0, + }), + ), + ), + ) + + scenario( + 'A standby singleton has no start deadline until its shard arrives, and the post-acquisition start is bounded', + Gherkin.Do.pipe( + Given('a cluster runner that owns no shard yet')( + 'environment', + () => Layer.build(Layer.merge(ClusterMedium.layer(), scriptedShardingOwning(false))), + ), + When('a never-ready singleton waits standby longer than its start timeout and then acquires its shard')( + 'observed', + (s) => Effect.provide(postAcquisitionTimeout, s.environment), + ), + Then('nothing fired while standby, and the deadline missed only after acquisition')((s, expect) => + expect(s.observed).toEqual({ + standbyStatuses: [['hanging', 0, 'standby']], + afterTimeoutStatuses: [], + fromAcquisition: ['acquired', 'timer', 'terminated'], + }) + ), + ), + ) + + scenario( + 'A re-acquired singleton is not killed by the deadline armed before its shard moved away', + Gherkin.Do.pipe( + Given('a cluster runner that owns no shard yet')( + 'environment', + () => Layer.build(Layer.merge(ClusterMedium.layer(), scriptedShardingOwning(false))), + ), + When('the singleton is promoted, handed off, and promoted again past the first deadline')( + 'observed', + (s) => Effect.provide(reacquiredTimerIsNotKilled, s.environment), + ), + Then('the stale deadline missed nothing and the singleton is ready')((s, expect) => + expect(s.observed).toEqual({ + statuses: [['epoch', 0, 'ready']], + terminated: 0, + readyEvents: 2, + }) + ), + ), + ) + + scenario( + 'A singleton interrupted before it signals ready stays standby and its dependent never starts', + Gherkin.Do.pipe( + Given('a cluster runner that owns no shard yet')( + 'environment', + () => Layer.build(Layer.merge(ClusterMedium.layer(), scriptedShardingOwning(false))), + ), + When('the shard is revoked while the singleton still initializes')( + 'observed', + (s) => Effect.provide(interruptedBeforeReadyStaysStandby, s.environment), + ), + Then('nothing was announced ready, nothing terminated, and no dependent started')((s, expect) => + expect(s.observed).toEqual({ + statuses: [['slow', 0, 'standby'], ['watchdog', 0, 'starting']], + held: ['orders/slow'], + slowReadyEvents: 0, + watchdogStarts: 0, + terminated: 0, + }) + ), + ), + ) + + scenario( + 'An independent one-for-one child restarts immediately while its standby singleton waits', + Gherkin.Do.pipe( + Given('a cluster runner that owns no shard yet')( + 'environment', + () => Layer.build(Layer.merge(ClusterMedium.layer(), scriptedShardingOwning(false))), + ), + When('the singleton hands off and the fiber sibling crashes')( + 'observed', + (s) => Effect.provide(oneForOneRestartIsUngated, s.environment), + ), + Then('the sibling restarted at once, ungated by the standby singleton')((s, expect) => + expect(s.observed).toEqual({ + standbyStatuses: [['gate', 0, 'standby'], ['worker', 0, 'ready']], + restartedStatuses: [['gate', 0, 'standby'], ['worker', 1, 'ready']], + workerStarts: 2, + terminated: 1, + }) + ), + ), + ) + + scenario( + 'A singleton whose shard is already local still announces standby and then acquires', + Gherkin.Do.pipe( + Given('a cluster runner that owns every shard')( + 'environment', + () => Layer.build(Layer.merge(ClusterMedium.layer(), scriptedShardingOwning(true))), + ), + When('a singleton is supervised on that runner')( + 'observed', + (s) => Effect.provide(ownedAtRegistrationStillAnnouncesStandby, s.environment), + ), + Then('the medium declares standby regardless of ownership, and the singleton is ready')( + (s, expect) => + expect(s.observed).toEqual({ + declaredStandby: true, + statuses: [['owned', 0, 'ready']], + }), + ), + ), + ) + + scenario( + 'A singleton acquiring during a group backoff keeps its acquisition and timer', + Gherkin.Do.pipe( + Given('a cluster runner that owns no shard yet')( + 'environment', + () => Layer.build(Layer.merge(ClusterMedium.layer(), scriptedShardingOwning(false))), + ), + When('a group restart is backoff-pending while the singleton re-acquires')( + 'observed', + (s) => Effect.provide(acquisitionDuringBackoffIsKept, s.environment), + ), + Then('the acquisition during backoff was kept and both children came back')((s, expect) => + expect(s.observed).toEqual({ + statuses: [['leader', 1, 'ready'], ['worker', 1, 'ready']], + acquisitions: 2, + terminated: 2, + }) + ), + ), + ) + + scenario( + 'A dynamic child waits for the declared standby singleton before it starts', + Gherkin.Do.pipe( + Given('a cluster runner that owns no shard yet')( + 'environment', + () => Layer.build(Layer.merge(ClusterMedium.layer(), scriptedShardingOwning(false))), + ), + When('a dynamic child is requested while the singleton is standby and the shard then arrives')( + 'observed', + (s) => Effect.provide(dynamicStartWaitsForDeclaredReadiness, s.environment), + ), + Then('the request was accepted but the child only ran once the singleton was ready')((s, expect) => + expect(s.observed).toEqual({ + reply: { outcome: 'accepted', childId: 'd0', generation: 0 }, + heldWhileStandby: ['orders/dyn-gate'], + statuses: [['gate', 0, 'ready'], ['d0', 0, 'ready']], + heldAfterGrant: ['orders/dyn-gate'], + }) + ), + ), + ) + }) diff --git a/packages/effect-daemon-conformance/src/observed-trace.ts b/packages/effect-daemon-conformance/src/observed-trace.ts index eb4ab95..17611a9 100644 --- a/packages/effect-daemon-conformance/src/observed-trace.ts +++ b/packages/effect-daemon-conformance/src/observed-trace.ts @@ -1,5 +1,5 @@ import type { Supervisor } from '@systemfsoftware/effect-daemon-spec' -import { Array as Arr, Match } from 'effect' +import { Array as Arr, Match, Option } from 'effect' import type { ChildRef, CommandKind, @@ -68,36 +68,45 @@ const timerChildOf = (target: TimerTarget): ChildRef | null => Match.exhaustive, ) -const eventOf = (event: Event): ObservedEvent => +const observedSome = ( + kind: EventKind, + child: ChildRef | null, + reason: ReasonKind | null, +): Option.Option => Option.some(observedEvent(kind, child, reason)) + +const eventOf = (event: Event): Option.Option => Match.value(event).pipe( - Match.tag('SupervisorStarted', () => observedEvent('SupervisorStarted', null, null)), + Match.tag('SupervisorStarted', () => observedSome('SupervisorStarted', null, null)), Match.tag('ChildStarted', (started) => - observedEvent('ChildStarted', childRef(started.childId, started.generation), null)), + observedSome('ChildStarted', childRef(started.childId, started.generation), null)), Match.tag('ChildReady', (ready) => - observedEvent('ChildReady', childRef(ready.childId, ready.generation), null)), + observedSome('ChildReady', childRef(ready.childId, ready.generation), null)), + Match.tag('ChildStandby', () => Option.none()), + Match.tag('ChildAcquired', () => Option.none()), + Match.tag('ChildReleased', () => Option.none()), Match.tag( 'ChildTerminated', (terminated) => - observedEvent( + observedSome( 'ChildTerminated', childRef(terminated.childId, terminated.generation), reasonKindOf(terminated.reason), ), ), Match.tag('ChildStopped', (stopped) => - observedEvent('ChildStopped', childRef(stopped.childId, stopped.generation), null)), + observedSome('ChildStopped', childRef(stopped.childId, stopped.generation), null)), Match.tag('ProbeResult', (probe) => - observedEvent('ProbeResult', childRef(probe.childId, probe.generation), null)), + observedSome('ProbeResult', childRef(probe.childId, probe.generation), null)), Match.tag('TimerElapsed', (timer) => - observedEvent('TimerElapsed', timerChildOf(timer.target), null)), - Match.tag('DynamicStartRequested', () => observedEvent('DynamicStartRequested', null, null)), + observedSome('TimerElapsed', timerChildOf(timer.target), null)), + Match.tag('DynamicStartRequested', () => observedSome('DynamicStartRequested', null, null)), Match.tag( 'DynamicStopRequested', - (stopping) => observedEvent('DynamicStopRequested', childRef(stopping.childId, stopping.generation), null), + (stopping) => observedSome('DynamicStopRequested', childRef(stopping.childId, stopping.generation), null), ), Match.tag( 'ShutdownRequested', - (shutdown) => observedEvent('ShutdownRequested', null, reasonKindOf(shutdown.reason)), + (shutdown) => observedSome('ShutdownRequested', null, reasonKindOf(shutdown.reason)), ), Match.exhaustive, ) @@ -137,11 +146,12 @@ const decisionOf = (decision: Decision): ObservedDecision => Match.exhaustive, ) -const stepOf = (entry: TraceEntry): ObservedStep => ({ - event: eventOf(entry.event), - decision: decisionOf(entry.decision), -}) +const stepOf = (entry: TraceEntry): Option.Option => + Option.map(eventOf(entry.event), (event) => ({ event, decision: decisionOf(entry.decision) })) + +const isIgnoredStep = (step: ObservedStep): boolean => + step.event.kind === 'TimerElapsed' && step.decision.kind === 'Stale' /** Reduces a supervisor's kernel trace to the steps a pairwise comparison reads. */ export const observedStepsOf = (entries: ReadonlyArray): ReadonlyArray => - Arr.map(entries, stepOf) + Arr.filter(Arr.flatMap(entries, (entry) => Option.toArray(stepOf(entry))), (step) => isIgnoredStep(step) === false) diff --git a/packages/effect-daemon-spec/README.md b/packages/effect-daemon-spec/README.md index c228ee4..53c0578 100644 --- a/packages/effect-daemon-spec/README.md +++ b/packages/effect-daemon-spec/README.md @@ -56,6 +56,8 @@ Use `app.scoped` instead of `app.layer` to get the running supervisor handle ins `Supervisor.intensity(n, periodMillis)` allows at most `n` restarts within `periodMillis`. The default is OTP's: one restart in five seconds. One more restart than that stops every child and terminates the supervisor; whoever is waiting on it observes a give-up (see [Observing a running supervisor](#observing-a-running-supervisor)). +Children start in declared order, and a later child starts only once the children before it are ready (or have exited): a dependent never starts against an uninitialized sibling. A child that must wait for a resource before its real startup — a cluster medium singleton waiting for its shard, say — registers as a `standby` instead of reporting ready, and its start timeout is measured from when the resource arrives, not from registration. + ## Children `Supervisor.ChildSpecs.make(id, program, options)` declares a child. Every option is optional; the first three default as OTP does. diff --git a/packages/effect-daemon-spec/src/Supervisor/Medium.ts b/packages/effect-daemon-spec/src/Supervisor/Medium.ts index f07e87a..c37463d 100644 --- a/packages/effect-daemon-spec/src/Supervisor/Medium.ts +++ b/packages/effect-daemon-spec/src/Supervisor/Medium.ts @@ -1,4 +1,4 @@ -import { Context, Effect } from 'effect' +import { Context, Effect, type Stream } from 'effect' import type { ShutdownMode } from '../kernel/SupervisorPolicy.schema.js' import type { TerminationReason } from '../kernel/TerminationReport.schema.js' export { BrutalShutdown, GracefulShutdown, InfinityShutdown, ShutdownMode } from '../kernel/SupervisorPolicy.schema.js' @@ -13,7 +13,9 @@ export { ShutdownTermination, TerminationReason, } from '../kernel/TerminationReport.schema.js' +export { Acquired, MediumActivity, Released } from './MediumActivity.schema.js' export { GroupStopGuarantee, MediumDeclaration, MediumReporting } from './MediumDeclaration.schema.js' +import type { MediumActivity } from './MediumActivity.schema.js' import type { MediumDeclaration } from './MediumDeclaration.schema.js' export const StartedTypeId: unique symbol = Symbol.for('@systemfsoftware/effect-daemon-spec/Medium/Started') @@ -22,16 +24,15 @@ export type StartedTypeId = typeof StartedTypeId export const StoppedTypeId: unique symbol = Symbol.for('@systemfsoftware/effect-daemon-spec/Medium/Stopped') export type StoppedTypeId = typeof StoppedTypeId -/** - * Evidence that a medium started a child (KTD8): unforgeable, because only a - * medium's `start` port can mint it. It carries the readiness signal the kernel - * races against its start-deadline timer (R10); the termination report and the - * single-shot liveness probe are the medium's `report` and `probe` ports run - * over this evidence, so neither can observe a child that never started. - */ +export interface StartedActivity { + readonly initiallyStandby: boolean + readonly changes: Stream.Stream +} + export interface Started { readonly [StartedTypeId]: StartedTypeId readonly ready: Effect.Effect + readonly activity?: StartedActivity | undefined } /** diff --git a/packages/effect-daemon-spec/src/Supervisor/MediumActivity.schema.ts b/packages/effect-daemon-spec/src/Supervisor/MediumActivity.schema.ts new file mode 100644 index 0000000..b228f90 --- /dev/null +++ b/packages/effect-daemon-spec/src/Supervisor/MediumActivity.schema.ts @@ -0,0 +1,13 @@ +import { Schema } from 'effect' + +export const Acquired = Schema.TaggedStruct('Acquired', {}) +export type Acquired = typeof Acquired.Type + +export const Ready = Schema.TaggedStruct('Ready', {}) +export type Ready = typeof Ready.Type + +export const Released = Schema.TaggedStruct('Released', {}) +export type Released = typeof Released.Type + +export const MediumActivity = Schema.Union([Acquired, Ready, Released]) +export type MediumActivity = typeof MediumActivity.Type diff --git a/packages/effect-daemon-spec/src/Supervisor/supervisor-commands.ts b/packages/effect-daemon-spec/src/Supervisor/supervisor-commands.ts index d231f20..7617704 100644 --- a/packages/effect-daemon-spec/src/Supervisor/supervisor-commands.ts +++ b/packages/effect-daemon-spec/src/Supervisor/supervisor-commands.ts @@ -12,9 +12,10 @@ import { Ref, Schema, Scope, + Stream, } from 'effect' import type { SupervisionEvent, TimerKind } from '../kernel/SupervisionEvent.schema.js' -import type { ChildId, Generation } from '../kernel/SupervisionLimits.schema.js' +import type { ChildId, Epoch, Generation } from '../kernel/SupervisionLimits.schema.js' import { ReplyStartAccepted } from '../kernel/SupervisorCommand.schema.js' import type { StartChild, @@ -30,7 +31,7 @@ import type { IntensityExceededExit } from '../kernel/TerminationReport.schema.j import type { BoundChild, Incarnation } from './bound-child.js' import { outcomeOf } from './DynamicOutcome.schema.js' import * as FiberMedium from './FiberMedium.js' -import type { Started } from './Medium.js' +import type { Started, StartedActivity } from './Medium.js' import type { RunningSupervisor } from './running-supervisor.handle.js' import { boundChildrenOf, @@ -69,6 +70,27 @@ const startedEventOf = (childId: ChildId, generation: Generation): SupervisionEv generation, }) +const standbyEventOf = (childId: ChildId, generation: Generation): SupervisionEvent => ({ + _tag: 'ChildStandby', + at: 0, + childId, + generation, +}) + +const acquiredEventOf = (childId: ChildId, generation: Generation): SupervisionEvent => ({ + _tag: 'ChildAcquired', + at: 0, + childId, + generation, +}) + +const releasedEventOf = (childId: ChildId, generation: Generation): SupervisionEvent => ({ + _tag: 'ChildReleased', + at: 0, + childId, + generation, +}) + const readyEventOf = (childId: ChildId, generation: Generation): SupervisionEvent => ({ _tag: 'ChildReady', at: 0, @@ -101,11 +123,12 @@ const timerEventOf = ( kind: TimerKind, childId: ChildId, generation: Generation, + epoch: Epoch, ): SupervisionEvent => ({ _tag: 'TimerElapsed', at: 0, kind, - target: { _tag: 'ChildTimer', childId, generation }, + target: { _tag: 'ChildTimer', childId, generation, epoch }, }) const supervisorTimerEventOf = (kind: TimerKind): SupervisionEvent => ({ @@ -137,6 +160,14 @@ const watchReadiness = ( evidence: Started, childId: ChildId, generation: Generation, +): Effect.Effect => + evidence.activity === undefined ? forkReadinessWatch(handle, evidence, childId, generation) : Effect.void + +const forkReadinessWatch = ( + handle: RunningSupervisor, + evidence: Started, + childId: ChildId, + generation: Generation, ): Effect.Effect => Effect.asVoid( Effect.forkIn( @@ -145,6 +176,33 @@ const watchReadiness = ( ), ) +const watchActivity = ( + handle: RunningSupervisor, + evidence: Started, + childId: ChildId, + generation: Generation, +): Effect.Effect => + evidence.activity === undefined ? Effect.void : forkActivityWatch(handle, evidence.activity, childId, generation) + +const forkActivityWatch = ( + handle: RunningSupervisor, + activity: StartedActivity, + childId: ChildId, + generation: Generation, +): Effect.Effect => + Effect.asVoid( + Effect.forkIn( + Stream.runForEach(activity.changes, (change) => + Match.value(change).pipe( + Match.tag('Acquired', () => stampedNow(handle, acquiredEventOf(childId, generation))), + Match.tag('Ready', () => stampedNow(handle, readyEventOf(childId, generation))), + Match.tag('Released', () => stampedNow(handle, releasedEventOf(childId, generation))), + Match.exhaustive, + )), + ownerScopeOf(handle), + ), + ) + const watchReport = ( handle: RunningSupervisor, child: BoundChild, @@ -187,8 +245,14 @@ const executeStart = ( Effect.andThen( stampedNow(acquired.handle, startedEventOf(command.childId, command.generation)), Effect.andThen( - watchReadiness(acquired.handle, incarnation.evidence, command.childId, command.generation), - watchReport(acquired.handle, child, incarnation, command.childId, command.generation), + announceStandby(acquired.handle, incarnation.evidence, command.childId, command.generation), + Effect.andThen( + watchReadiness(acquired.handle, incarnation.evidence, command.childId, command.generation), + Effect.andThen( + watchActivity(acquired.handle, incarnation.evidence, command.childId, command.generation), + watchReport(acquired.handle, child, incarnation, command.childId, command.generation), + ), + ), ), ), ), @@ -200,6 +264,21 @@ const executeStart = ( }) }) +const initiallyStandbyOf = (evidence: Started): boolean => { + const activity = evidence.activity + return activity === undefined ? false : activity.initiallyStandby +} + +const announceStandby = ( + handle: RunningSupervisor, + evidence: Started, + childId: ChildId, + generation: Generation, +): Effect.Effect => + initiallyStandbyOf(evidence) + ? stampedNow(handle, standbyEventOf(childId, generation)) + : Effect.void + /** * Moves each answered request's pending child: an accepted start binds it to the * child the kernel allocated, and every answered request drops its pending entry. @@ -259,7 +338,7 @@ const executeArm = ( ): Effect.Effect => Match.value(arm).pipe( Match.tag('ArmChildTimer', (timer) => - armTimer(acquired, timer.deadline, timerEventOf(timer.kind, timer.childId, timer.generation))), + armTimer(acquired, timer.deadline, timerEventOf(timer.kind, timer.childId, timer.generation, timer.epoch))), Match.tag('ArmSupervisorTimer', (timer) => armTimer(acquired, timer.deadline, supervisorTimerEventOf(timer.kind))), Match.exhaustive, diff --git a/packages/effect-daemon-spec/src/__tests__/interpret-supervision-event.workflow.property.test.ts b/packages/effect-daemon-spec/src/__tests__/interpret-supervision-event.workflow.property.test.ts index 0be0f98..4330eb1 100644 --- a/packages/effect-daemon-spec/src/__tests__/interpret-supervision-event.workflow.property.test.ts +++ b/packages/effect-daemon-spec/src/__tests__/interpret-supervision-event.workflow.property.test.ts @@ -82,10 +82,11 @@ const declaredChild = (childId: string, restartType: RestartType, significant: b probeFailureThreshold: 2, }) -const childOf = (childId: string, generation: number, status: ChildStatus): ChildInstance => ({ +const childOf = (childId: string, generation: number, status: ChildStatus, epoch = 0): ChildInstance => ({ childId, generation, status, + epoch, consecutiveRestarts: 0, probeFailures: 0, }) @@ -94,7 +95,7 @@ const coreWith = ( policy: SupervisionPolicy, children: ReadonlyArray, restartStamps: ReadonlyArray, -): SupervisorCore => ({ policy, children, restartStamps, nextOrdinal: 0 }) +): SupervisorCore => ({ policy, children, restartStamps, nextOrdinal: 0, awaiting: [] }) const runningOf = (core: SupervisorCore): SupervisorState => new Running({ core }) @@ -139,7 +140,7 @@ const childTimerEvent = (kind: TimerKind, childId: string, generation: number, a _tag: 'TimerElapsed', at, kind, - target: { _tag: 'ChildTimer', childId, generation }, + target: { _tag: 'ChildTimer', childId, generation, epoch: 0 }, }) const supervisorTimerEvent = (kind: TimerKind, at: EventTime): SupervisionEvent => ({ @@ -404,6 +405,8 @@ const textOf = (items: ReadonlyArray): ReadonlyArray => const idSeqOf = (children: ReadonlyArray): ReadonlyArray => Arr.map(children, (child) => child.childId) +const awaitingIdsOf = (core: SupervisorCore): ReadonlyArray => Arr.map(core.awaiting, (start) => start.childId) + const labelOf = (child: ChildInstance): string => `${child.childId}:${child.generation}:${child.status}` const startIdsFromDecision = (decision: SupervisionDecision): ReadonlyArray => @@ -492,7 +495,8 @@ it.prop( Match.value(coreOf(decision)).pipe( Match.tag('Some', ({ value }) => sameSeq(stopIdsOf(commandsOf(decision)), Arr.reverse(groupIds)) && - sameSeq(startIdsOf(commandsOf(decision)), comingIds) && + sameSeq(startIdsOf(commandsOf(decision)), Arr.take(comingIds, 1)) && + sameSeq(awaitingIdsOf(value), Arr.drop(comingIds, 1)) && sameSeq(textOf(value.restartStamps), textOf([at])) && sameSeq( Arr.map( @@ -599,6 +603,51 @@ it.prop( }, ) +it.prop( + '∀c_CoolDown_=ResetsEpoch', + { of: [PositiveMillis, EventTime], subject: interpretSupervisionEvent }, + (subject, [millis, at0]) => { + const t0 = at0 + const policy = policyWith({ + strategy: 'one_for_one', + intensity: 1, + coolDown: { _tag: 'CoolDownAfter', millis }, + childDeclarations: [declaredChild('c0', 'permanent', false)], + }) + const cooled = new CoolingDown({ + core: coreWith(policy, [childOf('c0', 2, 'ready', 3)], []), + millis, + }) + const resumedAt = t0 + 1 + millis + const resumed = folded(subject, cooled, [supervisorTimerEvent('cool_down', resumedAt)]) + return Match.value(lastDecisionOf(resumed)).pipe( + Match.tag('StartChildren', (decision) => + Match.value(coreOf(decision)).pipe( + Match.tag('Some', ({ value }) => + Match.value(Arr.findFirst(value.children, (child) => + child.childId === 'c0')).pipe( + Match.tag('Some', ({ value: child }) => + child.epoch === 0 && + Arr.some(commandsOf(decision).arms, (arm) => + Match.value(arm).pipe( + Match.tag('ArmChildTimer', (timer) => + timer.kind === 'start_deadline' && timer.childId === 'c0' && timer.epoch === 0 && + timer.deadline === resumedAt + 50), + Match.tag('ArmSupervisorTimer', () => false), + Match.exhaustive, + ))), + Match.tag('None', () => false), + Match.exhaustive, + )), + Match.tag('None', () => + false), + Match.exhaustive, + )), + Match.orElse(() => false), + ) + }, +) + it.prop( '∀g_WindowPrune_≡Period', { of: [PositiveMillis, EventTime], subject: interpretSupervisionEvent }, @@ -643,7 +692,12 @@ it.prop( return isRestartFamily(decision) && stamped && Arr.length(stopIdsOf(commandsOf(decision))) === 3 && - Arr.length(startIdsOf(commandsOf(decision))) === 3 + sameSeq(startIdsOf(commandsOf(decision)), ['c0']) && + Match.value(coreOf(decision)).pipe( + Match.tag('Some', ({ value }) => sameSeq(awaitingIdsOf(value), ['c1', 'c2'])), + Match.tag('None', () => false), + Match.exhaustive, + ) }, ) @@ -1060,7 +1114,10 @@ it.prop( strategy: 'one_for_one', childDeclarations: [declaredChild(childId, 'permanent', false)], }) - const state = new Restarting({ core: coreWith(policy, [], []), pending: [{ childId, generation: 1 }] }) + const state = new Restarting({ + core: coreWith(policy, [childOf(childId, 1, 'starting')], []), + pending: [{ childId, generation: 1 }], + }) const decision = decidedOf(subject, stepOf(state, childTimerEvent('backoff', childId, 1, 10))) const started = startIdsOf(commandsOf(decision)) diff --git a/packages/effect-daemon-spec/src/__tests__/standby-acquisition.workflow.property.test.ts b/packages/effect-daemon-spec/src/__tests__/standby-acquisition.workflow.property.test.ts new file mode 100644 index 0000000..1b0a5b6 --- /dev/null +++ b/packages/effect-daemon-spec/src/__tests__/standby-acquisition.workflow.property.test.ts @@ -0,0 +1,389 @@ +import { it } from '@systemfsoftware/vitest' +import { Array as Arr, Match, Result } from 'effect' +import { absurd } from 'effect/Function' +import { evolveSupervisor, SupervisionEvolution } from '../kernel/evolve-supervisor.workflow.js' +import { + interpretSupervisionEvent, + type SupervisionDecision, + SupervisionStep, +} from '../kernel/interpret-supervision-event.workflow.js' +import type { SupervisionEvent } from '../kernel/SupervisionEvent.schema.js' +import { EventTime, Generation } from '../kernel/SupervisionLimits.schema.js' +import type { SupervisorArm } from '../kernel/SupervisorCommand.schema.js' +import { SupervisionPolicy } from '../kernel/SupervisorPolicy.schema.js' +import type { RestartType } from '../kernel/SupervisorPolicy.schema.js' +import type { ChildInstance, SupervisorCore } from '../kernel/SupervisorState.schema.js' +import { Restarting, Running } from '../kernel/SupervisorState.schema.js' +import type { SupervisorState } from '../kernel/SupervisorState.schema.js' +import type { TerminationReason } from '../kernel/TerminationReport.schema.js' + +type Interpret = typeof interpretSupervisionEvent + +const PERIOD_MILLIS = 100 +const START_TIMEOUT_MILLIS = 50 +const LIVENESS_TICK_MILLIS = 10 + +const policyWith = ( + strategy: 'one_for_one' | 'rest_for_one' | 'one_for_all', + childDeclarations: ReadonlyArray<{ + readonly childId: string + readonly restartType: RestartType + }>, +): SupervisionPolicy => ({ + strategy, + intensity: 8, + periodMillis: PERIOD_MILLIS, + autoShutdown: 'never', + coolDown: { _tag: 'NoCoolDown' }, + backoff: { baseMillis: 0, multiplier: 2, capMillis: 0 }, + dynamic: { _tag: 'NoDynamicChildren' }, + livenessTickMillis: LIVENESS_TICK_MILLIS, + childDeclarations: Arr.map(childDeclarations, (child) => ({ + childId: child.childId, + restartType: child.restartType, + shutdown: { _tag: 'Brutal' } as const, + significant: false, + startTimeoutMillis: START_TIMEOUT_MILLIS, + probeFailureThreshold: 2, + })), +}) + +const childOf = ( + childId: string, + generation: Generation, + status: ChildInstance['status'], + epoch = 0, +): ChildInstance => ({ + childId, + generation, + status, + epoch, + consecutiveRestarts: 0, + probeFailures: 0, +}) + +const coreWith = ( + policy: SupervisionPolicy, + children: ReadonlyArray, + awaiting: ReadonlyArray<{ readonly childId: string; readonly generation: Generation }>, +): SupervisorCore => ({ policy, children, restartStamps: [], nextOrdinal: 0, awaiting }) + +const runningOf = (core: SupervisorCore): SupervisorState => new Running({ core }) + +const decidedOf = (interpret: Interpret, step: SupervisionStep): SupervisionDecision => + Result.match(interpret(step), { + onFailure: (error: never): never => absurd(error), + onSuccess: (decision) => decision, + }) + +const stepOf = (state: SupervisorState, event: SupervisionEvent): SupervisionStep => + new SupervisionStep({ state, event }) + +const startDeadlineTimerOf = ( + childId: string, + generation: Generation, + epoch: number, + at: EventTime, +): SupervisionEvent => ({ + _tag: 'TimerElapsed', + at, + kind: 'start_deadline', + target: { _tag: 'ChildTimer', childId, generation, epoch }, +}) + +const acquiredEventOf = (childId: string, generation: Generation, at: EventTime): SupervisionEvent => ({ + _tag: 'ChildAcquired', + at, + childId, + generation, +}) + +const releasedEventOf = (childId: string, generation: Generation, at: EventTime): SupervisionEvent => ({ + _tag: 'ChildReleased', + at, + childId, + generation, +}) + +const readyEventOf = (childId: string, generation: Generation, at: EventTime): SupervisionEvent => ({ + _tag: 'ChildReady', + at, + childId, + generation, +}) + +const terminatedEventOf = ( + childId: string, + generation: Generation, + at: EventTime, + reason: TerminationReason, +): SupervisionEvent => ({ _tag: 'ChildTerminated', at, childId, generation, reason }) + +const abnormalOf = (cause: string): TerminationReason => ({ + _tag: 'Abnormal', + report: { _tag: 'CauseReport', cause }, +}) + +const isStale = (decision: SupervisionDecision): boolean => + Match.value(decision).pipe( + Match.tag('Stale', () => true), + Match.orElse(() => false), + ) + +const statusOf = (decision: SupervisionDecision, childId: string): string | null => + Match.value(decision).pipe( + Match.tag('Continue', ({ core }) => + Match.value(Arr.findFirst(core.children, (child) => + child.childId === childId)).pipe( + Match.tag('Some', ({ value }) => value.status), + Match.tag('None', () => null), + Match.exhaustive, + )), + Match.orElse(() => + null + ), + ) + +const epochOf = (decision: SupervisionDecision, childId: string): number | null => + Match.value(decision).pipe( + Match.tag('Continue', ({ core }) => + Match.value(Arr.findFirst(core.children, (child) => + child.childId === childId)).pipe( + Match.tag('Some', ({ value }) => value.epoch), + Match.tag('None', () => null), + Match.exhaustive, + )), + Match.orElse(() => + null + ), + ) + +const startedIdsOf = (decision: SupervisionDecision): ReadonlyArray => + Match.value(decision).pipe( + Match.tag('Continue', ({ commands }) => Arr.map(commands.starts, (start) => start.childId)), + Match.tag('StartChildren', ({ commands }) => Arr.map(commands.starts, (start) => start.childId)), + Match.orElse((): ReadonlyArray => []), + ) + +const isStartDeadlineFor = (arm: SupervisorArm, childId: string, epoch: number, at: EventTime): boolean => + Match.value(arm).pipe( + Match.tag( + 'ArmChildTimer', + (timer) => + timer.kind === 'start_deadline' && timer.childId === childId && timer.epoch === epoch && + timer.deadline === at + START_TIMEOUT_MILLIS, + ), + Match.orElse(() => false), + ) + +const armedDeadlineFor = (decision: SupervisionDecision, childId: string, epoch: number, at: EventTime): boolean => + Match.value(decision).pipe( + Match.tag( + 'Continue', + ({ commands }) => Arr.some(commands.arms, (arm) => isStartDeadlineFor(arm, childId, epoch, at)), + ), + Match.orElse(() => false), + ) + +it.prop( + '∀s_StandbyDeadline_=Stale', + { of: [EventTime, Generation], subject: interpretSupervisionEvent }, + (subject, [at, generation]) => { + const policy = policyWith('rest_for_one', [{ childId: 'c0', restartType: 'permanent' }]) + const state = runningOf(coreWith(policy, [childOf('c0', generation, 'standby')], [])) + return isStale(decidedOf(subject, stepOf(state, startDeadlineTimerOf('c0', generation, 0, at)))) + }, +) + +it.prop( + '∀a_Acquired_=StartingWithEpochDeadline', + { of: [EventTime, Generation], subject: interpretSupervisionEvent }, + (subject, [at, generation]) => { + const policy = policyWith('rest_for_one', [{ childId: 'c0', restartType: 'permanent' }]) + const state = runningOf(coreWith(policy, [childOf('c0', generation, 'standby')], [])) + const decision = decidedOf(subject, stepOf(state, acquiredEventOf('c0', generation, at))) + return statusOf(decision, 'c0') === 'starting' && epochOf(decision, 'c0') === 1 && + armedDeadlineFor(decision, 'c0', 1, at) + }, +) + +it.prop( + '∀r_Released_=StandbyWithoutTermination', + { of: [EventTime, Generation], subject: interpretSupervisionEvent }, + (subject, [at, generation]) => { + const policy = policyWith('rest_for_one', [{ childId: 'c0', restartType: 'permanent' }]) + const state = runningOf(coreWith(policy, [childOf('c0', generation, 'ready')], [])) + const decision = decidedOf(subject, stepOf(state, releasedEventOf('c0', generation, at))) + return Match.value(decision).pipe( + Match.tag( + 'Continue', + ({ commands }) => + statusOf(decision, 'c0') === 'standby' && commands.stops.length === 0 && commands.starts.length === 0 && + commands.terminates.length === 0, + ), + Match.orElse(() => false), + ) + }, +) + +it.prop( + '∀r_ReadyWhileStandby_=Held', + { of: [EventTime, Generation], subject: interpretSupervisionEvent }, + (subject, [at, generation]) => { + const policy = policyWith('rest_for_one', [{ childId: 'c0', restartType: 'permanent' }]) + const state = runningOf(coreWith(policy, [childOf('c0', generation, 'standby')], [])) + const decision = decidedOf(subject, stepOf(state, readyEventOf('c0', generation, at))) + return statusOf(decision, 'c0') === 'standby' && isStale(decision) === false + }, +) + +it.prop( + '∀p_AcquiredInRestarting_=KeepsPending', + { of: [EventTime, Generation], subject: interpretSupervisionEvent }, + (subject, [at, generation]) => { + const policy = policyWith('one_for_all', [ + { childId: 'c0', restartType: 'permanent' }, + { childId: 'c1', restartType: 'permanent' }, + ]) + const core = coreWith(policy, [ + childOf('c0', generation, 'standby'), + childOf('c1', generation, 'starting'), + ], []) + const state = new Restarting({ core, pending: [{ childId: 'c1', generation: generation + 1 }] }) + const decision = decidedOf(subject, stepOf(state, acquiredEventOf('c0', generation, at))) + return Result.match( + evolveSupervisor(new SupervisionEvolution({ state, decision })), + { + onFailure: () => false, + onSuccess: (evolved) => + Match.value(evolved).pipe( + Match.tag('Restarting', (restarting) => + restarting.pending.length === 1 && + Match.value(Arr.findFirst(restarting.core.children, (child) => child.childId === 'c0')).pipe( + Match.tag('Some', ({ value }) => value.status === 'starting' && value.epoch === 1), + Match.tag('None', () => false), + Match.exhaustive, + )), + Match.orElse(() => false), + ), + }, + ) + }, +) + +it.prop( + '∀e_StaleEpochDeadline_=Stale', + { of: [EventTime, Generation], subject: interpretSupervisionEvent }, + (subject, [at, generation]) => { + const policy = policyWith('one_for_one', [{ childId: 'c0', restartType: 'permanent' }]) + const state = runningOf(coreWith(policy, [childOf('c0', generation, 'starting', 2)], [])) + return isStale(decidedOf(subject, stepOf(state, startDeadlineTimerOf('c0', generation, 1, at)))) + }, +) + +it.prop( + '∀e_CurrentEpochDeadline_=Missed', + { of: [EventTime, Generation], subject: interpretSupervisionEvent }, + (subject, [at, generation]) => { + const policy = policyWith('one_for_one', [{ childId: 'c0', restartType: 'permanent' }]) + const state = runningOf(coreWith(policy, [childOf('c0', generation, 'starting', 2)], [])) + const decision = decidedOf(subject, stepOf(state, startDeadlineTimerOf('c0', generation, 2, at))) + return isStale(decision) === false && + Arr.some(startedIdsOf(decision), (childId) => childId === 'c0') + }, +) + +it.prop( + '∀o_OneForOneRestart_=Ungated', + { of: [EventTime, Generation], subject: interpretSupervisionEvent }, + (subject, [at, generation]) => { + const policy = policyWith('one_for_one', [ + { childId: 'c0', restartType: 'permanent' }, + { childId: 'c1', restartType: 'permanent' }, + ]) + const state = runningOf(coreWith(policy, [ + childOf('c0', generation, 'standby'), + childOf('c1', generation, 'ready'), + ], [])) + const decision = decidedOf( + subject, + stepOf(state, terminatedEventOf('c1', generation, at, abnormalOf('boom'))), + ) + return Arr.contains(startedIdsOf(decision), 'c1') + }, +) + +it.prop( + '∀q_AcquisitionBlocksAwaiting_=NoStart', + { of: [EventTime, Generation], subject: interpretSupervisionEvent }, + (subject, [at, generation]) => { + const policy = policyWith('rest_for_one', [ + { childId: 'c0', restartType: 'permanent' }, + { childId: 'c1', restartType: 'permanent' }, + ]) + const state = runningOf(coreWith(policy, [ + childOf('c0', generation, 'standby'), + childOf('c1', generation, 'starting'), + ], [{ childId: 'c1', generation }])) + const acquired = decidedOf(subject, stepOf(state, acquiredEventOf('c0', generation, at))) + return statusOf(acquired, 'c0') === 'starting' && Arr.contains(startedIdsOf(acquired), 'c1') === false + }, +) + +it.prop( + '∀q_ReadinessDrainsAwaiting_=StartNext', + { of: [EventTime, Generation], subject: interpretSupervisionEvent }, + (subject, [at, generation]) => { + const policy = policyWith('rest_for_one', [ + { childId: 'c0', restartType: 'permanent' }, + { childId: 'c1', restartType: 'permanent' }, + ]) + const state = runningOf(coreWith(policy, [ + childOf('c0', generation, 'starting', 1), + childOf('c1', generation, 'starting'), + ], [{ childId: 'c1', generation }])) + const readied = decidedOf(subject, stepOf(state, readyEventOf('c0', generation, at))) + return statusOf(readied, 'c0') === 'ready' && Arr.contains(startedIdsOf(readied), 'c1') && + armedDeadlineFor(readied, 'c1', 0, at) + }, +) + +it.prop( + '∀q_RemovedPredecessor_=UnblocksAwaiting', + { of: [EventTime, Generation], subject: interpretSupervisionEvent }, + (subject, [at, generation]) => { + const policy = policyWith('rest_for_one', [ + { childId: 'c0', restartType: 'temporary' }, + { childId: 'c1', restartType: 'permanent' }, + ]) + const state = runningOf(coreWith(policy, [ + childOf('c0', generation, 'ready'), + childOf('c1', generation, 'starting'), + ], [{ childId: 'c1', generation }])) + const terminated = decidedOf( + subject, + stepOf(state, terminatedEventOf('c0', generation, at, { _tag: 'Normal' })), + ) + return Arr.contains(startedIdsOf(terminated), 'c1') + }, +) + +it.prop( + '∀d_DeadEntry_=PrunedOnDrain', + { of: [EventTime, Generation], subject: interpretSupervisionEvent }, + (subject, [at, generation]) => { + const policy = policyWith('rest_for_one', [ + { childId: 'c0', restartType: 'permanent' }, + { childId: 'c1', restartType: 'permanent' }, + ]) + const state = runningOf(coreWith(policy, [ + childOf('c0', generation, 'starting'), + childOf('c1', generation, 'starting'), + ], [{ childId: 'gone', generation: 5 }, { childId: 'c1', generation }])) + const readied = decidedOf(subject, stepOf(state, readyEventOf('c0', generation, at))) + return Match.value(readied).pipe( + Match.tag('Continue', ({ core }) => Arr.contains(startedIdsOf(readied), 'c1') && core.awaiting.length === 0), + Match.orElse(() => false), + ) + }, +) diff --git a/packages/effect-daemon-spec/src/__tests__/supervisor-boot.workflow.property.test.ts b/packages/effect-daemon-spec/src/__tests__/supervisor-boot.workflow.property.test.ts index 10b2eb2..fef7b24 100644 --- a/packages/effect-daemon-spec/src/__tests__/supervisor-boot.workflow.property.test.ts +++ b/packages/effect-daemon-spec/src/__tests__/supervisor-boot.workflow.property.test.ts @@ -75,16 +75,27 @@ const bootCommandsOf = ( const startIdOf = (start: StartChild): string => start.childId +const awaitingIdOf = (start: { readonly childId: string }): string => start.childId + it.prop( - '∀b_BootStart_=DeclaredOrder', + '∀b_BootStart_=FirstAndQueueRest', { of: [Schema.Array(ChildDeclaration), EventTime], subject: interpretSupervisionEvent }, (subject, [drawn, at]) => { const ids = idsOf(drawn) - const commands = bootCommandsOf(subject, policyOf('one_for_one', declarationsOf(drawn)), at) - return commands.stops.length === 0 && - commands.replies.length === 0 && - commands.terminates.length === 0 && - Arr.join(Arr.map(commands.starts, startIdOf), '|') === Arr.join(ids, '|') + if (Arr.dedupe(ids).length !== ids.length) return true + const decision = bootsTo(subject, policyOf('one_for_one', declarationsOf(drawn)), at) + return Match.value(decision).pipe( + Match.tag( + 'StartChildren', + (started) => + Arr.join(Arr.map(started.commands.starts, startIdOf), '|') === Arr.join(Arr.take(ids, 1), '|') && + Arr.join(Arr.map(started.core.awaiting, awaitingIdOf), '|') === Arr.join(Arr.drop(ids, 1), '|') && + started.commands.stops.length === 0 && + started.commands.replies.length === 0 && + started.commands.terminates.length === 0, + ), + Match.orElse(() => false), + ) }, ) @@ -101,7 +112,7 @@ it.prop( Match.tag('ArmSupervisorTimer', () => false), Match.exhaustive, ) - return commands.arms.length === ids.length && Arr.every(commands.arms, deadlineHolds) + return commands.arms.length === Arr.length(Arr.take(ids, 1)) && Arr.every(commands.arms, deadlineHolds) }, ) diff --git a/packages/effect-daemon-spec/src/kernel/SupervisionEvent.schema.ts b/packages/effect-daemon-spec/src/kernel/SupervisionEvent.schema.ts index fbc90d3..7fd1c36 100644 --- a/packages/effect-daemon-spec/src/kernel/SupervisionEvent.schema.ts +++ b/packages/effect-daemon-spec/src/kernel/SupervisionEvent.schema.ts @@ -1,11 +1,15 @@ import { Schema } from 'effect' -import { ChildId, EventTime, Generation, RequestId } from './SupervisionLimits.schema.js' +import { ChildId, Epoch, EventTime, Generation, RequestId } from './SupervisionLimits.schema.js' import { TerminationReason } from './TerminationReport.schema.js' export const TimerKind = Schema.Literals(['start_deadline', 'backoff', 'liveness_tick', 'cool_down']) export type TimerKind = typeof TimerKind.Type -export const ChildTimerTarget = Schema.TaggedStruct('ChildTimer', { childId: ChildId, generation: Generation }) +export const ChildTimerTarget = Schema.TaggedStruct('ChildTimer', { + childId: ChildId, + generation: Generation, + epoch: Epoch, +}) export type ChildTimerTarget = typeof ChildTimerTarget.Type export const SupervisorTimerTarget = Schema.TaggedStruct('SupervisorTimer', {}) @@ -38,6 +42,27 @@ export const ChildReady = Schema.TaggedStruct('ChildReady', { }) export type ChildReady = typeof ChildReady.Type +export const ChildStandby = Schema.TaggedStruct('ChildStandby', { + at: EventTime, + childId: ChildId, + generation: Generation, +}) +export type ChildStandby = typeof ChildStandby.Type + +export const ChildAcquired = Schema.TaggedStruct('ChildAcquired', { + at: EventTime, + childId: ChildId, + generation: Generation, +}) +export type ChildAcquired = typeof ChildAcquired.Type + +export const ChildReleased = Schema.TaggedStruct('ChildReleased', { + at: EventTime, + childId: ChildId, + generation: Generation, +}) +export type ChildReleased = typeof ChildReleased.Type + export const ChildTerminated = Schema.TaggedStruct('ChildTerminated', { at: EventTime, childId: ChildId, @@ -92,6 +117,9 @@ export const SupervisionEvent = Schema.Union([ SupervisorStarted, ChildStarted, ChildReady, + ChildStandby, + ChildAcquired, + ChildReleased, ChildTerminated, ChildStopped, ProbeResult, diff --git a/packages/effect-daemon-spec/src/kernel/SupervisionLimits.schema.ts b/packages/effect-daemon-spec/src/kernel/SupervisionLimits.schema.ts index c0f7e80..6a4e809 100644 --- a/packages/effect-daemon-spec/src/kernel/SupervisionLimits.schema.ts +++ b/packages/effect-daemon-spec/src/kernel/SupervisionLimits.schema.ts @@ -40,6 +40,9 @@ export type Deadline = typeof Deadline.Type export const Generation = NonNegativeSafeInt.pipe(Schema.check(generatedBetween(0, 1_024))) export type Generation = typeof Generation.Type +export const Epoch = NonNegativeSafeInt.pipe(Schema.check(generatedBetween(0, 1_024))) +export type Epoch = typeof Epoch.Type + /** OTP intensity (R3): any non-negative integer restarts per period. Generation stays small because laws fold intensity+1 restarts. */ export const Intensity = NonNegativeSafeInt.pipe(Schema.check(generatedBetween(0, 8))) export type Intensity = typeof Intensity.Type diff --git a/packages/effect-daemon-spec/src/kernel/SupervisorCommand.schema.ts b/packages/effect-daemon-spec/src/kernel/SupervisorCommand.schema.ts index fa1609c..fcafbca 100644 --- a/packages/effect-daemon-spec/src/kernel/SupervisorCommand.schema.ts +++ b/packages/effect-daemon-spec/src/kernel/SupervisorCommand.schema.ts @@ -1,6 +1,6 @@ import { Schema } from 'effect' import { TimerKind } from './SupervisionEvent.schema.js' -import { ChildId, Deadline, Generation, RequestId } from './SupervisionLimits.schema.js' +import { ChildId, Deadline, Epoch, Generation, RequestId } from './SupervisionLimits.schema.js' import { ShutdownMode } from './SupervisorPolicy.schema.js' import { SupervisorExit, TerminationReason } from './TerminationReport.schema.js' @@ -18,6 +18,7 @@ export const ArmChildTimer = Schema.TaggedStruct('ArmChildTimer', { kind: TimerKind, childId: ChildId, generation: Generation, + epoch: Epoch, deadline: Deadline, }) export type ArmChildTimer = typeof ArmChildTimer.Type diff --git a/packages/effect-daemon-spec/src/kernel/SupervisorState.schema.ts b/packages/effect-daemon-spec/src/kernel/SupervisorState.schema.ts index a3ae498..dfdc3dd 100644 --- a/packages/effect-daemon-spec/src/kernel/SupervisorState.schema.ts +++ b/packages/effect-daemon-spec/src/kernel/SupervisorState.schema.ts @@ -3,6 +3,7 @@ import { dual } from 'effect/Function' import type { SupervisionDecision } from './interpret-supervision-event.workflow.js' import { ChildId, + Epoch, EventTime, Generation, Ordinal, @@ -15,29 +16,31 @@ import { SupervisorExit, TerminationReason } from './TerminationReport.schema.js const StateTypeId: unique symbol = Symbol.for('@systemfsoftware/effect-daemon/SupervisorState') -export const ChildStatus = Schema.Literals(['starting', 'ready', 'stopping']) +export const ChildStatus = Schema.Literals(['standby', 'starting', 'ready', 'stopping']) export type ChildStatus = typeof ChildStatus.Type export const ChildInstance = Schema.Struct({ childId: ChildId, generation: Generation, status: ChildStatus, + epoch: Epoch, consecutiveRestarts: RestartCount, probeFailures: ProbeFailures, }) export type ChildInstance = typeof ChildInstance.Type +export const ChildStart = Schema.Struct({ childId: ChildId, generation: Generation }) +export type ChildStart = typeof ChildStart.Type + export const SupervisorCore = Schema.Struct({ policy: SupervisionPolicy, children: Schema.Array(ChildInstance), restartStamps: Schema.Array(EventTime), nextOrdinal: Ordinal, + awaiting: Schema.Array(ChildStart), }) export type SupervisorCore = typeof SupervisorCore.Type -export const ChildStart = Schema.Struct({ childId: ChildId, generation: Generation }) -export type ChildStart = typeof ChildStart.Type - export class Running extends Schema.TaggedClass()('Running', { core: SupervisorCore }) { @@ -83,11 +86,13 @@ export const initialStateOf = (policy: SupervisionPolicy): SupervisorCore => ({ childId: declaration.childId, generation: 0, status: 'starting', + epoch: 0, consecutiveRestarts: 0, probeFailures: 0, })), restartStamps: [], nextOrdinal: 0, + awaiting: [], }) const withCore = (state: SupervisorState, core: SupervisorCore): SupervisorState => diff --git a/packages/effect-daemon-spec/src/kernel/interpret-supervision-event.workflow.ts b/packages/effect-daemon-spec/src/kernel/interpret-supervision-event.workflow.ts index 9db39d1..c2a0af3 100644 --- a/packages/effect-daemon-spec/src/kernel/interpret-supervision-event.workflow.ts +++ b/packages/effect-daemon-spec/src/kernel/interpret-supervision-event.workflow.ts @@ -3,7 +3,7 @@ import { Array as Arr, Cause, Match, Number as Num, Option, Result, Schema } fro import { SupervisionEvent } from './SupervisionEvent.schema.js' import type { TimerKind, TimerTarget } from './SupervisionEvent.schema.js' import { Millis, PositiveMillis } from './SupervisionLimits.schema.js' -import type { EventTime, Generation, RequestId } from './SupervisionLimits.schema.js' +import type { Epoch, EventTime, Generation, RequestId } from './SupervisionLimits.schema.js' import { ChildId } from './SupervisionLimits.schema.js' import { StopChild, SupervisorCommands } from './SupervisorCommand.schema.js' import type { @@ -131,6 +131,64 @@ type CommandBuckets = { const commandsIn = (buckets: CommandBuckets): SupervisorCommands => ({ ...noCommands, ...buckets }) +const appendCommands = (first: SupervisorCommands, second: SupervisorCommands): SupervisorCommands => ({ + stops: [...first.stops, ...second.stops], + starts: [...first.starts, ...second.starts], + arms: [...first.arms, ...second.arms], + replies: [...first.replies, ...second.replies], + terminates: [...first.terminates, ...second.terminates], +}) + +const startTimeoutOf = (core: SupervisorCore, start: ChildStart): PositiveMillis => + kindOfPolicy(core.policy, start.childId).startTimeoutMillis + +const startIsLive = (core: SupervisorCore, start: ChildStart): boolean => + Option.isSome(currentIncarnation(core, start.childId, start.generation)) + +const predecessorsReady = (core: SupervisorCore, start: ChildStart): boolean => + Option.match(indexOfChild(core, start.childId), { + onNone: () => false, + onSome: (index) => Arr.every(Arr.take(core.children, index), (child) => child.status === 'ready'), + }) + +const startIssues = (core: SupervisorCore, start: ChildStart): boolean => + [startIsLive(core, start), predecessorsReady(core, start)].every((holds) => holds) + +const enqueueStarts = ( + awaiting: ReadonlyArray, + starts: ReadonlyArray, +): ReadonlyArray => [ + ...Arr.filter(awaiting, (queued) => Arr.every(starts, (start) => start.childId !== queued.childId)), + ...starts, +] + +const drainAwaitingOf = ( + core: SupervisorCore, + at: EventTime, +): { readonly core: SupervisorCore; readonly commands: SupervisorCommands } => { + const issuable = Arr.filter(core.awaiting, (start) => startIssues(core, start)) + const remaining = Arr.filter( + core.awaiting, + (start) => [Arr.contains(issuable, start) === false, startIsLive(core, start)].every((holds) => holds), + ) + return { + core: { ...core, awaiting: remaining }, + commands: commandsIn({ + starts: Arr.map(issuable, startCommandOf), + arms: Arr.map(issuable, (start) => deadlineCommandOf(start, at + startTimeoutOf(core, start), 0)), + }), + } +} + +const drainedDecisionOf = ( + core: SupervisorCore, + at: EventTime, + commands: SupervisorCommands, +): SupervisionDecision => { + const drained = drainAwaitingOf(core, at) + return new Continue({ core: drained.core, commands: appendCommands(commands, drained.commands) }) +} + const restartsOn = (restartType: RestartType, reason: TerminationReason): boolean => Match.value(reason._tag).pipe( Match.when('Abnormal', () => restartType !== 'temporary'), @@ -224,6 +282,7 @@ const restartingOf = (child: ChildInstance, generation: Generation): ChildInstan childId: child.childId, generation, status: 'starting', + epoch: 0, consecutiveRestarts: child.consecutiveRestarts + 1, probeFailures: 0, }) @@ -231,10 +290,29 @@ const readiedOf = (child: ChildInstance): ChildInstance => ({ childId: child.childId, generation: child.generation, status: 'ready', + epoch: child.epoch, consecutiveRestarts: 0, probeFailures: 0, }) +const acquiredOf = (child: ChildInstance): ChildInstance => ({ + childId: child.childId, + generation: child.generation, + status: 'starting', + epoch: child.epoch + 1, + consecutiveRestarts: child.consecutiveRestarts, + probeFailures: 0, +}) + +const standbyOf = (child: ChildInstance): ChildInstance => ({ + childId: child.childId, + generation: child.generation, + status: 'standby', + epoch: child.epoch, + consecutiveRestarts: child.consecutiveRestarts, + probeFailures: child.probeFailures, +}) + const probedAs = (child: ChildInstance, alive: boolean): ChildInstance => Match.value(alive).pipe( Match.when(true, () => ({ ...child, probeFailures: 0 })), @@ -290,14 +368,16 @@ const backoffCommandOf = (start: ChildStart, deadline: EventTime): ArmChildTimer kind: 'backoff', childId: start.childId, generation: start.generation, + epoch: 0, deadline, }) -const deadlineCommandOf = (start: ChildStart, deadline: EventTime): ArmChildTimer => ({ +const deadlineCommandOf = (start: ChildStart, deadline: EventTime, epoch: Epoch): ArmChildTimer => ({ _tag: 'ArmChildTimer', kind: 'start_deadline', childId: start.childId, generation: start.generation, + epoch, deadline, }) @@ -306,6 +386,7 @@ const tickCommandOf = (child: ChildInstance, deadline: EventTime): ArmChildTimer kind: 'liveness_tick', childId: child.childId, generation: child.generation, + epoch: child.epoch, deadline, }) @@ -461,24 +542,21 @@ const phasedCommands = ( core: SupervisorCore, at: EventTime, coming: ReadonlyArray, -): { readonly commands: SupervisorCommands; readonly pending: ReadonlyArray } => { +): { + readonly issuable: ReadonlyArray + readonly pending: ReadonlyArray + readonly arms: ReadonlyArray +} => { const steps: ReadonlyArray = Arr.map(coming, (child) => { const start: ChildStart = { childId: child.childId, generation: child.generation + 1 } return { start, delay: backoffDelay(core.policy.backoff, child.consecutiveRestarts) } }) const ready = Arr.filter(steps, (step: RestartStep) => step.delay === 0) const deferred = Arr.filter(steps, (step: RestartStep) => step.delay > 0) - const timeoutOf = (step: RestartStep): EventTime => - at + kindOfPolicy(core.policy, step.start.childId).startTimeoutMillis return { - commands: commandsIn({ - starts: Arr.map(ready, (step: RestartStep) => startCommandOf(step.start)), - arms: Arr.appendAll( - Arr.map(ready, (step: RestartStep) => deadlineCommandOf(step.start, timeoutOf(step))), - Arr.map(deferred, (step: RestartStep) => backoffCommandOf(step.start, at + step.delay)), - ), - }), + issuable: Arr.map(ready, (step: RestartStep) => step.start), pending: Arr.map(deferred, (step: RestartStep) => step.start), + arms: Arr.map(deferred, (step: RestartStep) => backoffCommandOf(step.start, at + step.delay)), } } @@ -517,13 +595,36 @@ type RestartOutcome = { const restartOutcomeOf = (core: SupervisorCore, scope: RestartScope, at: EventTime): RestartOutcome => { const nextCore = stampedCoreOf(core, scope, at) const phased = phasedCommands(core, at, scope.coming) - return { - core: nextCore, - stops: scope.stops, - pending: phased.pending, - commands: commandsIn({ stops: scope.stops, starts: phased.commands.starts, arms: phased.commands.arms }), - exhausted: Arr.length(nextCore.restartStamps) > nextCore.policy.intensity, - } + return Match.value(core.policy.strategy).pipe( + Match.when('one_for_one', () => ({ + core: { ...nextCore, awaiting: Arr.filter(nextCore.awaiting, (start) => startIsLive(nextCore, start)) }, + stops: scope.stops, + pending: phased.pending, + commands: commandsIn({ + stops: scope.stops, + arms: [ + ...phased.arms, + ...Arr.map(phased.issuable, (start) => deadlineCommandOf(start, at + startTimeoutOf(core, start), 0)), + ], + starts: Arr.map(phased.issuable, startCommandOf), + }), + exhausted: Arr.length(nextCore.restartStamps) > nextCore.policy.intensity, + })), + Match.orElse((): RestartOutcome => { + const queued: SupervisorCore = { + ...nextCore, + awaiting: enqueueStarts(nextCore.awaiting, phased.issuable), + } + const drained = drainAwaitingOf(queued, at) + return { + core: drained.core, + stops: scope.stops, + pending: phased.pending, + commands: appendCommands(commandsIn({ stops: scope.stops, arms: phased.arms }), drained.commands), + exhausted: Arr.length(nextCore.restartStamps) > nextCore.policy.intensity, + } + }), + ) } const restartedDecisionOf = ( @@ -568,10 +669,11 @@ const shutdownOrContinue = ( core: SupervisorCore, removed: ReadonlyArray, remaining: ReadonlyArray, + at: EventTime, ): SupervisionDecision => Match.value(autoShutsDown(core.policy, removed, remaining)).pipe( Match.when(true, () => stopEverything(withChildren(core, remaining), terminatedReason, requestedExit)), - Match.when(false, () => new Continue({ core: withChildren(core, remaining), commands: noCommands })), + Match.when(false, () => drainedDecisionOf(withChildren(core, remaining), at, noCommands)), Match.exhaustive, ) const RestartedBase = Schema.TaggedStruct('Restarted', { @@ -638,6 +740,7 @@ const terminatedIndexed = (core: SupervisorCore, failedIndex: number, terminatio core, Arr.filter(core.children, (child) => Arr.contains(quiet.removedIds, child.childId)), quiet.remaining, + termination.at, )), Match.exhaustive, ) @@ -660,6 +763,7 @@ const terminatedCurrent = ( ): SupervisionDecision => Match.value(child.status).pipe( Match.when('starting', () => terminatedIfIndexed(core, child, termination)), + Match.when('standby', () => terminatedIfIndexed(core, child, termination)), Match.when('ready', () => terminatedIfIndexed(core, child, termination)), Match.when('stopping', () => removalContinue(core, termination)), Match.exhaustive, @@ -669,7 +773,7 @@ const removalOf = (core: SupervisorCore, childId: ChildId, generation: Generatio withChildren(core, Arr.filter(core.children, (child) => incarnationMatches(childId, generation)(child) === false)) const removalContinue = (core: SupervisorCore, termination: Termination): SupervisionDecision => - new Continue({ core: removalOf(core, termination.childId, termination.generation), commands: noCommands }) + drainedDecisionOf(removalOf(core, termination.childId, termination.generation), termination.at, noCommands) const terminatedRunning = (core: SupervisorCore, event: { readonly childId: ChildId @@ -709,19 +813,23 @@ const readyInRunning = ( const becomingReady = (core: SupervisorCore, child: ChildInstance, at: EventTime): SupervisionDecision => Match.value(child.status).pipe( - Match.when('starting', () => { - const next = readiedOf(child) - const nextCore = withChildren(core, Arr.map(core.children, replacing(next))) - return new Continue({ - core: nextCore, - commands: commandsIn({ arms: [tickCommandOf(next, at + core.policy.livenessTickMillis)] }), - }) - }), + Match.when('starting', () => readiedDecisionOf(core, child, at)), + Match.when('standby', () => holdOf(core)), Match.when('ready', () => holdOf(core)), Match.when('stopping', () => holdOf(core)), Match.exhaustive, ) +const readiedDecisionOf = (core: SupervisorCore, child: ChildInstance, at: EventTime): SupervisionDecision => { + const next = readiedOf(child) + const nextCore = withChildren(core, Arr.map(core.children, replacing(next))) + return drainedDecisionOf( + nextCore, + at, + commandsIn({ arms: [tickCommandOf(next, at + core.policy.livenessTickMillis)] }), + ) +} + const stoppedInShuttingDown = ( core: SupervisorCore, reason: TerminationReason, @@ -759,6 +867,7 @@ const probedOutcomeOf = ( Match.value(child.status).pipe( Match.when('ready', () => probedReadyOutcomeOf(core, child, alive, at)), Match.when('starting', () => holdOf(core)), + Match.when('standby', () => holdOf(core)), Match.when('stopping', () => holdOf(core)), Match.exhaustive, ) @@ -801,15 +910,23 @@ const failedProbeOutcomeOf = (core: SupervisorCore, child: ChildInstance, at: Ev Match.exhaustive, ) +const epochedDecision = (child: ChildInstance, epoch: Epoch, decision: SupervisionDecision): SupervisionDecision => + Match.value(child.epoch === epoch).pipe( + Match.when(true, () => decision), + Match.when(false, () => new Stale({})), + Match.exhaustive, + ) + const namedTimerInRunning = ( core: SupervisorCore, child: ChildInstance, kind: TimerKind, at: EventTime, + epoch: Epoch, ): SupervisionDecision => Match.value(kind).pipe( - Match.when('start_deadline', () => deadlineInRunning(core, child, at)), - Match.when('liveness_tick', () => tickInRunning(core, child, at)), + Match.when('start_deadline', () => epochedDecision(child, epoch, deadlineInRunning(core, child, at))), + Match.when('liveness_tick', () => epochedDecision(child, epoch, tickInRunning(core, child, at))), Match.when('backoff', () => new Stale({})), Match.when('cool_down', () => new Stale({})), Match.exhaustive, @@ -828,6 +945,7 @@ const deadlineInRunning = (core: SupervisorCore, child: ChildInstance, at: Event at, })), Match.when('ready', () => new Stale({})), + Match.when('standby', () => new Stale({})), Match.when('stopping', () => new Stale({})), Match.exhaustive, ) @@ -843,6 +961,7 @@ const tickInRunning = (core: SupervisorCore, child: ChildInstance, at: EventTime }), ), Match.when('starting', () => new Stale({})), + Match.when('standby', () => new Stale({})), Match.when('stopping', () => new Stale({})), Match.exhaustive, ) @@ -853,10 +972,11 @@ const timerInRunning = ( generation: Generation, kind: TimerKind, at: EventTime, + epoch: Epoch, ): SupervisionDecision => Match.value(currentIncarnation(core, childId, generation)).pipe( Match.tag('None', () => new Stale({})), - Match.tag('Some', (found) => namedTimerInRunning(core, found.value, kind, at)), + Match.tag('Some', (found) => namedTimerInRunning(core, found.value, kind, at, epoch)), Match.exhaustive, ) @@ -869,14 +989,30 @@ const fireBackoffOf = ( ): SupervisionDecision => { const start: ChildStart = { childId, generation } const remaining = Arr.filter(pending, (plan) => planMatches(childId, generation)(plan) === false) - const commands: SupervisorCommands = commandsIn({ - starts: [startCommandOf(start)], - arms: [deadlineCommandOf(start, at + kindOfPolicy(core.policy, childId).startTimeoutMillis)], - }) - return Match.value(Arr.length(remaining) === 0).pipe( - Match.when(true, () => new StartChildren({ core, commands })), - Match.when(false, () => new RestartChildren({ core, pending: remaining, commands })), - Match.exhaustive, + return Match.value(core.policy.strategy).pipe( + Match.when('one_for_one', () => { + const commands: SupervisorCommands = commandsIn({ + starts: [startCommandOf(start)], + arms: [deadlineCommandOf(start, at + startTimeoutOf(core, start), 0)], + }) + return Match.value(Arr.length(remaining) === 0).pipe( + Match.when(true, () => new StartChildren({ core, commands })), + Match.when(false, () => new RestartChildren({ core, pending: remaining, commands })), + Match.exhaustive, + ) + }), + Match.orElse(() => { + const queued: SupervisorCore = { ...core, awaiting: enqueueStarts(core.awaiting, [start]) } + const drained = drainAwaitingOf(queued, at) + return Match.value(Arr.length(remaining) === 0).pipe( + Match.when(true, () => new StartChildren({ core: drained.core, commands: drained.commands })), + Match.when( + false, + () => new RestartChildren({ core: drained.core, pending: remaining, commands: drained.commands }), + ), + Match.exhaustive, + ) + }), ) } @@ -901,35 +1037,26 @@ const backoffInRestarting = ( Match.exhaustive, ) -const cooledStartsOf = (core: SupervisorCore, at: EventTime): { - readonly fresh: ReadonlyArray - readonly commands: SupervisorCommands -} => { +const cooledRestartOf = (core: SupervisorCore, at: EventTime): SupervisionDecision => { const fresh: ReadonlyArray = Arr.map(core.children, (child) => ({ ...child, status: 'starting', + epoch: 0, consecutiveRestarts: 0, probeFailures: 0, })) - const deadlineOf = (child: ChildInstance): ArmChildTimer => - deadlineCommandOf( - { childId: child.childId, generation: child.generation }, - at + kindOfPolicy(core.policy, child.childId).startTimeoutMillis, - ) - return { - fresh, - commands: commandsIn({ - starts: Arr.map(fresh, (child) => startCommandOf({ childId: child.childId, generation: child.generation })), - arms: Arr.map(fresh, deadlineOf), - }), - } -} - -const cooledRestartOf = (core: SupervisorCore, at: EventTime): SupervisionDecision => { - const starts = cooledStartsOf(core, at) const clearedStamps: ReadonlyArray = [] - const restarted: SupervisorCore = { ...core, children: starts.fresh, restartStamps: clearedStamps } - return new StartChildren({ core: restarted, commands: starts.commands }) + const queued: SupervisorCore = { + ...core, + children: fresh, + restartStamps: clearedStamps, + awaiting: enqueueStarts( + core.awaiting, + Arr.map(fresh, (child) => ({ childId: child.childId, generation: child.generation })), + ), + } + const drained = drainAwaitingOf(queued, at) + return new StartChildren({ core: drained.core, commands: drained.commands }) } const runningDynamicCount = (core: SupervisorCore): number => @@ -946,6 +1073,7 @@ const dynamicInstanceOf = (core: SupervisorCore): { readonly core: SupervisorCor childId: start.childId, generation: start.generation, status: 'starting', + epoch: 0, consecutiveRestarts: 0, probeFailures: 0, } @@ -961,18 +1089,14 @@ const allocatedDecisionOf = ( requestId: RequestId, at: EventTime, ): SupervisionDecision => { - const timeoutMillis = Match.value(dynamicKindOf(allocated.core.policy)).pipe( - Match.tag('Some', ({ value }) => value.startTimeoutMillis), - Match.tag('None', () => UNKNOWN_TIMEOUT_MILLIS), - Match.exhaustive, - ) + const queued: SupervisorCore = { + ...allocated.core, + awaiting: enqueueStarts(allocated.core.awaiting, [allocated.start]), + } + const drained = drainAwaitingOf(queued, at) return new StartChildren({ - core: allocated.core, - commands: commandsIn({ - starts: [startCommandOf(allocated.start)], - arms: [deadlineCommandOf(allocated.start, at + timeoutMillis)], - replies: [replyAcceptedOf(requestId, allocated.start)], - }), + core: drained.core, + commands: appendCommands(drained.commands, commandsIn({ replies: [replyAcceptedOf(requestId, allocated.start)] })), }) } @@ -1119,7 +1243,10 @@ const onTimerElapsed = ( Match.value(state).pipe( Match.tag('Running', ({ core }) => Match.value(target).pipe( - Match.tag('ChildTimer', (child) => timerInRunning(core, child.childId, child.generation, kind, at)), + Match.tag( + 'ChildTimer', + (child) => timerInRunning(core, child.childId, child.generation, kind, at, child.epoch), + ), Match.tag('SupervisorTimer', () => new Stale({})), Match.exhaustive, )), @@ -1192,22 +1319,15 @@ const onShutdownRequested = (state: SupervisorState, reason: TerminationReason): ) const bootedChildrenOf = (core: SupervisorCore, at: EventTime): SupervisionDecision => { - const incarnationOf = (child: ChildInstance): ChildStart => ({ - childId: child.childId, - generation: child.generation, - }) - const deadlineOf = (child: ChildInstance): ArmChildTimer => - deadlineCommandOf( - incarnationOf(child), - at + kindOfPolicy(core.policy, child.childId).startTimeoutMillis, - ) - return new StartChildren({ - core, - commands: commandsIn({ - starts: Arr.map(core.children, (child) => startCommandOf(incarnationOf(child))), - arms: Arr.map(core.children, deadlineOf), - }), - }) + const queued: SupervisorCore = { + ...core, + awaiting: enqueueStarts( + core.awaiting, + Arr.map(core.children, (child) => ({ childId: child.childId, generation: child.generation })), + ), + } + const drained = drainAwaitingOf(queued, at) + return new StartChildren({ core: drained.core, commands: drained.commands }) } const onSupervisorStarted = (state: SupervisorState, at: EventTime): SupervisionDecision => @@ -1220,11 +1340,121 @@ const onSupervisorStarted = (state: SupervisorState, at: EventTime): Supervision Match.exhaustive, ) +const standbyOfChild = (core: SupervisorCore, child: ChildInstance): SupervisionDecision => + Match.value(child.status).pipe( + Match.when('starting', () => + new Continue({ + core: withChildren(core, Arr.map(core.children, replacing(standbyOf(child)))), + commands: noCommands, + })), + Match.when('standby', () => holdOf(core)), + Match.when('ready', () => holdOf(core)), + Match.when('stopping', () => holdOf(core)), + Match.exhaustive, + ) + +const standbyInRunning = (core: SupervisorCore, childId: ChildId, generation: Generation): SupervisionDecision => + Match.value(currentIncarnation(core, childId, generation)).pipe( + Match.tag('None', () => new Stale({})), + Match.tag('Some', ({ value }) => standbyOfChild(core, value)), + Match.exhaustive, + ) + +const acquiredOfChild = (core: SupervisorCore, child: ChildInstance, at: EventTime): SupervisionDecision => + Match.value(child.status).pipe( + Match.when('standby', () => + new Continue({ + core: withChildren(core, Arr.map(core.children, replacing(acquiredOf(child)))), + commands: commandsIn({ + arms: [ + deadlineCommandOf( + { childId: child.childId, generation: child.generation }, + at + startTimeoutOf(core, { childId: child.childId, generation: child.generation }), + child.epoch + 1, + ), + ], + }), + })), + Match.when('starting', () => holdOf(core)), + Match.when('ready', () => holdOf(core)), + Match.when('stopping', () => holdOf(core)), + Match.exhaustive, + ) + +const acquiredInRunning = ( + core: SupervisorCore, + childId: ChildId, + generation: Generation, + at: EventTime, +): SupervisionDecision => + Match.value(currentIncarnation(core, childId, generation)).pipe( + Match.tag('None', () => new Stale({})), + Match.tag('Some', ({ value }) => acquiredOfChild(core, value, at)), + Match.exhaustive, + ) + +const releasedDecisionOf = (core: SupervisorCore, child: ChildInstance): SupervisionDecision => + new Continue({ + core: withChildren(core, Arr.map(core.children, replacing(standbyOf(child)))), + commands: noCommands, + }) + +const releasedOfChild = (core: SupervisorCore, child: ChildInstance): SupervisionDecision => + Match.value(child.status).pipe( + Match.when('starting', () => releasedDecisionOf(core, child)), + Match.when('ready', () => releasedDecisionOf(core, child)), + Match.when('standby', () => holdOf(core)), + Match.when('stopping', () => holdOf(core)), + Match.exhaustive, + ) + +const releasedInRunning = (core: SupervisorCore, childId: ChildId, generation: Generation): SupervisionDecision => + Match.value(currentIncarnation(core, childId, generation)).pipe( + Match.tag('None', () => new Stale({})), + Match.tag('Some', ({ value }) => releasedOfChild(core, value)), + Match.exhaustive, + ) + +const coreOfLive = (state: SupervisorState): Option.Option => + Match.value(state).pipe( + Match.tag('Terminated', () => Option.none()), + Match.orElse((live) => Option.some(live.core)), + ) + +const onChildStandby = (state: SupervisorState, childId: ChildId, generation: Generation): SupervisionDecision => + Option.match(coreOfLive(state), { + onNone: () => new Stale({}), + onSome: (core) => standbyInRunning(core, childId, generation), + }) + +const onChildAcquired = ( + state: SupervisorState, + childId: ChildId, + generation: Generation, + at: EventTime, +): SupervisionDecision => + Option.match(coreOfLive(state), { + onNone: () => new Stale({}), + onSome: (core) => acquiredInRunning(core, childId, generation, at), + }) + +const onChildReleased = (state: SupervisorState, childId: ChildId, generation: Generation): SupervisionDecision => + Option.match(coreOfLive(state), { + onNone: () => new Stale({}), + onSome: (core) => releasedInRunning(core, childId, generation), + }) + const decided = (state: SupervisorState, event: SupervisionEvent): SupervisionDecision => Match.value(event).pipe( Match.tag('SupervisorStarted', (started) => onSupervisorStarted(state, started.at)), Match.tag('ChildStarted', (started) => onChildStarted(state, started.childId, started.generation)), Match.tag('ChildReady', (ready) => onChildReady(state, ready.childId, ready.generation, ready.at)), + Match.tag('ChildStandby', (standby) => onChildStandby(state, standby.childId, standby.generation)), + Match.tag( + 'ChildAcquired', + (acquired) => onChildAcquired(state, acquired.childId, acquired.generation, acquired.at), + ), + Match.tag('ChildReleased', (released) => onChildReleased(state, released.childId, released.generation)), Match.tag('ChildTerminated', (terminated) => onChildTerminated( state, diff --git a/packages/effect-daemon-spec/tests/supervisor-stale-reports.integration.test.ts b/packages/effect-daemon-spec/tests/supervisor-stale-reports.integration.test.ts index ac4b9a5..de95edf 100644 --- a/packages/effect-daemon-spec/tests/supervisor-stale-reports.integration.test.ts +++ b/packages/effect-daemon-spec/tests/supervisor-stale-reports.integration.test.ts @@ -2,7 +2,7 @@ import { Supervisor } from '@systemfsoftware/effect-daemon-spec' import { Gherkin, Given, it, makeFeature, Then, When } from '@systemfsoftware/effect-gherkin-spec' import { Array as Arr, Effect, Match, Queue } from 'effect' import { fiberMediumLayer } from './__fixtures__/FiberMediumHarness.js' -import { crashingChild, settled, traceUntil } from './__fixtures__/SupervisorHarness.js' +import { crashingChild, neverChild, settled, traceUntil } from './__fixtures__/SupervisorHarness.js' const Feature = makeFeature({ it }) @@ -39,37 +39,36 @@ const terminationReportDecisions = (childId: string) => (trace: Trace) => Arr.map(Arr.filter(trace, reportsTerminationOf(childId)), (entry) => entry.decision) const oneForAllPair = Effect.gen(function*() { - const firstCrash = yield* Queue.unbounded() - const secondCrash = yield* Queue.unbounded() + const crash = yield* Queue.unbounded() const supervisor = yield* Supervisor.make('one-for-all-pair').pipe( Supervisor.strategy('one_for_all'), Supervisor.children([ - Supervisor.ChildSpecs.make('a', crashingChild(firstCrash)), - Supervisor.ChildSpecs.make('b', crashingChild(secondCrash)), + Supervisor.ChildSpecs.make('a', neverChild), + Supervisor.ChildSpecs.make('b', crashingChild(crash)), ]), ).scoped - return { firstCrash, supervisor } + return { crash, supervisor } }) Feature('Reports from a child the supervisor has already replaced') .withLayer(fiberMediumLayer) .body(({ scenario }) => { scenario( - "A one-for-all restart leaves the replaced incarnation's report without effect", + 'A one-for-all restart leaves the replaced first incarnation’s report without effect', Gherkin.Do.pipe( - Given('a one-for-all supervisor running two children, each of which crashes when signalled')( + Given('a one-for-all supervisor whose first child runs on and whose second crashes when signalled')( 'tree', () => oneForAllPair, ), - When('the first child crashes')('after', ({ tree }) => + When('the later child crashes')('after', ({ tree }) => Effect.gen(function*() { const watching = yield* traceUntil( tree.supervisor, (trace) => - terminationReportSeen('b')(trace) && + terminationReportSeen('a')(trace) && Arr.every(['a', 'b'], (childId) => readyAtGeneration(1)(childId)(trace)), ) - yield* Queue.offer(tree.firstCrash, void 0) + yield* Queue.offer(tree.crash, void 0) const trace = yield* settled(watching) yield* Effect.yieldNow const incarnations = yield* runningIncarnations(tree.supervisor) @@ -78,7 +77,7 @@ Feature('Reports from a child the supervisor has already replaced') Then("the replaced incarnation's report has no effect and both children run in their new incarnation")( ({ after }, expect) => expect({ - decisions: terminationReportDecisions('b')(after.trace), + decisions: terminationReportDecisions('a')(after.trace), incarnations: after.incarnations, }).toEqual({ decisions: [{ _tag: 'Stale' }],