From 8a2dd515a483f490c46df923c9f5032d81064185 Mon Sep 17 00:00:00 2001 From: systemfsoftware-bot <318394648+systemfsoftware-bot@users.noreply.github.com> Date: Thu, 19 Nov 2026 12:44:28 -0500 Subject: [PATCH 1/3] feat(effect-daemon-cluster): standby singletons report ready only after shard acquisition A cluster singleton whose shard another runner owns used to be announced ready at registration, so rest-for-one dependents started against an uninitialized singleton and startTimeoutMillis could not bound the real post-acquisition initialization. The medium now reports standby truthfully: a standby singleton is not ready, is not bound by the start deadline, and children declared after it do not start until it owns its shard and its run signals ready. The start deadline is armed when the shard arrives, and losing the shard is a handoff back to standby rather than a termination: nothing is reported, restarted, or charged against the restart intensity. The supervisor starts children in declared order, each gated on the readiness of the ones before it, matching the documented OTP behaviour. Status and the trace expose standby, acquired, ready and released transitions; media that cannot standby are unchanged. --- .changeset/standby-singleton-readiness.md | 7 + packages/effect-daemon-cluster/README.md | 9 +- .../src/ClusterMedium/medium.ts | 74 ++++- .../tests/cluster-medium.conformance.test.ts | 22 +- ...ster-standby-lifecycle.integration.test.ts | 250 ++++++++++++++ .../src/Trace.schema.ts | 3 + .../src/observed-trace.ts | 15 +- packages/effect-daemon-spec/README.md | 2 + .../src/Supervisor/Medium.ts | 17 +- .../src/Supervisor/MediumActivity.schema.ts | 13 + .../src/Supervisor/supervisor-commands.ts | 76 ++++- ...upervision-event.workflow.property.test.ts | 19 +- ...ndby-acquisition.workflow.property.test.ts | 244 ++++++++++++++ .../supervisor-boot.workflow.property.test.ts | 25 +- .../src/kernel/SupervisionEvent.schema.ts | 24 ++ .../src/kernel/SupervisorState.schema.ts | 10 +- .../interpret-supervision-event.workflow.ts | 312 ++++++++++++++---- ...pervisor-stale-reports.integration.test.ts | 23 +- 18 files changed, 1017 insertions(+), 128 deletions(-) create mode 100644 .changeset/standby-singleton-readiness.md create mode 100644 packages/effect-daemon-cluster/tests/cluster-standby-lifecycle.integration.test.ts create mode 100644 packages/effect-daemon-spec/src/Supervisor/MediumActivity.schema.ts create mode 100644 packages/effect-daemon-spec/src/__tests__/standby-acquisition.workflow.property.test.ts diff --git a/.changeset/standby-singleton-readiness.md b/.changeset/standby-singleton-readiness.md new file mode 100644 index 0000000..a6d3240 --- /dev/null +++ b/.changeset/standby-singleton-readiness.md @@ -0,0 +1,7 @@ +--- +"@systemfsoftware/effect-daemon-spec": patch +"@systemfsoftware/effect-daemon-conformance": patch +"@systemfsoftware/effect-daemon-cluster": patch +--- + +Standby singletons no longer report ready before shard acquisition. A cluster singleton whose shard another runner owns now registers as a `standby`: it is not ready, children declared after it do not start until it owns its shard and its run signals ready, and `startTimeoutMillis` is measured from shard acquisition rather than registration. Losing the shard is a handoff, not a failure: the child returns to standby without a termination, restart, or intensity charge. The supervisor starts children in declared order, each gated on the readiness of the ones before it, and status and the trace expose `standby`, `acquired`, `ready` and `released` transitions for every child. diff --git a/packages/effect-daemon-cluster/README.md b/packages/effect-daemon-cluster/README.md index ffa82a4..39eca9e 100644 --- a/packages/effect-daemon-cluster/README.md +++ b/packages/effect-daemon-cluster/README.md @@ -21,9 +21,12 @@ Its peer dependencies, `effect@^4.1.2` and `typescript@>=7.0.0`, come from your an inferred failure. - **The declared loss**: the medium declares `{ reporting: 'inferred', groupStop: 'eventual' }`, so the conformance kit holds it only to what it can observe and honour. -- **Standby and handoff**: a singleton whose shard another runner owns is announced ready at once - and waits; it starts when the shard arrives. A run interrupted because its shard moved away is a - handoff, not a termination: the child stays registered and nothing is reported. +- **Standby and handoff**: a singleton whose shard another runner owns registers as a standby: it is + not ready, it is not bound by the start deadline, and children declared after it do not start until + it owns its shard and its run signals ready. The start deadline is armed when the shard arrives, so + `startTimeoutMillis` bounds the real post-acquisition initialization. A run interrupted because its + shard moved away is a handoff, not a termination: the child returns to standby, stays registered, + and nothing is reported, restarted or charged against the restart intensity. - **Singletons across processes over Postgres**: `ClusterMedium.sqlRunnerLayer` runs `Sharding` over `SqlRunnerStorage` with no-op runner transport, health and message storage. The oldest live runner owns every shard, so a joining process waits as a standby, and a runner whose sessions are diff --git a/packages/effect-daemon-cluster/src/ClusterMedium/medium.ts b/packages/effect-daemon-cluster/src/ClusterMedium/medium.ts index 17329b5..bff8e25 100644 --- a/packages/effect-daemon-cluster/src/ClusterMedium/medium.ts +++ b/packages/effect-daemon-cluster/src/ClusterMedium/medium.ts @@ -1,5 +1,19 @@ import { Supervisor } from '@systemfsoftware/effect-daemon-spec' -import { Cause, Deferred, Duration, Effect, Exit, Layer, Match, Option, Ref, Result, Scope } from 'effect' +import { + Cause, + Deferred, + Duration, + Effect, + Exit, + Layer, + Match, + Option, + Queue, + Ref, + Result, + Scope, + Stream, +} from 'effect' import { EntityId, Sharding } from 'effect/cluster' import { absurd } from 'effect/Function' import type { ClusterProgram, ClusterProgramRequirements, EntityChild, SingletonChild } from './ClusterProgram.js' @@ -30,6 +44,7 @@ const clusterStarted = ( ended: Deferred.Deferred, stopping: Ref.Ref, probe: Effect.Effect, + activity?: Supervisor.Medium.StartedActivity, ): ClusterStarted => ({ ...Supervisor.Medium.started(ready), [ClusterStartedTypeId]: ClusterStartedTypeId, @@ -37,6 +52,7 @@ const clusterStarted = ( ended, stopping, probe, + ...(activity === undefined ? {} : { activity }), }) const isClusterStarted = (evidence: Supervisor.Medium.Started): evidence is ClusterStarted => @@ -81,8 +97,43 @@ const reportedOnExit = ( const ownsSingleton = (sharding: Sharding.Sharding['Service'], name: string, shardGroup: string | undefined) => sharding.hasShardId(sharding.getShardId(EntityId.make(name), shardGroup ?? 'default')) -const readyWhen = (owned: boolean, signalled: Deferred.Deferred): Effect.Effect => - owned ? Deferred.await(signalled) : Effect.void +const announceAcquire = (changes: Queue.Queue): Effect.Effect => + Queue.offer(changes, { _tag: 'Acquired' }).pipe(Effect.asVoid) + +const announcesAcquire = (initiallyStandby: boolean, seen: number): boolean => initiallyStandby || seen > 0 + +const singletonReady = ( + signalled: Deferred.Deferred, + changes: Queue.Queue, +): Effect.Effect => + Effect.flatMap(Deferred.isDone(signalled), (alreadySignalled) => + alreadySignalled + ? Queue.offer(changes, { _tag: 'Ready' }).pipe(Effect.asVoid) + : Effect.asVoid(Deferred.succeed(signalled, void 0))) + +const singletonRun = ( + program: SingletonChild, + signalled: Deferred.Deferred, + ended: Deferred.Deferred, + stopping: Ref.Ref, + changes: Queue.Queue, + initiallyStandby: boolean, + runs: Ref.Ref, +): Effect.Effect => + Effect.gen(function*() { + const seen = yield* Ref.getAndUpdate(runs, (count) => count + 1) + if (announcesAcquire(initiallyStandby, seen)) { + yield* announceAcquire(changes) + } + yield* program.run(singletonReady(signalled, changes)) + }).pipe( + Effect.onExit((exit) => + Effect.flatMap(Ref.get(stopping), (stopped) => + lostItsShard(stopped, exit) + ? Effect.asVoid(Queue.offer(changes, { _tag: 'Released' })) + : reportedOnExit(ended, stopping)(exit)) + ), + ) const startSingleton = ( program: SingletonChild, @@ -95,15 +146,22 @@ const startSingleton = ( const ended = yield* Deferred.make() const stopping = yield* Ref.make(false) const sharding = yield* Sharding.Sharding - const run = program.run(Deferred.succeed(signalled, void 0)).pipe( - Effect.onExit(reportedOnExit(ended, stopping)), - ) + const changes = yield* Queue.unbounded() + const runs = yield* Ref.make(0) + const initiallyStandby = ownsSingleton(sharding, program.name, shardGroup) === false + const run = singletonRun(program, signalled, ended, stopping, changes, initiallyStandby, runs) yield* sharding.registerSingleton(program.name, run, { shardGroup }).pipe( Effect.provideService(Scope.Scope, registration), ) - const ready = readyWhen(ownsSingleton(sharding, program.name, shardGroup), signalled) const probe = Deferred.isDone(ended).pipe(Effect.map((done) => done === false)) - return clusterStarted(ready, registration, ended, stopping, probe) + return clusterStarted( + Deferred.await(signalled), + registration, + ended, + stopping, + probe, + { initiallyStandby, changes: Stream.fromQueue(changes) }, + ) }) const startEntity = ( diff --git a/packages/effect-daemon-cluster/tests/cluster-medium.conformance.test.ts b/packages/effect-daemon-cluster/tests/cluster-medium.conformance.test.ts index 6f2d050..f6836a1 100644 --- a/packages/effect-daemon-cluster/tests/cluster-medium.conformance.test.ts +++ b/packages/effect-daemon-cluster/tests/cluster-medium.conformance.test.ts @@ -114,13 +114,14 @@ const standbyThenOwned = Effect.gen(function*() { const shards = yield* ShardScript const { runs, program } = yield* countedSingleton const evidence = yield* medium.start(program) - yield* evidence.ready + const readyWhileStandby = yield* Effect.timeoutOption(evidence.ready, Duration.millis(100)) const aliveAsStandby = yield* medium.probe(evidence) const runsAsStandby = yield* Ref.get(runs) yield* shards.grant(SINGLETON) + yield* evidence.ready const startedOnceOwned = yield* runsReach(runs, 1) const reported = yield* reportedWithin(medium.report(evidence)) - return { aliveAsStandby, runsAsStandby, startedOnceOwned, reported } + return { readyWhileStandby, aliveAsStandby, runsAsStandby, startedOnceOwned, reported } }) const handedOffAndBack = Effect.gen(function*() { @@ -141,7 +142,6 @@ const standbyStopped = Effect.gen(function*() { const { medium } = yield* ClusterMedium.port const { program } = yield* countedSingleton const evidence = yield* medium.start(program) - yield* evidence.ready yield* medium.stop(evidence, { _tag: 'Brutal' }) const reported = yield* reportedWithin(medium.report(evidence)) yield* nothingLeftRegistered @@ -163,13 +163,15 @@ Feature('Supervising children hosted by the cluster', { timeout: 120_000 }) 'observed', (s) => Effect.provide(standbyThenOwned, s.environment), ), - Then('it was ready and alive without running, ran once its shard arrived, and nothing ended')((s, expect) => - expect(s.observed).toEqual({ - aliveAsStandby: true, - runsAsStandby: 0, - startedOnceOwned: true, - reported: Option.none(), - }) + Then('it was not ready yet stayed alive without running, ran once its shard arrived, and nothing ended')( + (s, expect) => + expect(s.observed).toEqual({ + readyWhileStandby: Option.none(), + aliveAsStandby: true, + runsAsStandby: 0, + startedOnceOwned: true, + reported: Option.none(), + }), ), ), ) diff --git a/packages/effect-daemon-cluster/tests/cluster-standby-lifecycle.integration.test.ts b/packages/effect-daemon-cluster/tests/cluster-standby-lifecycle.integration.test.ts new file mode 100644 index 0000000..b7e0905 --- /dev/null +++ b/packages/effect-daemon-cluster/tests/cluster-standby-lifecycle.integration.test.ts @@ -0,0 +1,250 @@ +import { ClusterMedium } from '@systemfsoftware/effect-daemon-cluster' +import { Supervisor } from '@systemfsoftware/effect-daemon-spec' +import { Gherkin, Given, it, makeFeature, Then, When } from '@systemfsoftware/effect-gherkin-spec' +import { Array as Arr, Effect, Layer, Match, Ref, Stream } from 'effect' +import { RegistrationLedger, scriptedShardingOwning, ShardScript } from './__fixtures__/scripted-sharding.fixture.js' + +const Feature = makeFeature({ it }) + +const SINGLETON = 'orders/settlement' +const DEPENDENT = 'orders/receipts' + +type Statuses = ReadonlyArray + +const statusesOf = (handle: Supervisor.RunningSupervisor): Effect.Effect => + Effect.map(Supervisor.statusOf(handle), (state) => + Match.value(state).pipe( + Match.tag('Terminated', () => Arr.empty()), + Match.orElse((running) => + Arr.map(running.core.children, (child) => [child.childId, child.generation, child.status] as const) + ), + )) + +const heldNames: Effect.Effect, never, RegistrationLedger> = Effect.flatten( + Effect.map(Effect.service(RegistrationLedger), (ledger) => ledger.held), +) + +interface LifecycleState { + readonly handle: Supervisor.RunningSupervisor + readonly trace: Ref.Ref> + readonly runs: Ref.Ref + readonly initialized: Ref.Ref + readonly dependentSaw: Ref.Ref +} + +const lifecycleTree = Effect.gen(function*() { + const runs = yield* Ref.make(0) + const initialized = yield* Ref.make(false) + const dependentSaw = yield* Ref.make(null) + const singleton = ClusterMedium.singletonChild({ + name: SINGLETON, + run: (ready) => + Effect.gen(function*() { + yield* Ref.update(runs, (count) => count + 1) + yield* Ref.set(initialized, true) + yield* ready + return yield* Effect.never + }), + }) + const dependent = ClusterMedium.singletonChild({ + name: DEPENDENT, + run: (ready) => + Effect.gen(function*() { + const saw = yield* Ref.get(initialized) + yield* Ref.set(dependentSaw, saw) + yield* ready + return yield* Effect.never + }), + }) + const handle = yield* Supervisor.make('standby-tree').pipe( + Supervisor.strategy('rest_for_one'), + Supervisor.intensity(1, 60_000), + Supervisor.children([ + Supervisor.ChildSpecs.on(ClusterMedium.port)('settlement', singleton, { startTimeoutMillis: 2_000 }), + Supervisor.ChildSpecs.on(ClusterMedium.port)('dependent', dependent, { startTimeoutMillis: 2_000 }), + ]), + ).scoped + const trace = yield* Ref.make>([]) + yield* Stream.runForEach(Supervisor.traceOf(handle), (entry) => Ref.update(trace, (seen) => [...seen, entry])).pipe( + Effect.forkScoped, + Effect.asVoid, + ) + return { handle, trace, runs, initialized, dependentSaw } satisfies LifecycleState +}) + +const eventLabel = (entry: Supervisor.TraceEntry): string => + Match.value(entry.event).pipe( + Match.tag('ChildStarted', (started) => `started:${started.childId}:${started.generation}`), + Match.tag('ChildStandby', (standby) => `standby:${standby.childId}:${standby.generation}`), + Match.tag('ChildAcquired', (acquired) => `acquired:${acquired.childId}:${acquired.generation}`), + Match.tag('ChildReady', (ready) => `ready:${ready.childId}:${ready.generation}`), + Match.tag('ChildReleased', (released) => `released:${released.childId}:${released.generation}`), + Match.tag('ChildTerminated', (terminated) => `terminated:${terminated.childId}:${terminated.generation}`), + Match.orElse(() => 'other'), + ) + +const traceLabelsOf = (state: LifecycleState): Effect.Effect> => + Effect.map(Ref.get(state.trace), (entries) => Arr.map(entries, eventLabel)) + +const countOf = (labels: ReadonlyArray, prefix: string): number => + Arr.filter(labels, (label) => label.startsWith(prefix)).length + +const firstIndexOf = (labels: ReadonlyArray, wanted: string): number => + Arr.findFirstIndex(labels, (label) => label === wanted).pipe( + Match.value, + Match.tag('Some', ({ value }) => value), + Match.tag('None', () => -1), + Match.exhaustive, + ) + +const standbyToHandoffAndBack = Effect.gen(function*() { + const state = yield* lifecycleTree + const shards = yield* ShardScript + yield* Effect.sleep('50 millis') + const bootStatuses = yield* statusesOf(state.handle) + const bootHeld = yield* heldNames + yield* shards.grant(SINGLETON) + yield* shards.grant(DEPENDENT) + yield* Effect.sleep('50 millis') + const activeStatuses = yield* statusesOf(state.handle) + const activeHeld = yield* heldNames + const runsAfterPromotion = yield* Ref.get(state.runs) + const dependentSawInitialized = yield* Ref.get(state.dependentSaw) + const activeLabels = yield* traceLabelsOf(state) + const dependentStartedAfterSingletonReady = firstIndexOf(activeLabels, 'ready:settlement:0') < firstIndexOf( + activeLabels, + 'started:dependent:0', + ) + yield* shards.revoke(SINGLETON) + yield* Effect.sleep('50 millis') + const handedOffStatuses = yield* statusesOf(state.handle) + const handedOffLabels = yield* traceLabelsOf(state) + const handoffLeftDependentRunning = countOf(handedOffLabels, 'terminated') === 0 && + countOf(handedOffLabels, 'started:dependent') === 1 && + countOf(handedOffLabels, 'released:settlement') === 1 + yield* shards.grant(SINGLETON) + yield* Effect.sleep('50 millis') + const repromotedStatuses = yield* statusesOf(state.handle) + const repromotedLabels = yield* traceLabelsOf(state) + const runsAfterRepromotion = yield* Ref.get(state.runs) + const supervisorRunning = Arr.length(repromotedStatuses) > 0 + yield* Supervisor.shutdown(state.handle) + return { + bootStatuses, + bootHeld, + activeStatuses, + activeHeld, + runsAfterPromotion, + dependentSawInitialized, + dependentStartedAfterSingletonReady, + handedOffStatuses, + handoffLeftDependentRunning, + repromotedStatuses, + supervisorRunning, + runsAfterRepromotion, + acquisitions: countOf(repromotedLabels, 'acquired:settlement'), + dependentStarts: countOf(repromotedLabels, 'started:dependent'), + terminated: countOf(repromotedLabels, 'terminated'), + } +}) + +const timeoutEventsOf = (entries: ReadonlyArray): ReadonlyArray => + Arr.map(entries, (entry) => + Match.value(entry.event).pipe( + Match.tag('SupervisorStarted', () => 'supervisor-started'), + Match.tag('ChildStarted', () => 'started'), + Match.tag('ChildStandby', () => 'standby'), + Match.tag('ChildAcquired', () => 'acquired'), + Match.tag('ChildTerminated', () => 'terminated'), + Match.tag('TimerElapsed', () => 'timer'), + Match.orElse(() => 'other'), + )) + +const postAcquisitionTimeout = Effect.gen(function*() { + const neverReady = ClusterMedium.singletonChild({ + name: 'orders/hanging', + run: () => Effect.never, + }) + const handle = yield* Supervisor.make('timeout-tree').pipe( + Supervisor.strategy('one_for_one'), + Supervisor.intensity(0, 60_000), + Supervisor.children([ + Supervisor.ChildSpecs.on(ClusterMedium.port)('hanging', neverReady, { startTimeoutMillis: 200 }), + ]), + ).scoped + const trace = yield* Ref.make>([]) + yield* Stream.runForEach(Supervisor.traceOf(handle), (entry) => Ref.update(trace, (seen) => [...seen, entry])).pipe( + Effect.forkScoped, + Effect.asVoid, + ) + yield* Effect.sleep('400 millis') + const standbyStatuses = yield* statusesOf(handle) + const shards = yield* ShardScript + yield* shards.grant('orders/hanging') + yield* Effect.sleep('2 seconds') + const afterTimeoutStatuses = yield* statusesOf(handle) + const events = yield* Effect.map(Ref.get(trace), timeoutEventsOf) + const acquiredAt = firstIndexOf(events, 'acquired') + const fromAcquisition = acquiredAt === -1 ? events : Arr.drop(events, acquiredAt) + return { standbyStatuses, afterTimeoutStatuses, fromAcquisition } +}) + +Feature('Standby singletons under shard movement', { timeout: 120_000 }) + .withLayer(Layer.empty) + .live('each scenario drives the simulation kernel itself, and a conformance check cannot run inside a kernel run') + .body(({ scenario }) => { + scenario( + 'A rest-for-one dependent starts only after the standby singleton owns its shard and runs ready, and handoff is not a failure', + Gherkin.Do.pipe( + Given('a cluster runner that owns no shard yet')( + 'environment', + () => Layer.build(Layer.merge(ClusterMedium.layer(), scriptedShardingOwning(false))), + ), + When('the singleton acquires its shard, hands off, and is promoted again')( + 'observed', + (s) => Effect.provide(standbyToHandoffAndBack, s.environment), + ), + Then('standby, acquisition, ordering, handoff and re-promotion are all visible with no termination')( + (s, expect) => + expect(s.observed).toEqual({ + bootStatuses: [['settlement', 0, 'standby'], ['dependent', 0, 'starting']], + bootHeld: [SINGLETON], + activeStatuses: [['settlement', 0, 'ready'], ['dependent', 0, 'ready']], + activeHeld: [SINGLETON, DEPENDENT], + runsAfterPromotion: 1, + dependentSawInitialized: true, + dependentStartedAfterSingletonReady: true, + handedOffStatuses: [['settlement', 0, 'standby'], ['dependent', 0, 'ready']], + handoffLeftDependentRunning: true, + repromotedStatuses: [['settlement', 0, 'ready'], ['dependent', 0, 'ready']], + supervisorRunning: true, + runsAfterRepromotion: 2, + acquisitions: 2, + dependentStarts: 1, + terminated: 0, + }), + ), + ), + ) + + scenario( + 'A standby singleton has no start deadline until its shard arrives, and the post-acquisition start is bounded', + Gherkin.Do.pipe( + Given('a cluster runner that owns no shard yet')( + 'environment', + () => Layer.build(Layer.merge(ClusterMedium.layer(), scriptedShardingOwning(false))), + ), + When('a never-ready singleton waits standby longer than its start timeout and then acquires its shard')( + 'observed', + (s) => Effect.provide(postAcquisitionTimeout, s.environment), + ), + Then('nothing fired while standby, and the deadline missed only after acquisition')((s, expect) => + expect(s.observed).toEqual({ + standbyStatuses: [['hanging', 0, 'standby']], + afterTimeoutStatuses: [], + fromAcquisition: ['acquired', 'timer', 'terminated'], + }) + ), + ), + ) + }) diff --git a/packages/effect-daemon-conformance/src/Trace.schema.ts b/packages/effect-daemon-conformance/src/Trace.schema.ts index ee6130b..cd97a71 100644 --- a/packages/effect-daemon-conformance/src/Trace.schema.ts +++ b/packages/effect-daemon-conformance/src/Trace.schema.ts @@ -27,6 +27,9 @@ export const EventKind = Schema.Literals([ 'SupervisorStarted', 'ChildStarted', 'ChildReady', + 'ChildStandby', + 'ChildAcquired', + 'ChildReleased', 'ChildTerminated', 'ChildStopped', 'ProbeResult', diff --git a/packages/effect-daemon-conformance/src/observed-trace.ts b/packages/effect-daemon-conformance/src/observed-trace.ts index eb4ab95..e1e580c 100644 --- a/packages/effect-daemon-conformance/src/observed-trace.ts +++ b/packages/effect-daemon-conformance/src/observed-trace.ts @@ -75,6 +75,12 @@ const eventOf = (event: Event): ObservedEvent => observedEvent('ChildStarted', childRef(started.childId, started.generation), null)), Match.tag('ChildReady', (ready) => observedEvent('ChildReady', childRef(ready.childId, ready.generation), null)), + Match.tag('ChildStandby', (standby) => + observedEvent('ChildStandby', childRef(standby.childId, standby.generation), null)), + Match.tag('ChildAcquired', (acquired) => + observedEvent('ChildAcquired', childRef(acquired.childId, acquired.generation), null)), + Match.tag('ChildReleased', (released) => + observedEvent('ChildReleased', childRef(released.childId, released.generation), null)), Match.tag( 'ChildTerminated', (terminated) => @@ -90,14 +96,17 @@ const eventOf = (event: Event): ObservedEvent => observedEvent('ProbeResult', childRef(probe.childId, probe.generation), null)), Match.tag('TimerElapsed', (timer) => observedEvent('TimerElapsed', timerChildOf(timer.target), null)), - Match.tag('DynamicStartRequested', () => observedEvent('DynamicStartRequested', null, null)), + Match.tag('DynamicStartRequested', () => + observedEvent('DynamicStartRequested', null, null)), Match.tag( 'DynamicStopRequested', - (stopping) => observedEvent('DynamicStopRequested', childRef(stopping.childId, stopping.generation), null), + (stopping) => + observedEvent('DynamicStopRequested', childRef(stopping.childId, stopping.generation), null), ), Match.tag( 'ShutdownRequested', - (shutdown) => observedEvent('ShutdownRequested', null, reasonKindOf(shutdown.reason)), + (shutdown) => + observedEvent('ShutdownRequested', null, reasonKindOf(shutdown.reason)), ), Match.exhaustive, ) diff --git a/packages/effect-daemon-spec/README.md b/packages/effect-daemon-spec/README.md index c228ee4..53c0578 100644 --- a/packages/effect-daemon-spec/README.md +++ b/packages/effect-daemon-spec/README.md @@ -56,6 +56,8 @@ Use `app.scoped` instead of `app.layer` to get the running supervisor handle ins `Supervisor.intensity(n, periodMillis)` allows at most `n` restarts within `periodMillis`. The default is OTP's: one restart in five seconds. One more restart than that stops every child and terminates the supervisor; whoever is waiting on it observes a give-up (see [Observing a running supervisor](#observing-a-running-supervisor)). +Children start in declared order, and a later child starts only once the children before it are ready (or have exited): a dependent never starts against an uninitialized sibling. A child that must wait for a resource before its real startup — a cluster medium singleton waiting for its shard, say — registers as a `standby` instead of reporting ready, and its start timeout is measured from when the resource arrives, not from registration. + ## Children `Supervisor.ChildSpecs.make(id, program, options)` declares a child. Every option is optional; the first three default as OTP does. diff --git a/packages/effect-daemon-spec/src/Supervisor/Medium.ts b/packages/effect-daemon-spec/src/Supervisor/Medium.ts index f07e87a..c37463d 100644 --- a/packages/effect-daemon-spec/src/Supervisor/Medium.ts +++ b/packages/effect-daemon-spec/src/Supervisor/Medium.ts @@ -1,4 +1,4 @@ -import { Context, Effect } from 'effect' +import { Context, Effect, type Stream } from 'effect' import type { ShutdownMode } from '../kernel/SupervisorPolicy.schema.js' import type { TerminationReason } from '../kernel/TerminationReport.schema.js' export { BrutalShutdown, GracefulShutdown, InfinityShutdown, ShutdownMode } from '../kernel/SupervisorPolicy.schema.js' @@ -13,7 +13,9 @@ export { ShutdownTermination, TerminationReason, } from '../kernel/TerminationReport.schema.js' +export { Acquired, MediumActivity, Released } from './MediumActivity.schema.js' export { GroupStopGuarantee, MediumDeclaration, MediumReporting } from './MediumDeclaration.schema.js' +import type { MediumActivity } from './MediumActivity.schema.js' import type { MediumDeclaration } from './MediumDeclaration.schema.js' export const StartedTypeId: unique symbol = Symbol.for('@systemfsoftware/effect-daemon-spec/Medium/Started') @@ -22,16 +24,15 @@ export type StartedTypeId = typeof StartedTypeId export const StoppedTypeId: unique symbol = Symbol.for('@systemfsoftware/effect-daemon-spec/Medium/Stopped') export type StoppedTypeId = typeof StoppedTypeId -/** - * Evidence that a medium started a child (KTD8): unforgeable, because only a - * medium's `start` port can mint it. It carries the readiness signal the kernel - * races against its start-deadline timer (R10); the termination report and the - * single-shot liveness probe are the medium's `report` and `probe` ports run - * over this evidence, so neither can observe a child that never started. - */ +export interface StartedActivity { + readonly initiallyStandby: boolean + readonly changes: Stream.Stream +} + export interface Started { readonly [StartedTypeId]: StartedTypeId readonly ready: Effect.Effect + readonly activity?: StartedActivity | undefined } /** diff --git a/packages/effect-daemon-spec/src/Supervisor/MediumActivity.schema.ts b/packages/effect-daemon-spec/src/Supervisor/MediumActivity.schema.ts new file mode 100644 index 0000000..b228f90 --- /dev/null +++ b/packages/effect-daemon-spec/src/Supervisor/MediumActivity.schema.ts @@ -0,0 +1,13 @@ +import { Schema } from 'effect' + +export const Acquired = Schema.TaggedStruct('Acquired', {}) +export type Acquired = typeof Acquired.Type + +export const Ready = Schema.TaggedStruct('Ready', {}) +export type Ready = typeof Ready.Type + +export const Released = Schema.TaggedStruct('Released', {}) +export type Released = typeof Released.Type + +export const MediumActivity = Schema.Union([Acquired, Ready, Released]) +export type MediumActivity = typeof MediumActivity.Type diff --git a/packages/effect-daemon-spec/src/Supervisor/supervisor-commands.ts b/packages/effect-daemon-spec/src/Supervisor/supervisor-commands.ts index d231f20..4ceeac1 100644 --- a/packages/effect-daemon-spec/src/Supervisor/supervisor-commands.ts +++ b/packages/effect-daemon-spec/src/Supervisor/supervisor-commands.ts @@ -12,6 +12,7 @@ import { Ref, Schema, Scope, + Stream, } from 'effect' import type { SupervisionEvent, TimerKind } from '../kernel/SupervisionEvent.schema.js' import type { ChildId, Generation } from '../kernel/SupervisionLimits.schema.js' @@ -30,7 +31,7 @@ import type { IntensityExceededExit } from '../kernel/TerminationReport.schema.j import type { BoundChild, Incarnation } from './bound-child.js' import { outcomeOf } from './DynamicOutcome.schema.js' import * as FiberMedium from './FiberMedium.js' -import type { Started } from './Medium.js' +import type { Started, StartedActivity } from './Medium.js' import type { RunningSupervisor } from './running-supervisor.handle.js' import { boundChildrenOf, @@ -69,6 +70,27 @@ const startedEventOf = (childId: ChildId, generation: Generation): SupervisionEv generation, }) +const standbyEventOf = (childId: ChildId, generation: Generation): SupervisionEvent => ({ + _tag: 'ChildStandby', + at: 0, + childId, + generation, +}) + +const acquiredEventOf = (childId: ChildId, generation: Generation): SupervisionEvent => ({ + _tag: 'ChildAcquired', + at: 0, + childId, + generation, +}) + +const releasedEventOf = (childId: ChildId, generation: Generation): SupervisionEvent => ({ + _tag: 'ChildReleased', + at: 0, + childId, + generation, +}) + const readyEventOf = (childId: ChildId, generation: Generation): SupervisionEvent => ({ _tag: 'ChildReady', at: 0, @@ -145,6 +167,33 @@ const watchReadiness = ( ), ) +const watchActivity = ( + handle: RunningSupervisor, + evidence: Started, + childId: ChildId, + generation: Generation, +): Effect.Effect => + evidence.activity === undefined ? Effect.void : forkActivityWatch(handle, evidence.activity, childId, generation) + +const forkActivityWatch = ( + handle: RunningSupervisor, + activity: StartedActivity, + childId: ChildId, + generation: Generation, +): Effect.Effect => + Effect.asVoid( + Effect.forkIn( + Stream.runForEach(activity.changes, (change) => + Match.value(change).pipe( + Match.tag('Acquired', () => stampedNow(handle, acquiredEventOf(childId, generation))), + Match.tag('Ready', () => stampedNow(handle, readyEventOf(childId, generation))), + Match.tag('Released', () => stampedNow(handle, releasedEventOf(childId, generation))), + Match.exhaustive, + )), + ownerScopeOf(handle), + ), + ) + const watchReport = ( handle: RunningSupervisor, child: BoundChild, @@ -187,8 +236,14 @@ const executeStart = ( Effect.andThen( stampedNow(acquired.handle, startedEventOf(command.childId, command.generation)), Effect.andThen( - watchReadiness(acquired.handle, incarnation.evidence, command.childId, command.generation), - watchReport(acquired.handle, child, incarnation, command.childId, command.generation), + announceStandby(acquired.handle, incarnation.evidence, command.childId, command.generation), + Effect.andThen( + watchReadiness(acquired.handle, incarnation.evidence, command.childId, command.generation), + Effect.andThen( + watchActivity(acquired.handle, incarnation.evidence, command.childId, command.generation), + watchReport(acquired.handle, child, incarnation, command.childId, command.generation), + ), + ), ), ), ), @@ -200,6 +255,21 @@ const executeStart = ( }) }) +const initiallyStandbyOf = (evidence: Started): boolean => { + const activity = evidence.activity + return activity === undefined ? false : activity.initiallyStandby +} + +const announceStandby = ( + handle: RunningSupervisor, + evidence: Started, + childId: ChildId, + generation: Generation, +): Effect.Effect => + initiallyStandbyOf(evidence) + ? stampedNow(handle, standbyEventOf(childId, generation)) + : Effect.void + /** * Moves each answered request's pending child: an accepted start binds it to the * child the kernel allocated, and every answered request drops its pending entry. diff --git a/packages/effect-daemon-spec/src/__tests__/interpret-supervision-event.workflow.property.test.ts b/packages/effect-daemon-spec/src/__tests__/interpret-supervision-event.workflow.property.test.ts index 0be0f98..27ced5d 100644 --- a/packages/effect-daemon-spec/src/__tests__/interpret-supervision-event.workflow.property.test.ts +++ b/packages/effect-daemon-spec/src/__tests__/interpret-supervision-event.workflow.property.test.ts @@ -94,7 +94,7 @@ const coreWith = ( policy: SupervisionPolicy, children: ReadonlyArray, restartStamps: ReadonlyArray, -): SupervisorCore => ({ policy, children, restartStamps, nextOrdinal: 0 }) +): SupervisorCore => ({ policy, children, restartStamps, nextOrdinal: 0, awaiting: [] }) const runningOf = (core: SupervisorCore): SupervisorState => new Running({ core }) @@ -404,6 +404,8 @@ const textOf = (items: ReadonlyArray): ReadonlyArray => const idSeqOf = (children: ReadonlyArray): ReadonlyArray => Arr.map(children, (child) => child.childId) +const awaitingIdsOf = (core: SupervisorCore): ReadonlyArray => Arr.map(core.awaiting, (start) => start.childId) + const labelOf = (child: ChildInstance): string => `${child.childId}:${child.generation}:${child.status}` const startIdsFromDecision = (decision: SupervisionDecision): ReadonlyArray => @@ -492,7 +494,8 @@ it.prop( Match.value(coreOf(decision)).pipe( Match.tag('Some', ({ value }) => sameSeq(stopIdsOf(commandsOf(decision)), Arr.reverse(groupIds)) && - sameSeq(startIdsOf(commandsOf(decision)), comingIds) && + sameSeq(startIdsOf(commandsOf(decision)), Arr.take(comingIds, 1)) && + sameSeq(awaitingIdsOf(value), Arr.drop(comingIds, 1)) && sameSeq(textOf(value.restartStamps), textOf([at])) && sameSeq( Arr.map( @@ -643,7 +646,12 @@ it.prop( return isRestartFamily(decision) && stamped && Arr.length(stopIdsOf(commandsOf(decision))) === 3 && - Arr.length(startIdsOf(commandsOf(decision))) === 3 + sameSeq(startIdsOf(commandsOf(decision)), ['c0']) && + Match.value(coreOf(decision)).pipe( + Match.tag('Some', ({ value }) => sameSeq(awaitingIdsOf(value), ['c1', 'c2'])), + Match.tag('None', () => false), + Match.exhaustive, + ) }, ) @@ -1060,7 +1068,10 @@ it.prop( strategy: 'one_for_one', childDeclarations: [declaredChild(childId, 'permanent', false)], }) - const state = new Restarting({ core: coreWith(policy, [], []), pending: [{ childId, generation: 1 }] }) + const state = new Restarting({ + core: coreWith(policy, [childOf(childId, 1, 'starting')], []), + pending: [{ childId, generation: 1 }], + }) const decision = decidedOf(subject, stepOf(state, childTimerEvent('backoff', childId, 1, 10))) const started = startIdsOf(commandsOf(decision)) diff --git a/packages/effect-daemon-spec/src/__tests__/standby-acquisition.workflow.property.test.ts b/packages/effect-daemon-spec/src/__tests__/standby-acquisition.workflow.property.test.ts new file mode 100644 index 0000000..acfdbde --- /dev/null +++ b/packages/effect-daemon-spec/src/__tests__/standby-acquisition.workflow.property.test.ts @@ -0,0 +1,244 @@ +import { it } from '@systemfsoftware/vitest' +import { Array as Arr, Match, Result } from 'effect' +import { absurd } from 'effect/Function' +import { + interpretSupervisionEvent, + type SupervisionDecision, + SupervisionStep, +} from '../kernel/interpret-supervision-event.workflow.js' +import type { SupervisionEvent } from '../kernel/SupervisionEvent.schema.js' +import { EventTime, Generation } from '../kernel/SupervisionLimits.schema.js' +import type { SupervisorArm } from '../kernel/SupervisorCommand.schema.js' +import { SupervisionPolicy } from '../kernel/SupervisorPolicy.schema.js' +import type { RestartType } from '../kernel/SupervisorPolicy.schema.js' +import type { ChildInstance, SupervisorCore } from '../kernel/SupervisorState.schema.js' +import { Running } from '../kernel/SupervisorState.schema.js' +import type { SupervisorState } from '../kernel/SupervisorState.schema.js' +import type { TerminationReason } from '../kernel/TerminationReport.schema.js' + +type Interpret = typeof interpretSupervisionEvent + +const PERIOD_MILLIS = 100 +const START_TIMEOUT_MILLIS = 50 +const LIVENESS_TICK_MILLIS = 10 + +const policyWith = ( + childDeclarations: ReadonlyArray<{ + readonly childId: string + readonly restartType: RestartType + }>, +): SupervisionPolicy => ({ + strategy: 'rest_for_one', + intensity: 8, + periodMillis: PERIOD_MILLIS, + autoShutdown: 'never', + coolDown: { _tag: 'NoCoolDown' }, + backoff: { baseMillis: 0, multiplier: 2, capMillis: 0 }, + dynamic: { _tag: 'NoDynamicChildren' }, + livenessTickMillis: LIVENESS_TICK_MILLIS, + childDeclarations: Arr.map(childDeclarations, (child) => ({ + childId: child.childId, + restartType: child.restartType, + shutdown: { _tag: 'Brutal' } as const, + significant: false, + startTimeoutMillis: START_TIMEOUT_MILLIS, + probeFailureThreshold: 2, + })), +}) + +const childOf = (childId: string, generation: Generation, status: ChildInstance['status']): ChildInstance => ({ + childId, + generation, + status, + consecutiveRestarts: 0, + probeFailures: 0, +}) + +const coreWith = ( + policy: SupervisionPolicy, + children: ReadonlyArray, + awaiting: ReadonlyArray<{ readonly childId: string; readonly generation: Generation }>, +): SupervisorCore => ({ policy, children, restartStamps: [], nextOrdinal: 0, awaiting }) + +const runningOf = (core: SupervisorCore): SupervisorState => new Running({ core }) + +const decidedOf = (interpret: Interpret, step: SupervisionStep): SupervisionDecision => + Result.match(interpret(step), { + onFailure: (error: never): never => absurd(error), + onSuccess: (decision) => decision, + }) + +const stepOf = (state: SupervisorState, event: SupervisionEvent): SupervisionStep => + new SupervisionStep({ state, event }) + +const startDeadlineTimerOf = (childId: string, generation: Generation, at: EventTime): SupervisionEvent => ({ + _tag: 'TimerElapsed', + at, + kind: 'start_deadline', + target: { _tag: 'ChildTimer', childId, generation }, +}) + +const acquiredEventOf = (childId: string, generation: Generation, at: EventTime): SupervisionEvent => ({ + _tag: 'ChildAcquired', + at, + childId, + generation, +}) + +const releasedEventOf = (childId: string, generation: Generation, at: EventTime): SupervisionEvent => ({ + _tag: 'ChildReleased', + at, + childId, + generation, +}) + +const readyEventOf = (childId: string, generation: Generation, at: EventTime): SupervisionEvent => ({ + _tag: 'ChildReady', + at, + childId, + generation, +}) + +const terminatedEventOf = ( + childId: string, + generation: Generation, + at: EventTime, + reason: TerminationReason, +): SupervisionEvent => ({ _tag: 'ChildTerminated', at, childId, generation, reason }) + +const isStale = (decision: SupervisionDecision): boolean => + Match.value(decision).pipe( + Match.tag('Stale', () => true), + Match.orElse(() => false), + ) + +const statusOf = (decision: SupervisionDecision, childId: string): string | null => + Match.value(decision).pipe( + Match.tag('Continue', ({ core }) => + Match.value(Arr.findFirst(core.children, (child) => + child.childId === childId)).pipe( + Match.tag('Some', ({ value }) => value.status), + Match.tag('None', () => null), + Match.exhaustive, + )), + Match.orElse(() => + null + ), + ) + +const startedIdsOf = (decision: SupervisionDecision): ReadonlyArray => + Match.value(decision).pipe( + Match.tag('Continue', ({ commands }) => Arr.map(commands.starts, (start) => start.childId)), + Match.tag('StartChildren', ({ commands }) => Arr.map(commands.starts, (start) => start.childId)), + Match.orElse((): ReadonlyArray => []), + ) + +const isStartDeadlineFor = (arm: SupervisorArm, childId: string, at: EventTime): boolean => + Match.value(arm).pipe( + Match.tag('ArmChildTimer', (timer) => + timer.kind === 'start_deadline' && timer.childId === childId && timer.deadline === at + START_TIMEOUT_MILLIS), + Match.orElse(() => + false + ), + ) + +const armedDeadlineFor = (decision: SupervisionDecision, childId: string, at: EventTime): boolean => + Match.value(decision).pipe( + Match.tag('Continue', ({ commands }) => Arr.some(commands.arms, (arm) => isStartDeadlineFor(arm, childId, at))), + Match.orElse(() => false), + ) + +it.prop( + '∀s_StandbyDeadline_=Stale', + { of: [EventTime, Generation], subject: interpretSupervisionEvent }, + (subject, [at, generation]) => { + const policy = policyWith([{ childId: 'c0', restartType: 'permanent' }]) + const state = runningOf(coreWith(policy, [childOf('c0', generation, 'standby')], [])) + return isStale(decidedOf(subject, stepOf(state, startDeadlineTimerOf('c0', generation, at)))) + }, +) + +it.prop( + '∀a_Acquired_=StartingWithDeadline', + { of: [EventTime, Generation], subject: interpretSupervisionEvent }, + (subject, [at, generation]) => { + const policy = policyWith([{ childId: 'c0', restartType: 'permanent' }]) + const state = runningOf(coreWith(policy, [childOf('c0', generation, 'standby')], [])) + const decision = decidedOf(subject, stepOf(state, acquiredEventOf('c0', generation, at))) + return statusOf(decision, 'c0') === 'starting' && armedDeadlineFor(decision, 'c0', at) + }, +) + +it.prop( + '∀r_Released_=StandbyWithoutTermination', + { of: [EventTime, Generation], subject: interpretSupervisionEvent }, + (subject, [at, generation]) => { + const policy = policyWith([{ childId: 'c0', restartType: 'permanent' }]) + const state = runningOf(coreWith(policy, [childOf('c0', generation, 'ready')], [])) + const decision = decidedOf(subject, stepOf(state, releasedEventOf('c0', generation, at))) + return Match.value(decision).pipe( + Match.tag( + 'Continue', + ({ commands }) => + statusOf(decision, 'c0') === 'standby' && commands.stops.length === 0 && commands.starts.length === 0 && + commands.terminates.length === 0, + ), + Match.orElse(() => false), + ) + }, +) + +it.prop( + '∀q_ReadyDrainsAwaiting_=StartNext', + { of: [EventTime, Generation], subject: interpretSupervisionEvent }, + (subject, [at, generation]) => { + const policy = policyWith([ + { childId: 'c0', restartType: 'permanent' }, + { childId: 'c1', restartType: 'permanent' }, + ]) + const state = runningOf(coreWith(policy, [ + childOf('c0', generation, 'standby'), + childOf('c1', generation, 'starting'), + ], [{ childId: 'c1', generation }])) + const decision = decidedOf(subject, stepOf(state, readyEventOf('c0', generation, at))) + return statusOf(decision, 'c0') === 'ready' && Arr.contains(startedIdsOf(decision), 'c1') && + armedDeadlineFor(decision, 'c1', at) + }, +) + +it.prop( + '∀q_StandbyBlocksAwaiting_=NoStart', + { of: [EventTime, Generation], subject: interpretSupervisionEvent }, + (subject, [at, generation]) => { + const policy = policyWith([ + { childId: 'c0', restartType: 'permanent' }, + { childId: 'c1', restartType: 'permanent' }, + ]) + const state = runningOf(coreWith(policy, [ + childOf('c0', generation, 'standby'), + childOf('c1', generation, 'starting'), + ], [{ childId: 'c1', generation }])) + const decision = decidedOf(subject, stepOf(state, acquiredEventOf('c0', generation, at))) + return statusOf(decision, 'c0') === 'starting' && Arr.contains(startedIdsOf(decision), 'c1') === false + }, +) + +it.prop( + '∀q_RemovedPredecessor_=UnblocksAwaiting', + { of: [EventTime, Generation], subject: interpretSupervisionEvent }, + (subject, [at, generation]) => { + const policy = policyWith([ + { childId: 'c0', restartType: 'temporary' }, + { childId: 'c1', restartType: 'permanent' }, + ]) + const state = runningOf(coreWith(policy, [ + childOf('c0', generation, 'ready'), + childOf('c1', generation, 'starting'), + ], [{ childId: 'c1', generation }])) + const terminated = decidedOf( + subject, + stepOf(state, terminatedEventOf('c0', generation, at, { _tag: 'Normal' })), + ) + return Arr.contains(startedIdsOf(terminated), 'c1') + }, +) diff --git a/packages/effect-daemon-spec/src/__tests__/supervisor-boot.workflow.property.test.ts b/packages/effect-daemon-spec/src/__tests__/supervisor-boot.workflow.property.test.ts index 10b2eb2..fef7b24 100644 --- a/packages/effect-daemon-spec/src/__tests__/supervisor-boot.workflow.property.test.ts +++ b/packages/effect-daemon-spec/src/__tests__/supervisor-boot.workflow.property.test.ts @@ -75,16 +75,27 @@ const bootCommandsOf = ( const startIdOf = (start: StartChild): string => start.childId +const awaitingIdOf = (start: { readonly childId: string }): string => start.childId + it.prop( - '∀b_BootStart_=DeclaredOrder', + '∀b_BootStart_=FirstAndQueueRest', { of: [Schema.Array(ChildDeclaration), EventTime], subject: interpretSupervisionEvent }, (subject, [drawn, at]) => { const ids = idsOf(drawn) - const commands = bootCommandsOf(subject, policyOf('one_for_one', declarationsOf(drawn)), at) - return commands.stops.length === 0 && - commands.replies.length === 0 && - commands.terminates.length === 0 && - Arr.join(Arr.map(commands.starts, startIdOf), '|') === Arr.join(ids, '|') + if (Arr.dedupe(ids).length !== ids.length) return true + const decision = bootsTo(subject, policyOf('one_for_one', declarationsOf(drawn)), at) + return Match.value(decision).pipe( + Match.tag( + 'StartChildren', + (started) => + Arr.join(Arr.map(started.commands.starts, startIdOf), '|') === Arr.join(Arr.take(ids, 1), '|') && + Arr.join(Arr.map(started.core.awaiting, awaitingIdOf), '|') === Arr.join(Arr.drop(ids, 1), '|') && + started.commands.stops.length === 0 && + started.commands.replies.length === 0 && + started.commands.terminates.length === 0, + ), + Match.orElse(() => false), + ) }, ) @@ -101,7 +112,7 @@ it.prop( Match.tag('ArmSupervisorTimer', () => false), Match.exhaustive, ) - return commands.arms.length === ids.length && Arr.every(commands.arms, deadlineHolds) + return commands.arms.length === Arr.length(Arr.take(ids, 1)) && Arr.every(commands.arms, deadlineHolds) }, ) diff --git a/packages/effect-daemon-spec/src/kernel/SupervisionEvent.schema.ts b/packages/effect-daemon-spec/src/kernel/SupervisionEvent.schema.ts index fbc90d3..095a582 100644 --- a/packages/effect-daemon-spec/src/kernel/SupervisionEvent.schema.ts +++ b/packages/effect-daemon-spec/src/kernel/SupervisionEvent.schema.ts @@ -38,6 +38,27 @@ export const ChildReady = Schema.TaggedStruct('ChildReady', { }) export type ChildReady = typeof ChildReady.Type +export const ChildStandby = Schema.TaggedStruct('ChildStandby', { + at: EventTime, + childId: ChildId, + generation: Generation, +}) +export type ChildStandby = typeof ChildStandby.Type + +export const ChildAcquired = Schema.TaggedStruct('ChildAcquired', { + at: EventTime, + childId: ChildId, + generation: Generation, +}) +export type ChildAcquired = typeof ChildAcquired.Type + +export const ChildReleased = Schema.TaggedStruct('ChildReleased', { + at: EventTime, + childId: ChildId, + generation: Generation, +}) +export type ChildReleased = typeof ChildReleased.Type + export const ChildTerminated = Schema.TaggedStruct('ChildTerminated', { at: EventTime, childId: ChildId, @@ -92,6 +113,9 @@ export const SupervisionEvent = Schema.Union([ SupervisorStarted, ChildStarted, ChildReady, + ChildStandby, + ChildAcquired, + ChildReleased, ChildTerminated, ChildStopped, ProbeResult, diff --git a/packages/effect-daemon-spec/src/kernel/SupervisorState.schema.ts b/packages/effect-daemon-spec/src/kernel/SupervisorState.schema.ts index a3ae498..c44c2ef 100644 --- a/packages/effect-daemon-spec/src/kernel/SupervisorState.schema.ts +++ b/packages/effect-daemon-spec/src/kernel/SupervisorState.schema.ts @@ -15,7 +15,7 @@ import { SupervisorExit, TerminationReason } from './TerminationReport.schema.js const StateTypeId: unique symbol = Symbol.for('@systemfsoftware/effect-daemon/SupervisorState') -export const ChildStatus = Schema.Literals(['starting', 'ready', 'stopping']) +export const ChildStatus = Schema.Literals(['standby', 'starting', 'ready', 'stopping']) export type ChildStatus = typeof ChildStatus.Type export const ChildInstance = Schema.Struct({ @@ -27,17 +27,18 @@ export const ChildInstance = Schema.Struct({ }) export type ChildInstance = typeof ChildInstance.Type +export const ChildStart = Schema.Struct({ childId: ChildId, generation: Generation }) +export type ChildStart = typeof ChildStart.Type + export const SupervisorCore = Schema.Struct({ policy: SupervisionPolicy, children: Schema.Array(ChildInstance), restartStamps: Schema.Array(EventTime), nextOrdinal: Ordinal, + awaiting: Schema.Array(ChildStart), }) export type SupervisorCore = typeof SupervisorCore.Type -export const ChildStart = Schema.Struct({ childId: ChildId, generation: Generation }) -export type ChildStart = typeof ChildStart.Type - export class Running extends Schema.TaggedClass()('Running', { core: SupervisorCore }) { @@ -88,6 +89,7 @@ export const initialStateOf = (policy: SupervisionPolicy): SupervisorCore => ({ })), restartStamps: [], nextOrdinal: 0, + awaiting: [], }) const withCore = (state: SupervisorState, core: SupervisorCore): SupervisorState => diff --git a/packages/effect-daemon-spec/src/kernel/interpret-supervision-event.workflow.ts b/packages/effect-daemon-spec/src/kernel/interpret-supervision-event.workflow.ts index 9db39d1..abf24fd 100644 --- a/packages/effect-daemon-spec/src/kernel/interpret-supervision-event.workflow.ts +++ b/packages/effect-daemon-spec/src/kernel/interpret-supervision-event.workflow.ts @@ -131,6 +131,61 @@ type CommandBuckets = { const commandsIn = (buckets: CommandBuckets): SupervisorCommands => ({ ...noCommands, ...buckets }) +const appendCommands = (first: SupervisorCommands, second: SupervisorCommands): SupervisorCommands => ({ + stops: [...first.stops, ...second.stops], + starts: [...first.starts, ...second.starts], + arms: [...first.arms, ...second.arms], + replies: [...first.replies, ...second.replies], + terminates: [...first.terminates, ...second.terminates], +}) + +const startTimeoutOf = (core: SupervisorCore, start: ChildStart): PositiveMillis => + kindOfPolicy(core.policy, start.childId).startTimeoutMillis + +const startIsLive = (core: SupervisorCore, start: ChildStart): boolean => + Option.isSome(currentIncarnation(core, start.childId, start.generation)) + +const predecessorsReady = (core: SupervisorCore, start: ChildStart): boolean => + Option.match(indexOfChild(core, start.childId), { + onNone: () => false, + onSome: (index) => Arr.every(Arr.take(core.children, index), (child) => child.status === 'ready'), + }) + +const startIssues = (core: SupervisorCore, start: ChildStart): boolean => + [startIsLive(core, start), predecessorsReady(core, start)].every((holds) => holds) + +const enqueueStarts = ( + awaiting: ReadonlyArray, + starts: ReadonlyArray, +): ReadonlyArray => [ + ...Arr.filter(awaiting, (queued) => Arr.every(starts, (start) => start.childId !== queued.childId)), + ...starts, +] + +const drainAwaitingOf = ( + core: SupervisorCore, + at: EventTime, +): { readonly core: SupervisorCore; readonly commands: SupervisorCommands } => { + const issuable = Arr.filter(core.awaiting, (start) => startIssues(core, start)) + const remaining = Arr.filter(core.awaiting, (start) => !Arr.contains(issuable, start)) + return { + core: { ...core, awaiting: remaining }, + commands: commandsIn({ + starts: Arr.map(issuable, startCommandOf), + arms: Arr.map(issuable, (start) => deadlineCommandOf(start, at + startTimeoutOf(core, start))), + }), + } +} + +const drainedDecisionOf = ( + core: SupervisorCore, + at: EventTime, + commands: SupervisorCommands, +): SupervisionDecision => { + const drained = drainAwaitingOf(core, at) + return new Continue({ core: drained.core, commands: appendCommands(commands, drained.commands) }) +} + const restartsOn = (restartType: RestartType, reason: TerminationReason): boolean => Match.value(reason._tag).pipe( Match.when('Abnormal', () => restartType !== 'temporary'), @@ -235,6 +290,22 @@ const readiedOf = (child: ChildInstance): ChildInstance => ({ probeFailures: 0, }) +const acquiredOf = (child: ChildInstance): ChildInstance => ({ + childId: child.childId, + generation: child.generation, + status: 'starting', + consecutiveRestarts: child.consecutiveRestarts, + probeFailures: 0, +}) + +const standbyOf = (child: ChildInstance): ChildInstance => ({ + childId: child.childId, + generation: child.generation, + status: 'standby', + consecutiveRestarts: child.consecutiveRestarts, + probeFailures: child.probeFailures, +}) + const probedAs = (child: ChildInstance, alive: boolean): ChildInstance => Match.value(alive).pipe( Match.when(true, () => ({ ...child, probeFailures: 0 })), @@ -461,24 +532,21 @@ const phasedCommands = ( core: SupervisorCore, at: EventTime, coming: ReadonlyArray, -): { readonly commands: SupervisorCommands; readonly pending: ReadonlyArray } => { +): { + readonly issuable: ReadonlyArray + readonly pending: ReadonlyArray + readonly arms: ReadonlyArray +} => { const steps: ReadonlyArray = Arr.map(coming, (child) => { const start: ChildStart = { childId: child.childId, generation: child.generation + 1 } return { start, delay: backoffDelay(core.policy.backoff, child.consecutiveRestarts) } }) const ready = Arr.filter(steps, (step: RestartStep) => step.delay === 0) const deferred = Arr.filter(steps, (step: RestartStep) => step.delay > 0) - const timeoutOf = (step: RestartStep): EventTime => - at + kindOfPolicy(core.policy, step.start.childId).startTimeoutMillis return { - commands: commandsIn({ - starts: Arr.map(ready, (step: RestartStep) => startCommandOf(step.start)), - arms: Arr.appendAll( - Arr.map(ready, (step: RestartStep) => deadlineCommandOf(step.start, timeoutOf(step))), - Arr.map(deferred, (step: RestartStep) => backoffCommandOf(step.start, at + step.delay)), - ), - }), + issuable: Arr.map(ready, (step: RestartStep) => step.start), pending: Arr.map(deferred, (step: RestartStep) => step.start), + arms: Arr.map(deferred, (step: RestartStep) => backoffCommandOf(step.start, at + step.delay)), } } @@ -517,11 +585,16 @@ type RestartOutcome = { const restartOutcomeOf = (core: SupervisorCore, scope: RestartScope, at: EventTime): RestartOutcome => { const nextCore = stampedCoreOf(core, scope, at) const phased = phasedCommands(core, at, scope.coming) + const queued: SupervisorCore = { + ...nextCore, + awaiting: enqueueStarts(nextCore.awaiting, phased.issuable), + } + const drained = drainAwaitingOf(queued, at) return { - core: nextCore, + core: drained.core, stops: scope.stops, pending: phased.pending, - commands: commandsIn({ stops: scope.stops, starts: phased.commands.starts, arms: phased.commands.arms }), + commands: appendCommands(commandsIn({ stops: scope.stops, arms: phased.arms }), drained.commands), exhausted: Arr.length(nextCore.restartStamps) > nextCore.policy.intensity, } } @@ -568,10 +641,11 @@ const shutdownOrContinue = ( core: SupervisorCore, removed: ReadonlyArray, remaining: ReadonlyArray, + at: EventTime, ): SupervisionDecision => Match.value(autoShutsDown(core.policy, removed, remaining)).pipe( Match.when(true, () => stopEverything(withChildren(core, remaining), terminatedReason, requestedExit)), - Match.when(false, () => new Continue({ core: withChildren(core, remaining), commands: noCommands })), + Match.when(false, () => drainedDecisionOf(withChildren(core, remaining), at, noCommands)), Match.exhaustive, ) const RestartedBase = Schema.TaggedStruct('Restarted', { @@ -638,6 +712,7 @@ const terminatedIndexed = (core: SupervisorCore, failedIndex: number, terminatio core, Arr.filter(core.children, (child) => Arr.contains(quiet.removedIds, child.childId)), quiet.remaining, + termination.at, )), Match.exhaustive, ) @@ -660,6 +735,7 @@ const terminatedCurrent = ( ): SupervisionDecision => Match.value(child.status).pipe( Match.when('starting', () => terminatedIfIndexed(core, child, termination)), + Match.when('standby', () => terminatedIfIndexed(core, child, termination)), Match.when('ready', () => terminatedIfIndexed(core, child, termination)), Match.when('stopping', () => removalContinue(core, termination)), Match.exhaustive, @@ -669,7 +745,7 @@ const removalOf = (core: SupervisorCore, childId: ChildId, generation: Generatio withChildren(core, Arr.filter(core.children, (child) => incarnationMatches(childId, generation)(child) === false)) const removalContinue = (core: SupervisorCore, termination: Termination): SupervisionDecision => - new Continue({ core: removalOf(core, termination.childId, termination.generation), commands: noCommands }) + drainedDecisionOf(removalOf(core, termination.childId, termination.generation), termination.at, noCommands) const terminatedRunning = (core: SupervisorCore, event: { readonly childId: ChildId @@ -709,19 +785,23 @@ const readyInRunning = ( const becomingReady = (core: SupervisorCore, child: ChildInstance, at: EventTime): SupervisionDecision => Match.value(child.status).pipe( - Match.when('starting', () => { - const next = readiedOf(child) - const nextCore = withChildren(core, Arr.map(core.children, replacing(next))) - return new Continue({ - core: nextCore, - commands: commandsIn({ arms: [tickCommandOf(next, at + core.policy.livenessTickMillis)] }), - }) - }), + Match.when('starting', () => readiedDecisionOf(core, child, at)), + Match.when('standby', () => readiedDecisionOf(core, child, at)), Match.when('ready', () => holdOf(core)), Match.when('stopping', () => holdOf(core)), Match.exhaustive, ) +const readiedDecisionOf = (core: SupervisorCore, child: ChildInstance, at: EventTime): SupervisionDecision => { + const next = readiedOf(child) + const nextCore = withChildren(core, Arr.map(core.children, replacing(next))) + return drainedDecisionOf( + nextCore, + at, + commandsIn({ arms: [tickCommandOf(next, at + core.policy.livenessTickMillis)] }), + ) +} + const stoppedInShuttingDown = ( core: SupervisorCore, reason: TerminationReason, @@ -759,6 +839,7 @@ const probedOutcomeOf = ( Match.value(child.status).pipe( Match.when('ready', () => probedReadyOutcomeOf(core, child, alive, at)), Match.when('starting', () => holdOf(core)), + Match.when('standby', () => holdOf(core)), Match.when('stopping', () => holdOf(core)), Match.exhaustive, ) @@ -828,6 +909,7 @@ const deadlineInRunning = (core: SupervisorCore, child: ChildInstance, at: Event at, })), Match.when('ready', () => new Stale({})), + Match.when('standby', () => new Stale({})), Match.when('stopping', () => new Stale({})), Match.exhaustive, ) @@ -843,6 +925,7 @@ const tickInRunning = (core: SupervisorCore, child: ChildInstance, at: EventTime }), ), Match.when('starting', () => new Stale({})), + Match.when('standby', () => new Stale({})), Match.when('stopping', () => new Stale({})), Match.exhaustive, ) @@ -869,13 +952,14 @@ const fireBackoffOf = ( ): SupervisionDecision => { const start: ChildStart = { childId, generation } const remaining = Arr.filter(pending, (plan) => planMatches(childId, generation)(plan) === false) - const commands: SupervisorCommands = commandsIn({ - starts: [startCommandOf(start)], - arms: [deadlineCommandOf(start, at + kindOfPolicy(core.policy, childId).startTimeoutMillis)], - }) + const queued: SupervisorCore = { ...core, awaiting: enqueueStarts(core.awaiting, [start]) } + const drained = drainAwaitingOf(queued, at) return Match.value(Arr.length(remaining) === 0).pipe( - Match.when(true, () => new StartChildren({ core, commands })), - Match.when(false, () => new RestartChildren({ core, pending: remaining, commands })), + Match.when(true, () => new StartChildren({ core: drained.core, commands: drained.commands })), + Match.when( + false, + () => new RestartChildren({ core: drained.core, pending: remaining, commands: drained.commands }), + ), Match.exhaustive, ) } @@ -901,35 +985,25 @@ const backoffInRestarting = ( Match.exhaustive, ) -const cooledStartsOf = (core: SupervisorCore, at: EventTime): { - readonly fresh: ReadonlyArray - readonly commands: SupervisorCommands -} => { +const cooledRestartOf = (core: SupervisorCore, at: EventTime): SupervisionDecision => { const fresh: ReadonlyArray = Arr.map(core.children, (child) => ({ ...child, status: 'starting', consecutiveRestarts: 0, probeFailures: 0, })) - const deadlineOf = (child: ChildInstance): ArmChildTimer => - deadlineCommandOf( - { childId: child.childId, generation: child.generation }, - at + kindOfPolicy(core.policy, child.childId).startTimeoutMillis, - ) - return { - fresh, - commands: commandsIn({ - starts: Arr.map(fresh, (child) => startCommandOf({ childId: child.childId, generation: child.generation })), - arms: Arr.map(fresh, deadlineOf), - }), - } -} - -const cooledRestartOf = (core: SupervisorCore, at: EventTime): SupervisionDecision => { - const starts = cooledStartsOf(core, at) const clearedStamps: ReadonlyArray = [] - const restarted: SupervisorCore = { ...core, children: starts.fresh, restartStamps: clearedStamps } - return new StartChildren({ core: restarted, commands: starts.commands }) + const queued: SupervisorCore = { + ...core, + children: fresh, + restartStamps: clearedStamps, + awaiting: enqueueStarts( + core.awaiting, + Arr.map(fresh, (child) => ({ childId: child.childId, generation: child.generation })), + ), + } + const drained = drainAwaitingOf(queued, at) + return new StartChildren({ core: drained.core, commands: drained.commands }) } const runningDynamicCount = (core: SupervisorCore): number => @@ -1192,22 +1266,15 @@ const onShutdownRequested = (state: SupervisorState, reason: TerminationReason): ) const bootedChildrenOf = (core: SupervisorCore, at: EventTime): SupervisionDecision => { - const incarnationOf = (child: ChildInstance): ChildStart => ({ - childId: child.childId, - generation: child.generation, - }) - const deadlineOf = (child: ChildInstance): ArmChildTimer => - deadlineCommandOf( - incarnationOf(child), - at + kindOfPolicy(core.policy, child.childId).startTimeoutMillis, - ) - return new StartChildren({ - core, - commands: commandsIn({ - starts: Arr.map(core.children, (child) => startCommandOf(incarnationOf(child))), - arms: Arr.map(core.children, deadlineOf), - }), - }) + const queued: SupervisorCore = { + ...core, + awaiting: enqueueStarts( + core.awaiting, + Arr.map(core.children, (child) => ({ childId: child.childId, generation: child.generation })), + ), + } + const drained = drainAwaitingOf(queued, at) + return new StartChildren({ core: drained.core, commands: drained.commands }) } const onSupervisorStarted = (state: SupervisorState, at: EventTime): SupervisionDecision => @@ -1220,11 +1287,124 @@ const onSupervisorStarted = (state: SupervisorState, at: EventTime): Supervision Match.exhaustive, ) +const standbyOfChild = (core: SupervisorCore, child: ChildInstance): SupervisionDecision => + Match.value(child.status).pipe( + Match.when('starting', () => + new Continue({ + core: withChildren(core, Arr.map(core.children, replacing(standbyOf(child)))), + commands: noCommands, + })), + Match.when('standby', () => holdOf(core)), + Match.when('ready', () => holdOf(core)), + Match.when('stopping', () => holdOf(core)), + Match.exhaustive, + ) + +const standbyInRunning = (core: SupervisorCore, childId: ChildId, generation: Generation): SupervisionDecision => + Match.value(currentIncarnation(core, childId, generation)).pipe( + Match.tag('None', () => new Stale({})), + Match.tag('Some', ({ value }) => standbyOfChild(core, value)), + Match.exhaustive, + ) + +const acquiredOfChild = (core: SupervisorCore, child: ChildInstance, at: EventTime): SupervisionDecision => + Match.value(child.status).pipe( + Match.when('standby', () => + new Continue({ + core: withChildren(core, Arr.map(core.children, replacing(acquiredOf(child)))), + commands: commandsIn({ + arms: [ + deadlineCommandOf( + { childId: child.childId, generation: child.generation }, + at + startTimeoutOf(core, { childId: child.childId, generation: child.generation }), + ), + ], + }), + })), + Match.when('starting', () => holdOf(core)), + Match.when('ready', () => holdOf(core)), + Match.when('stopping', () => holdOf(core)), + Match.exhaustive, + ) + +const acquiredInRunning = ( + core: SupervisorCore, + childId: ChildId, + generation: Generation, + at: EventTime, +): SupervisionDecision => + Match.value(currentIncarnation(core, childId, generation)).pipe( + Match.tag('None', () => new Stale({})), + Match.tag('Some', ({ value }) => acquiredOfChild(core, value, at)), + Match.exhaustive, + ) + +const releasedDecisionOf = (core: SupervisorCore, child: ChildInstance): SupervisionDecision => + new Continue({ + core: withChildren(core, Arr.map(core.children, replacing(standbyOf(child)))), + commands: noCommands, + }) + +const releasedOfChild = (core: SupervisorCore, child: ChildInstance): SupervisionDecision => + Match.value(child.status).pipe( + Match.when('starting', () => releasedDecisionOf(core, child)), + Match.when('ready', () => releasedDecisionOf(core, child)), + Match.when('standby', () => holdOf(core)), + Match.when('stopping', () => holdOf(core)), + Match.exhaustive, + ) + +const releasedInRunning = (core: SupervisorCore, childId: ChildId, generation: Generation): SupervisionDecision => + Match.value(currentIncarnation(core, childId, generation)).pipe( + Match.tag('None', () => new Stale({})), + Match.tag('Some', ({ value }) => releasedOfChild(core, value)), + Match.exhaustive, + ) + +const holdInOtherPhases = (state: SupervisorState): SupervisionDecision => + Match.value(state).pipe( + Match.tag('Running', ({ core }) => holdOf(core)), + Match.tag('Restarting', ({ core }) => holdOf(core)), + Match.tag('CoolingDown', ({ core }) => holdOf(core)), + Match.tag('ShuttingDown', ({ core }) => holdOf(core)), + Match.tag('Terminated', () => new Stale({})), + Match.exhaustive, + ) + +const onChildStandby = (state: SupervisorState, childId: ChildId, generation: Generation): SupervisionDecision => + Match.value(state).pipe( + Match.tag('Running', ({ core }) => standbyInRunning(core, childId, generation)), + Match.orElse(() => holdInOtherPhases(state)), + ) + +const onChildAcquired = ( + state: SupervisorState, + childId: ChildId, + generation: Generation, + at: EventTime, +): SupervisionDecision => + Match.value(state).pipe( + Match.tag('Running', ({ core }) => acquiredInRunning(core, childId, generation, at)), + Match.orElse(() => holdInOtherPhases(state)), + ) + +const onChildReleased = (state: SupervisorState, childId: ChildId, generation: Generation): SupervisionDecision => + Match.value(state).pipe( + Match.tag('Running', ({ core }) => releasedInRunning(core, childId, generation)), + Match.orElse(() => holdInOtherPhases(state)), + ) + const decided = (state: SupervisorState, event: SupervisionEvent): SupervisionDecision => Match.value(event).pipe( Match.tag('SupervisorStarted', (started) => onSupervisorStarted(state, started.at)), Match.tag('ChildStarted', (started) => onChildStarted(state, started.childId, started.generation)), Match.tag('ChildReady', (ready) => onChildReady(state, ready.childId, ready.generation, ready.at)), + Match.tag('ChildStandby', (standby) => onChildStandby(state, standby.childId, standby.generation)), + Match.tag( + 'ChildAcquired', + (acquired) => onChildAcquired(state, acquired.childId, acquired.generation, acquired.at), + ), + Match.tag('ChildReleased', (released) => onChildReleased(state, released.childId, released.generation)), Match.tag('ChildTerminated', (terminated) => onChildTerminated( state, diff --git a/packages/effect-daemon-spec/tests/supervisor-stale-reports.integration.test.ts b/packages/effect-daemon-spec/tests/supervisor-stale-reports.integration.test.ts index ac4b9a5..de95edf 100644 --- a/packages/effect-daemon-spec/tests/supervisor-stale-reports.integration.test.ts +++ b/packages/effect-daemon-spec/tests/supervisor-stale-reports.integration.test.ts @@ -2,7 +2,7 @@ import { Supervisor } from '@systemfsoftware/effect-daemon-spec' import { Gherkin, Given, it, makeFeature, Then, When } from '@systemfsoftware/effect-gherkin-spec' import { Array as Arr, Effect, Match, Queue } from 'effect' import { fiberMediumLayer } from './__fixtures__/FiberMediumHarness.js' -import { crashingChild, settled, traceUntil } from './__fixtures__/SupervisorHarness.js' +import { crashingChild, neverChild, settled, traceUntil } from './__fixtures__/SupervisorHarness.js' const Feature = makeFeature({ it }) @@ -39,37 +39,36 @@ const terminationReportDecisions = (childId: string) => (trace: Trace) => Arr.map(Arr.filter(trace, reportsTerminationOf(childId)), (entry) => entry.decision) const oneForAllPair = Effect.gen(function*() { - const firstCrash = yield* Queue.unbounded() - const secondCrash = yield* Queue.unbounded() + const crash = yield* Queue.unbounded() const supervisor = yield* Supervisor.make('one-for-all-pair').pipe( Supervisor.strategy('one_for_all'), Supervisor.children([ - Supervisor.ChildSpecs.make('a', crashingChild(firstCrash)), - Supervisor.ChildSpecs.make('b', crashingChild(secondCrash)), + Supervisor.ChildSpecs.make('a', neverChild), + Supervisor.ChildSpecs.make('b', crashingChild(crash)), ]), ).scoped - return { firstCrash, supervisor } + return { crash, supervisor } }) Feature('Reports from a child the supervisor has already replaced') .withLayer(fiberMediumLayer) .body(({ scenario }) => { scenario( - "A one-for-all restart leaves the replaced incarnation's report without effect", + 'A one-for-all restart leaves the replaced first incarnation’s report without effect', Gherkin.Do.pipe( - Given('a one-for-all supervisor running two children, each of which crashes when signalled')( + Given('a one-for-all supervisor whose first child runs on and whose second crashes when signalled')( 'tree', () => oneForAllPair, ), - When('the first child crashes')('after', ({ tree }) => + When('the later child crashes')('after', ({ tree }) => Effect.gen(function*() { const watching = yield* traceUntil( tree.supervisor, (trace) => - terminationReportSeen('b')(trace) && + terminationReportSeen('a')(trace) && Arr.every(['a', 'b'], (childId) => readyAtGeneration(1)(childId)(trace)), ) - yield* Queue.offer(tree.firstCrash, void 0) + yield* Queue.offer(tree.crash, void 0) const trace = yield* settled(watching) yield* Effect.yieldNow const incarnations = yield* runningIncarnations(tree.supervisor) @@ -78,7 +77,7 @@ Feature('Reports from a child the supervisor has already replaced') Then("the replaced incarnation's report has no effect and both children run in their new incarnation")( ({ after }, expect) => expect({ - decisions: terminationReportDecisions('b')(after.trace), + decisions: terminationReportDecisions('a')(after.trace), incarnations: after.incarnations, }).toEqual({ decisions: [{ _tag: 'Stale' }], From c6791454753cd7db4eb9302592e443e6bab9f86b Mon Sep 17 00:00:00 2001 From: systemfsoftware-bot <318394648+systemfsoftware-bot@users.noreply.github.com> Date: Thu, 19 Nov 2026 13:36:21 -0500 Subject: [PATCH 2/3] fix(effect-daemon-cluster): address review findings on standby lifecycle Review fixes on top of the standby-readiness change: - P0-1: ChildStandby/ChildAcquired/ChildReleased now apply to the core in every live supervisor phase (Restarting keeps its pending backoff plans, CoolingDown its window), so an acquisition during a group backoff is never dropped. - P0-2: child timer targets carry an acquisition epoch; a start deadline or liveness tick armed before a release cannot fire against the re-acquisition, and duplicate ticks after re-promotion are stale. - P0-3: children with an activity stream report readiness only through that ordered stream (the readiness watcher is not forked for them), and ChildReady no longer promotes a standby child, so a release processed ahead of an in-flight ready cannot mark a non-owner ready. - P1-1: one_for_one restarts issue directly instead of gating on earlier children, so an independent sibling restarts immediately while a standby singleton waits. - P1-2: the medium no longer pre-checks shard ownership at registration (a TOCTOU race); every singleton declares standby and emits Acquired at the start of every run. - P1-3: awaiting drops entries whose incarnation is gone, and dynamic starts route through the same enqueue/drain gating. - P1-4: stopping a singleton shuts its activity queue down so the watcher fiber ends with the incarnation. Counterexample tests cover each case: acquisition during a group backoff, a re-acquisition crossing the pre-release deadline, interruption before readiness, an ungated one_for_one restart, owned-at-registration standby declaration, dynamic-start gating, plus kernel property laws for phase-preserving transitions, epoch-matched timers, ready-while-standby rejection, ungated restarts, and dead-entry pruning. The conformance kit projects standby/acquired/released pacing and stale-timer no-ops out of the compared trace. --- .../src/ClusterMedium/medium.ts | 48 +- ...ster-standby-lifecycle.integration.test.ts | 419 ++++++++++++++++-- .../src/Trace.schema.ts | 3 - .../src/observed-trace.ts | 53 +-- .../src/Supervisor/supervisor-commands.ts | 15 +- ...upervision-event.workflow.property.test.ts | 3 +- ...ndby-acquisition.workflow.property.test.ts | 203 +++++++-- .../src/kernel/SupervisionEvent.schema.ts | 8 +- .../src/kernel/SupervisionLimits.schema.ts | 3 + .../src/kernel/SupervisorCommand.schema.ts | 3 +- .../src/kernel/SupervisorState.schema.ts | 3 + .../interpret-supervision-event.workflow.ts | 169 ++++--- 12 files changed, 749 insertions(+), 181 deletions(-) diff --git a/packages/effect-daemon-cluster/src/ClusterMedium/medium.ts b/packages/effect-daemon-cluster/src/ClusterMedium/medium.ts index bff8e25..894916a 100644 --- a/packages/effect-daemon-cluster/src/ClusterMedium/medium.ts +++ b/packages/effect-daemon-cluster/src/ClusterMedium/medium.ts @@ -14,7 +14,7 @@ import { Scope, Stream, } from 'effect' -import { EntityId, Sharding } from 'effect/cluster' +import { Sharding } from 'effect/cluster' import { absurd } from 'effect/Function' import type { ClusterProgram, ClusterProgramRequirements, EntityChild, SingletonChild } from './ClusterProgram.js' @@ -36,8 +36,17 @@ interface ClusterStarted extends Supervisor.Medium.Started { readonly ended: Deferred.Deferred readonly stopping: Ref.Ref readonly probe: Effect.Effect + readonly activityQueue?: Queue.Queue | undefined } +const activityExtras = ( + activity?: Supervisor.Medium.StartedActivity, + activityQueue?: Queue.Queue, +): { + readonly activity?: Supervisor.Medium.StartedActivity + readonly activityQueue?: Queue.Queue | undefined +} => activity === undefined ? {} : { activity, activityQueue } + const clusterStarted = ( ready: Effect.Effect, registration: Scope.Closeable, @@ -45,6 +54,7 @@ const clusterStarted = ( stopping: Ref.Ref, probe: Effect.Effect, activity?: Supervisor.Medium.StartedActivity, + activityQueue?: Queue.Queue, ): ClusterStarted => ({ ...Supervisor.Medium.started(ready), [ClusterStartedTypeId]: ClusterStartedTypeId, @@ -52,7 +62,7 @@ const clusterStarted = ( ended, stopping, probe, - ...(activity === undefined ? {} : { activity }), + ...activityExtras(activity, activityQueue), }) const isClusterStarted = (evidence: Supervisor.Medium.Started): evidence is ClusterStarted => @@ -94,22 +104,20 @@ const reportedOnExit = ( lostItsShard(stopped, exit) ? Effect.void : Effect.asVoid(Deferred.succeed(ended, terminationOf(stopped, exit))), ) -const ownsSingleton = (sharding: Sharding.Sharding['Service'], name: string, shardGroup: string | undefined) => - sharding.hasShardId(sharding.getShardId(EntityId.make(name), shardGroup ?? 'default')) - const announceAcquire = (changes: Queue.Queue): Effect.Effect => Queue.offer(changes, { _tag: 'Acquired' }).pipe(Effect.asVoid) -const announcesAcquire = (initiallyStandby: boolean, seen: number): boolean => initiallyStandby || seen > 0 - const singletonReady = ( signalled: Deferred.Deferred, changes: Queue.Queue, ): Effect.Effect => - Effect.flatMap(Deferred.isDone(signalled), (alreadySignalled) => - alreadySignalled - ? Queue.offer(changes, { _tag: 'Ready' }).pipe(Effect.asVoid) - : Effect.asVoid(Deferred.succeed(signalled, void 0))) + Effect.flatMap( + Deferred.isDone(signalled), + (alreadySignalled) => alreadySignalled ? Effect.void : Effect.asVoid(Deferred.succeed(signalled, void 0)), + ).pipe( + Effect.andThen(Queue.offer(changes, { _tag: 'Ready' })), + Effect.asVoid, + ) const singletonRun = ( program: SingletonChild, @@ -117,14 +125,9 @@ const singletonRun = ( ended: Deferred.Deferred, stopping: Ref.Ref, changes: Queue.Queue, - initiallyStandby: boolean, - runs: Ref.Ref, ): Effect.Effect => Effect.gen(function*() { - const seen = yield* Ref.getAndUpdate(runs, (count) => count + 1) - if (announcesAcquire(initiallyStandby, seen)) { - yield* announceAcquire(changes) - } + yield* announceAcquire(changes) yield* program.run(singletonReady(signalled, changes)) }).pipe( Effect.onExit((exit) => @@ -147,9 +150,7 @@ const startSingleton = ( const stopping = yield* Ref.make(false) const sharding = yield* Sharding.Sharding const changes = yield* Queue.unbounded() - const runs = yield* Ref.make(0) - const initiallyStandby = ownsSingleton(sharding, program.name, shardGroup) === false - const run = singletonRun(program, signalled, ended, stopping, changes, initiallyStandby, runs) + const run = singletonRun(program, signalled, ended, stopping, changes) yield* sharding.registerSingleton(program.name, run, { shardGroup }).pipe( Effect.provideService(Scope.Scope, registration), ) @@ -160,7 +161,8 @@ const startSingleton = ( ended, stopping, probe, - { initiallyStandby, changes: Stream.fromQueue(changes) }, + { initiallyStandby: true, changes: Stream.fromQueue(changes) }, + changes, ) }) @@ -213,6 +215,9 @@ const probeOf = ( onSome: (self) => self.probe, }) +const shutdownActivityOf = (self: ClusterStarted): Effect.Effect => + self.activityQueue === undefined ? Effect.void : Queue.shutdown(self.activityQueue).pipe(Effect.ignore) + const stopOf = ( evidence: Supervisor.Medium.Started, ): Effect.Effect => @@ -221,6 +226,7 @@ const stopOf = ( onSome: (self) => Ref.set(self.stopping, true).pipe( Effect.andThen(Scope.close(self.registration, Exit.void)), + Effect.andThen(shutdownActivityOf(self)), Effect.andThen(Deferred.succeed(self.ended, shutdownTermination)), Effect.as(Supervisor.Medium.stopped), ), diff --git a/packages/effect-daemon-cluster/tests/cluster-standby-lifecycle.integration.test.ts b/packages/effect-daemon-cluster/tests/cluster-standby-lifecycle.integration.test.ts index b7e0905..229d91e 100644 --- a/packages/effect-daemon-cluster/tests/cluster-standby-lifecycle.integration.test.ts +++ b/packages/effect-daemon-cluster/tests/cluster-standby-lifecycle.integration.test.ts @@ -1,7 +1,7 @@ import { ClusterMedium } from '@systemfsoftware/effect-daemon-cluster' import { Supervisor } from '@systemfsoftware/effect-daemon-spec' import { Gherkin, Given, it, makeFeature, Then, When } from '@systemfsoftware/effect-gherkin-spec' -import { Array as Arr, Effect, Layer, Match, Ref, Stream } from 'effect' +import { Array as Arr, Duration, Effect, Layer, Match, Queue, Ref, Stream } from 'effect' import { RegistrationLedger, scriptedShardingOwning, ShardScript } from './__fixtures__/scripted-sharding.fixture.js' const Feature = makeFeature({ it }) @@ -24,6 +24,247 @@ const heldNames: Effect.Effect, never, RegistrationLedger> Effect.map(Effect.service(RegistrationLedger), (ledger) => ledger.held), ) +const readyAfter = (millis: Duration.Input, ready: Effect.Effect): Effect.Effect => + Effect.sleep(millis).pipe(Effect.andThen(ready), Effect.asVoid) + +const traceCollector = (handle: Supervisor.RunningSupervisor) => + Effect.map(Ref.make>([]), (trace) => ({ + trace, + collect: Stream.runForEach(Supervisor.traceOf(handle), (entry) => Ref.update(trace, (seen) => [...seen, entry])) + .pipe(Effect.forkScoped, Effect.asVoid), + })) + +const eventLabel = (entry: Supervisor.TraceEntry): string => + Match.value(entry.event).pipe( + Match.tag('ChildStarted', (started) => `started:${started.childId}:${started.generation}`), + Match.tag('ChildStandby', (standby) => `standby:${standby.childId}:${standby.generation}`), + Match.tag('ChildAcquired', (acquired) => `acquired:${acquired.childId}:${acquired.generation}`), + Match.tag('ChildReady', (ready) => `ready:${ready.childId}:${ready.generation}`), + Match.tag('ChildReleased', (released) => `released:${released.childId}:${released.generation}`), + Match.tag('ChildTerminated', (terminated) => `terminated:${terminated.childId}:${terminated.generation}`), + Match.orElse(() => 'other'), + ) + +const countOf = (labels: ReadonlyArray, prefix: string): number => + labels.filter((label) => label.startsWith(prefix)).length + +const firstIndexOf = (labels: ReadonlyArray, wanted: string): number => labels.indexOf(wanted) + +const reacquiredTimerIsNotKilled = Effect.gen(function*() { + const shards = yield* ShardScript + const singleton = ClusterMedium.singletonChild({ + name: 'orders/epoch', + run: (ready) => + Effect.gen(function*() { + yield* readyAfter('250 millis', ready) + return yield* Effect.never + }), + }) + const handle = yield* Supervisor.make('epoch-tree').pipe( + Supervisor.strategy('one_for_one'), + Supervisor.intensity(3, 60_000), + Supervisor.children([ + Supervisor.ChildSpecs.on(ClusterMedium.port)('epoch', singleton, { startTimeoutMillis: 400 }), + ]), + ).scoped + const collector = yield* traceCollector(handle) + yield* collector.collect + yield* Effect.sleep('50 millis') + yield* shards.grant('orders/epoch') + yield* Effect.sleep('300 millis') + yield* shards.revoke('orders/epoch') + yield* Effect.sleep('50 millis') + yield* shards.grant('orders/epoch') + yield* Effect.sleep('500 millis') + const statuses = yield* statusesOf(handle) + const labels = yield* Effect.map(Ref.get(collector.trace), (entries) => Arr.map(entries, eventLabel)) + return { statuses, terminated: countOf(labels, 'terminated'), readyEvents: countOf(labels, 'ready:epoch') } +}) + +const interruptedBeforeReadyStaysStandby = Effect.gen(function*() { + const shards = yield* ShardScript + const slow = ClusterMedium.singletonChild({ + name: 'orders/slow', + run: (ready) => + Effect.gen(function*() { + yield* readyAfter('100 millis', ready) + return yield* Effect.never + }), + }) + const watchdog = ClusterMedium.singletonChild({ + name: 'orders/watchdog', + run: (ready) => Effect.andThen(ready, Effect.never), + }) + const handle = yield* Supervisor.make('interrupted-tree').pipe( + Supervisor.strategy('rest_for_one'), + Supervisor.intensity(3, 60_000), + Supervisor.children([ + Supervisor.ChildSpecs.on(ClusterMedium.port)('slow', slow, { startTimeoutMillis: 2_000 }), + Supervisor.ChildSpecs.on(ClusterMedium.port)('watchdog', watchdog, { startTimeoutMillis: 2_000 }), + ]), + ).scoped + const collector = yield* traceCollector(handle) + yield* collector.collect + yield* Effect.sleep('50 millis') + yield* shards.grant('orders/slow') + yield* Effect.sleep('20 millis') + yield* shards.revoke('orders/slow') + yield* Effect.sleep('150 millis') + const statuses = yield* statusesOf(handle) + const held = yield* heldNames + const labels = yield* Effect.map(Ref.get(collector.trace), (entries) => Arr.map(entries, eventLabel)) + return { + statuses, + held, + slowReadyEvents: countOf(labels, 'ready:slow'), + watchdogStarts: countOf(labels, 'started:watchdog'), + terminated: countOf(labels, 'terminated'), + } +}) + +const oneForOneRestartIsUngated = Effect.gen(function*() { + const crashes = yield* Queue.unbounded() + const shards = yield* ShardScript + const gate = ClusterMedium.singletonChild({ + name: 'orders/gate', + run: (ready) => Effect.andThen(ready, Effect.never), + }) + const worker = ClusterMedium.singletonChild({ + name: 'orders/worker', + run: (ready) => + Effect.gen(function*() { + yield* ready + yield* Queue.take(crashes) + return yield* Effect.die('crashed') + }), + }) + const handle = yield* Supervisor.make('ungated-tree').pipe( + Supervisor.strategy('one_for_one'), + Supervisor.intensity(3, 60_000), + Supervisor.children([ + Supervisor.ChildSpecs.on(ClusterMedium.port)('gate', gate, { startTimeoutMillis: 2_000 }), + Supervisor.ChildSpecs.on(ClusterMedium.port)('worker', worker, { startTimeoutMillis: 2_000 }), + ]), + ).scoped + const collector = yield* traceCollector(handle) + yield* collector.collect + yield* shards.grant('orders/gate') + yield* Effect.sleep('50 millis') + yield* shards.grant('orders/worker') + yield* Effect.sleep('100 millis') + yield* shards.revoke('orders/gate') + yield* Effect.sleep('50 millis') + const standbyStatuses = yield* statusesOf(handle) + yield* Queue.offer(crashes, void 0) + yield* Effect.sleep('100 millis') + const restartedStatuses = yield* statusesOf(handle) + const labels = yield* Effect.map(Ref.get(collector.trace), (entries) => Arr.map(entries, eventLabel)) + return { + standbyStatuses, + restartedStatuses, + workerStarts: countOf(labels, 'started:worker'), + terminated: countOf(labels, 'terminated'), + } +}) + +const acquisitionDuringBackoffIsKept = Effect.gen(function*() { + const crashes = yield* Queue.unbounded() + const shards = yield* ShardScript + const leader = ClusterMedium.singletonChild({ + name: 'orders/leader', + run: (ready) => + Effect.gen(function*() { + yield* readyAfter('150 millis', ready) + return yield* Effect.never + }), + }) + const worker = ClusterMedium.singletonChild({ + name: 'orders/worker', + run: (ready) => + Effect.gen(function*() { + yield* ready + yield* Queue.take(crashes) + return yield* Effect.die('crashed') + }), + }) + const handle = yield* Supervisor.make('backoff-tree').pipe( + Supervisor.strategy('one_for_all'), + Supervisor.intensity(3, 60_000), + Supervisor.backoff({ baseMillis: 100, multiplier: 1, capMillis: 100 }), + Supervisor.children([ + Supervisor.ChildSpecs.on(ClusterMedium.port)('leader', leader, { startTimeoutMillis: 2_000 }), + Supervisor.ChildSpecs.on(ClusterMedium.port)('worker', worker, { startTimeoutMillis: 2_000 }), + ]), + ).scoped + const collector = yield* traceCollector(handle) + yield* collector.collect + yield* shards.grant('orders/leader') + yield* shards.grant('orders/worker') + yield* Effect.sleep('300 millis') + yield* Queue.offer(crashes, void 0) + yield* Effect.sleep('500 millis') + const statuses = (yield* statusesOf(handle)).slice().sort((left, right) => left[0] < right[0] ? -1 : 1) + const labels = yield* Effect.map(Ref.get(collector.trace), (entries) => Arr.map(entries, eventLabel)) + return { + statuses, + acquisitions: countOf(labels, 'acquired:leader'), + terminated: countOf(labels, 'terminated'), + } +}) + +const ownedAtRegistrationStillAnnouncesStandby = Effect.gen(function*() { + const { medium } = yield* ClusterMedium.port + const probe = ClusterMedium.singletonChild({ + name: 'orders/probe', + run: (ready) => Effect.andThen(ready, Effect.never), + }) + const evidence = yield* medium.start(probe) + const declaredStandby = evidence.activity?.initiallyStandby === true + yield* evidence.ready + yield* medium.stop(evidence, { _tag: 'Brutal' }) + const singleton = ClusterMedium.singletonChild({ + name: 'orders/owned', + run: (ready) => Effect.andThen(ready, Effect.never), + }) + const handle = yield* Supervisor.make('owned-tree').pipe( + Supervisor.strategy('one_for_one'), + Supervisor.intensity(3, 60_000), + Supervisor.children([ + Supervisor.ChildSpecs.on(ClusterMedium.port)('owned', singleton, { startTimeoutMillis: 2_000 }), + ]), + ).scoped + yield* Effect.sleep('200 millis') + const statuses = yield* statusesOf(handle) + return { declaredStandby, statuses } +}) + +const dynamicStartWaitsForDeclaredReadiness = Effect.gen(function*() { + const shards = yield* ShardScript + const gate = ClusterMedium.singletonChild({ + name: 'orders/dyn-gate', + run: (ready) => Effect.andThen(ready, Effect.never), + }) + const handle = yield* Supervisor.make('dynamic-tree').pipe( + Supervisor.strategy('one_for_one'), + Supervisor.intensity(3, 60_000), + Supervisor.dynamic({ ceiling: 2, restartType: 'temporary' }), + Supervisor.children([ + Supervisor.ChildSpecs.on(ClusterMedium.port)('gate', gate, { startTimeoutMillis: 2_000 }), + ]), + ).scoped + const collector = yield* traceCollector(handle) + yield* collector.collect + yield* Effect.sleep('50 millis') + const reply = yield* Supervisor.startChild(handle, Supervisor.readyOnStart(Effect.never)) + const heldWhileStandby = yield* heldNames + yield* Effect.sleep('50 millis') + yield* shards.grant('orders/dyn-gate') + yield* Effect.sleep('100 millis') + const statuses = yield* statusesOf(handle) + const heldAfterGrant = yield* heldNames + return { reply, heldWhileStandby, statuses, heldAfterGrant } +}) + interface LifecycleState { readonly handle: Supervisor.RunningSupervisor readonly trace: Ref.Ref> @@ -64,39 +305,20 @@ const lifecycleTree = Effect.gen(function*() { Supervisor.ChildSpecs.on(ClusterMedium.port)('dependent', dependent, { startTimeoutMillis: 2_000 }), ]), ).scoped - const trace = yield* Ref.make>([]) - yield* Stream.runForEach(Supervisor.traceOf(handle), (entry) => Ref.update(trace, (seen) => [...seen, entry])).pipe( - Effect.forkScoped, - Effect.asVoid, - ) - return { handle, trace, runs, initialized, dependentSaw } satisfies LifecycleState + const collector = yield* traceCollector(handle) + yield* collector.collect + return { + handle, + trace: collector.trace, + runs, + initialized, + dependentSaw, + } satisfies LifecycleState }) -const eventLabel = (entry: Supervisor.TraceEntry): string => - Match.value(entry.event).pipe( - Match.tag('ChildStarted', (started) => `started:${started.childId}:${started.generation}`), - Match.tag('ChildStandby', (standby) => `standby:${standby.childId}:${standby.generation}`), - Match.tag('ChildAcquired', (acquired) => `acquired:${acquired.childId}:${acquired.generation}`), - Match.tag('ChildReady', (ready) => `ready:${ready.childId}:${ready.generation}`), - Match.tag('ChildReleased', (released) => `released:${released.childId}:${released.generation}`), - Match.tag('ChildTerminated', (terminated) => `terminated:${terminated.childId}:${terminated.generation}`), - Match.orElse(() => 'other'), - ) - const traceLabelsOf = (state: LifecycleState): Effect.Effect> => Effect.map(Ref.get(state.trace), (entries) => Arr.map(entries, eventLabel)) -const countOf = (labels: ReadonlyArray, prefix: string): number => - Arr.filter(labels, (label) => label.startsWith(prefix)).length - -const firstIndexOf = (labels: ReadonlyArray, wanted: string): number => - Arr.findFirstIndex(labels, (label) => label === wanted).pipe( - Match.value, - Match.tag('Some', ({ value }) => value), - Match.tag('None', () => -1), - Match.exhaustive, - ) - const standbyToHandoffAndBack = Effect.gen(function*() { const state = yield* lifecycleTree const shards = yield* ShardScript @@ -172,18 +394,15 @@ const postAcquisitionTimeout = Effect.gen(function*() { Supervisor.ChildSpecs.on(ClusterMedium.port)('hanging', neverReady, { startTimeoutMillis: 200 }), ]), ).scoped - const trace = yield* Ref.make>([]) - yield* Stream.runForEach(Supervisor.traceOf(handle), (entry) => Ref.update(trace, (seen) => [...seen, entry])).pipe( - Effect.forkScoped, - Effect.asVoid, - ) + const collector = yield* traceCollector(handle) + yield* collector.collect yield* Effect.sleep('400 millis') const standbyStatuses = yield* statusesOf(handle) const shards = yield* ShardScript yield* shards.grant('orders/hanging') yield* Effect.sleep('2 seconds') const afterTimeoutStatuses = yield* statusesOf(handle) - const events = yield* Effect.map(Ref.get(trace), timeoutEventsOf) + const events = yield* Effect.map(Ref.get(collector.trace), timeoutEventsOf) const acquiredAt = firstIndexOf(events, 'acquired') const fromAcquisition = acquiredAt === -1 ? events : Arr.drop(events, acquiredAt) return { standbyStatuses, afterTimeoutStatuses, fromAcquisition } @@ -247,4 +466,134 @@ Feature('Standby singletons under shard movement', { timeout: 120_000 }) ), ), ) + + scenario( + 'A re-acquired singleton is not killed by the deadline armed before its shard moved away', + Gherkin.Do.pipe( + Given('a cluster runner that owns no shard yet')( + 'environment', + () => Layer.build(Layer.merge(ClusterMedium.layer(), scriptedShardingOwning(false))), + ), + When('the singleton is promoted, handed off, and promoted again past the first deadline')( + 'observed', + (s) => Effect.provide(reacquiredTimerIsNotKilled, s.environment), + ), + Then('the stale deadline missed nothing and the singleton is ready')((s, expect) => + expect(s.observed).toEqual({ + statuses: [['epoch', 0, 'ready']], + terminated: 0, + readyEvents: 2, + }) + ), + ), + ) + + scenario( + 'A singleton interrupted before it signals ready stays standby and its dependent never starts', + Gherkin.Do.pipe( + Given('a cluster runner that owns no shard yet')( + 'environment', + () => Layer.build(Layer.merge(ClusterMedium.layer(), scriptedShardingOwning(false))), + ), + When('the shard is revoked while the singleton still initializes')( + 'observed', + (s) => Effect.provide(interruptedBeforeReadyStaysStandby, s.environment), + ), + Then('nothing was announced ready, nothing terminated, and no dependent started')((s, expect) => + expect(s.observed).toEqual({ + statuses: [['slow', 0, 'standby'], ['watchdog', 0, 'starting']], + held: ['orders/slow'], + slowReadyEvents: 0, + watchdogStarts: 0, + terminated: 0, + }) + ), + ), + ) + + scenario( + 'An independent one-for-one child restarts immediately while its standby singleton waits', + Gherkin.Do.pipe( + Given('a cluster runner that owns no shard yet')( + 'environment', + () => Layer.build(Layer.merge(ClusterMedium.layer(), scriptedShardingOwning(false))), + ), + When('the singleton hands off and the fiber sibling crashes')( + 'observed', + (s) => Effect.provide(oneForOneRestartIsUngated, s.environment), + ), + Then('the sibling restarted at once, ungated by the standby singleton')((s, expect) => + expect(s.observed).toEqual({ + standbyStatuses: [['gate', 0, 'standby'], ['worker', 0, 'ready']], + restartedStatuses: [['gate', 0, 'standby'], ['worker', 1, 'ready']], + workerStarts: 2, + terminated: 1, + }) + ), + ), + ) + + scenario( + 'A singleton whose shard is already local still announces standby and then acquires', + Gherkin.Do.pipe( + Given('a cluster runner that owns every shard')( + 'environment', + () => Layer.build(Layer.merge(ClusterMedium.layer(), scriptedShardingOwning(true))), + ), + When('a singleton is supervised on that runner')( + 'observed', + (s) => Effect.provide(ownedAtRegistrationStillAnnouncesStandby, s.environment), + ), + Then('the medium declares standby regardless of ownership, and the singleton is ready')( + (s, expect) => + expect(s.observed).toEqual({ + declaredStandby: true, + statuses: [['owned', 0, 'ready']], + }), + ), + ), + ) + + scenario( + 'A singleton acquiring during a group backoff keeps its acquisition and timer', + Gherkin.Do.pipe( + Given('a cluster runner that owns no shard yet')( + 'environment', + () => Layer.build(Layer.merge(ClusterMedium.layer(), scriptedShardingOwning(false))), + ), + When('a group restart is backoff-pending while the singleton re-acquires')( + 'observed', + (s) => Effect.provide(acquisitionDuringBackoffIsKept, s.environment), + ), + Then('the acquisition during backoff was kept and both children came back')((s, expect) => + expect(s.observed).toEqual({ + statuses: [['leader', 1, 'ready'], ['worker', 1, 'ready']], + acquisitions: 2, + terminated: 2, + }) + ), + ), + ) + + scenario( + 'A dynamic child waits for the declared standby singleton before it starts', + Gherkin.Do.pipe( + Given('a cluster runner that owns no shard yet')( + 'environment', + () => Layer.build(Layer.merge(ClusterMedium.layer(), scriptedShardingOwning(false))), + ), + When('a dynamic child is requested while the singleton is standby and the shard then arrives')( + 'observed', + (s) => Effect.provide(dynamicStartWaitsForDeclaredReadiness, s.environment), + ), + Then('the request was accepted but the child only ran once the singleton was ready')((s, expect) => + expect(s.observed).toEqual({ + reply: { outcome: 'accepted', childId: 'd0', generation: 0 }, + heldWhileStandby: ['orders/dyn-gate'], + statuses: [['gate', 0, 'ready'], ['d0', 0, 'ready']], + heldAfterGrant: ['orders/dyn-gate'], + }) + ), + ), + ) }) diff --git a/packages/effect-daemon-conformance/src/Trace.schema.ts b/packages/effect-daemon-conformance/src/Trace.schema.ts index cd97a71..ee6130b 100644 --- a/packages/effect-daemon-conformance/src/Trace.schema.ts +++ b/packages/effect-daemon-conformance/src/Trace.schema.ts @@ -27,9 +27,6 @@ export const EventKind = Schema.Literals([ 'SupervisorStarted', 'ChildStarted', 'ChildReady', - 'ChildStandby', - 'ChildAcquired', - 'ChildReleased', 'ChildTerminated', 'ChildStopped', 'ProbeResult', diff --git a/packages/effect-daemon-conformance/src/observed-trace.ts b/packages/effect-daemon-conformance/src/observed-trace.ts index e1e580c..17611a9 100644 --- a/packages/effect-daemon-conformance/src/observed-trace.ts +++ b/packages/effect-daemon-conformance/src/observed-trace.ts @@ -1,5 +1,5 @@ import type { Supervisor } from '@systemfsoftware/effect-daemon-spec' -import { Array as Arr, Match } from 'effect' +import { Array as Arr, Match, Option } from 'effect' import type { ChildRef, CommandKind, @@ -68,45 +68,45 @@ const timerChildOf = (target: TimerTarget): ChildRef | null => Match.exhaustive, ) -const eventOf = (event: Event): ObservedEvent => +const observedSome = ( + kind: EventKind, + child: ChildRef | null, + reason: ReasonKind | null, +): Option.Option => Option.some(observedEvent(kind, child, reason)) + +const eventOf = (event: Event): Option.Option => Match.value(event).pipe( - Match.tag('SupervisorStarted', () => observedEvent('SupervisorStarted', null, null)), + Match.tag('SupervisorStarted', () => observedSome('SupervisorStarted', null, null)), Match.tag('ChildStarted', (started) => - observedEvent('ChildStarted', childRef(started.childId, started.generation), null)), + observedSome('ChildStarted', childRef(started.childId, started.generation), null)), Match.tag('ChildReady', (ready) => - observedEvent('ChildReady', childRef(ready.childId, ready.generation), null)), - Match.tag('ChildStandby', (standby) => - observedEvent('ChildStandby', childRef(standby.childId, standby.generation), null)), - Match.tag('ChildAcquired', (acquired) => - observedEvent('ChildAcquired', childRef(acquired.childId, acquired.generation), null)), - Match.tag('ChildReleased', (released) => - observedEvent('ChildReleased', childRef(released.childId, released.generation), null)), + observedSome('ChildReady', childRef(ready.childId, ready.generation), null)), + Match.tag('ChildStandby', () => Option.none()), + Match.tag('ChildAcquired', () => Option.none()), + Match.tag('ChildReleased', () => Option.none()), Match.tag( 'ChildTerminated', (terminated) => - observedEvent( + observedSome( 'ChildTerminated', childRef(terminated.childId, terminated.generation), reasonKindOf(terminated.reason), ), ), Match.tag('ChildStopped', (stopped) => - observedEvent('ChildStopped', childRef(stopped.childId, stopped.generation), null)), + observedSome('ChildStopped', childRef(stopped.childId, stopped.generation), null)), Match.tag('ProbeResult', (probe) => - observedEvent('ProbeResult', childRef(probe.childId, probe.generation), null)), + observedSome('ProbeResult', childRef(probe.childId, probe.generation), null)), Match.tag('TimerElapsed', (timer) => - observedEvent('TimerElapsed', timerChildOf(timer.target), null)), - Match.tag('DynamicStartRequested', () => - observedEvent('DynamicStartRequested', null, null)), + observedSome('TimerElapsed', timerChildOf(timer.target), null)), + Match.tag('DynamicStartRequested', () => observedSome('DynamicStartRequested', null, null)), Match.tag( 'DynamicStopRequested', - (stopping) => - observedEvent('DynamicStopRequested', childRef(stopping.childId, stopping.generation), null), + (stopping) => observedSome('DynamicStopRequested', childRef(stopping.childId, stopping.generation), null), ), Match.tag( 'ShutdownRequested', - (shutdown) => - observedEvent('ShutdownRequested', null, reasonKindOf(shutdown.reason)), + (shutdown) => observedSome('ShutdownRequested', null, reasonKindOf(shutdown.reason)), ), Match.exhaustive, ) @@ -146,11 +146,12 @@ const decisionOf = (decision: Decision): ObservedDecision => Match.exhaustive, ) -const stepOf = (entry: TraceEntry): ObservedStep => ({ - event: eventOf(entry.event), - decision: decisionOf(entry.decision), -}) +const stepOf = (entry: TraceEntry): Option.Option => + Option.map(eventOf(entry.event), (event) => ({ event, decision: decisionOf(entry.decision) })) + +const isIgnoredStep = (step: ObservedStep): boolean => + step.event.kind === 'TimerElapsed' && step.decision.kind === 'Stale' /** Reduces a supervisor's kernel trace to the steps a pairwise comparison reads. */ export const observedStepsOf = (entries: ReadonlyArray): ReadonlyArray => - Arr.map(entries, stepOf) + Arr.filter(Arr.flatMap(entries, (entry) => Option.toArray(stepOf(entry))), (step) => isIgnoredStep(step) === false) diff --git a/packages/effect-daemon-spec/src/Supervisor/supervisor-commands.ts b/packages/effect-daemon-spec/src/Supervisor/supervisor-commands.ts index 4ceeac1..7617704 100644 --- a/packages/effect-daemon-spec/src/Supervisor/supervisor-commands.ts +++ b/packages/effect-daemon-spec/src/Supervisor/supervisor-commands.ts @@ -15,7 +15,7 @@ import { Stream, } from 'effect' import type { SupervisionEvent, TimerKind } from '../kernel/SupervisionEvent.schema.js' -import type { ChildId, Generation } from '../kernel/SupervisionLimits.schema.js' +import type { ChildId, Epoch, Generation } from '../kernel/SupervisionLimits.schema.js' import { ReplyStartAccepted } from '../kernel/SupervisorCommand.schema.js' import type { StartChild, @@ -123,11 +123,12 @@ const timerEventOf = ( kind: TimerKind, childId: ChildId, generation: Generation, + epoch: Epoch, ): SupervisionEvent => ({ _tag: 'TimerElapsed', at: 0, kind, - target: { _tag: 'ChildTimer', childId, generation }, + target: { _tag: 'ChildTimer', childId, generation, epoch }, }) const supervisorTimerEventOf = (kind: TimerKind): SupervisionEvent => ({ @@ -159,6 +160,14 @@ const watchReadiness = ( evidence: Started, childId: ChildId, generation: Generation, +): Effect.Effect => + evidence.activity === undefined ? forkReadinessWatch(handle, evidence, childId, generation) : Effect.void + +const forkReadinessWatch = ( + handle: RunningSupervisor, + evidence: Started, + childId: ChildId, + generation: Generation, ): Effect.Effect => Effect.asVoid( Effect.forkIn( @@ -329,7 +338,7 @@ const executeArm = ( ): Effect.Effect => Match.value(arm).pipe( Match.tag('ArmChildTimer', (timer) => - armTimer(acquired, timer.deadline, timerEventOf(timer.kind, timer.childId, timer.generation))), + armTimer(acquired, timer.deadline, timerEventOf(timer.kind, timer.childId, timer.generation, timer.epoch))), Match.tag('ArmSupervisorTimer', (timer) => armTimer(acquired, timer.deadline, supervisorTimerEventOf(timer.kind))), Match.exhaustive, diff --git a/packages/effect-daemon-spec/src/__tests__/interpret-supervision-event.workflow.property.test.ts b/packages/effect-daemon-spec/src/__tests__/interpret-supervision-event.workflow.property.test.ts index 27ced5d..e361fc2 100644 --- a/packages/effect-daemon-spec/src/__tests__/interpret-supervision-event.workflow.property.test.ts +++ b/packages/effect-daemon-spec/src/__tests__/interpret-supervision-event.workflow.property.test.ts @@ -86,6 +86,7 @@ const childOf = (childId: string, generation: number, status: ChildStatus): Chil childId, generation, status, + epoch: 0, consecutiveRestarts: 0, probeFailures: 0, }) @@ -139,7 +140,7 @@ const childTimerEvent = (kind: TimerKind, childId: string, generation: number, a _tag: 'TimerElapsed', at, kind, - target: { _tag: 'ChildTimer', childId, generation }, + target: { _tag: 'ChildTimer', childId, generation, epoch: 0 }, }) const supervisorTimerEvent = (kind: TimerKind, at: EventTime): SupervisionEvent => ({ diff --git a/packages/effect-daemon-spec/src/__tests__/standby-acquisition.workflow.property.test.ts b/packages/effect-daemon-spec/src/__tests__/standby-acquisition.workflow.property.test.ts index acfdbde..1b0a5b6 100644 --- a/packages/effect-daemon-spec/src/__tests__/standby-acquisition.workflow.property.test.ts +++ b/packages/effect-daemon-spec/src/__tests__/standby-acquisition.workflow.property.test.ts @@ -1,6 +1,7 @@ import { it } from '@systemfsoftware/vitest' import { Array as Arr, Match, Result } from 'effect' import { absurd } from 'effect/Function' +import { evolveSupervisor, SupervisionEvolution } from '../kernel/evolve-supervisor.workflow.js' import { interpretSupervisionEvent, type SupervisionDecision, @@ -12,7 +13,7 @@ import type { SupervisorArm } from '../kernel/SupervisorCommand.schema.js' import { SupervisionPolicy } from '../kernel/SupervisorPolicy.schema.js' import type { RestartType } from '../kernel/SupervisorPolicy.schema.js' import type { ChildInstance, SupervisorCore } from '../kernel/SupervisorState.schema.js' -import { Running } from '../kernel/SupervisorState.schema.js' +import { Restarting, Running } from '../kernel/SupervisorState.schema.js' import type { SupervisorState } from '../kernel/SupervisorState.schema.js' import type { TerminationReason } from '../kernel/TerminationReport.schema.js' @@ -23,12 +24,13 @@ const START_TIMEOUT_MILLIS = 50 const LIVENESS_TICK_MILLIS = 10 const policyWith = ( + strategy: 'one_for_one' | 'rest_for_one' | 'one_for_all', childDeclarations: ReadonlyArray<{ readonly childId: string readonly restartType: RestartType }>, ): SupervisionPolicy => ({ - strategy: 'rest_for_one', + strategy, intensity: 8, periodMillis: PERIOD_MILLIS, autoShutdown: 'never', @@ -46,10 +48,16 @@ const policyWith = ( })), }) -const childOf = (childId: string, generation: Generation, status: ChildInstance['status']): ChildInstance => ({ +const childOf = ( + childId: string, + generation: Generation, + status: ChildInstance['status'], + epoch = 0, +): ChildInstance => ({ childId, generation, status, + epoch, consecutiveRestarts: 0, probeFailures: 0, }) @@ -71,11 +79,16 @@ const decidedOf = (interpret: Interpret, step: SupervisionStep): SupervisionDeci const stepOf = (state: SupervisorState, event: SupervisionEvent): SupervisionStep => new SupervisionStep({ state, event }) -const startDeadlineTimerOf = (childId: string, generation: Generation, at: EventTime): SupervisionEvent => ({ +const startDeadlineTimerOf = ( + childId: string, + generation: Generation, + epoch: number, + at: EventTime, +): SupervisionEvent => ({ _tag: 'TimerElapsed', at, kind: 'start_deadline', - target: { _tag: 'ChildTimer', childId, generation }, + target: { _tag: 'ChildTimer', childId, generation, epoch }, }) const acquiredEventOf = (childId: string, generation: Generation, at: EventTime): SupervisionEvent => ({ @@ -106,6 +119,11 @@ const terminatedEventOf = ( reason: TerminationReason, ): SupervisionEvent => ({ _tag: 'ChildTerminated', at, childId, generation, reason }) +const abnormalOf = (cause: string): TerminationReason => ({ + _tag: 'Abnormal', + report: { _tag: 'CauseReport', cause }, +}) + const isStale = (decision: SupervisionDecision): boolean => Match.value(decision).pipe( Match.tag('Stale', () => true), @@ -126,6 +144,20 @@ const statusOf = (decision: SupervisionDecision, childId: string): string | null ), ) +const epochOf = (decision: SupervisionDecision, childId: string): number | null => + Match.value(decision).pipe( + Match.tag('Continue', ({ core }) => + Match.value(Arr.findFirst(core.children, (child) => + child.childId === childId)).pipe( + Match.tag('Some', ({ value }) => value.epoch), + Match.tag('None', () => null), + Match.exhaustive, + )), + Match.orElse(() => + null + ), + ) + const startedIdsOf = (decision: SupervisionDecision): ReadonlyArray => Match.value(decision).pipe( Match.tag('Continue', ({ commands }) => Arr.map(commands.starts, (start) => start.childId)), @@ -133,18 +165,23 @@ const startedIdsOf = (decision: SupervisionDecision): ReadonlyArray => Match.orElse((): ReadonlyArray => []), ) -const isStartDeadlineFor = (arm: SupervisorArm, childId: string, at: EventTime): boolean => +const isStartDeadlineFor = (arm: SupervisorArm, childId: string, epoch: number, at: EventTime): boolean => Match.value(arm).pipe( - Match.tag('ArmChildTimer', (timer) => - timer.kind === 'start_deadline' && timer.childId === childId && timer.deadline === at + START_TIMEOUT_MILLIS), - Match.orElse(() => - false + Match.tag( + 'ArmChildTimer', + (timer) => + timer.kind === 'start_deadline' && timer.childId === childId && timer.epoch === epoch && + timer.deadline === at + START_TIMEOUT_MILLIS, ), + Match.orElse(() => false), ) -const armedDeadlineFor = (decision: SupervisionDecision, childId: string, at: EventTime): boolean => +const armedDeadlineFor = (decision: SupervisionDecision, childId: string, epoch: number, at: EventTime): boolean => Match.value(decision).pipe( - Match.tag('Continue', ({ commands }) => Arr.some(commands.arms, (arm) => isStartDeadlineFor(arm, childId, at))), + Match.tag( + 'Continue', + ({ commands }) => Arr.some(commands.arms, (arm) => isStartDeadlineFor(arm, childId, epoch, at)), + ), Match.orElse(() => false), ) @@ -152,20 +189,21 @@ it.prop( '∀s_StandbyDeadline_=Stale', { of: [EventTime, Generation], subject: interpretSupervisionEvent }, (subject, [at, generation]) => { - const policy = policyWith([{ childId: 'c0', restartType: 'permanent' }]) + const policy = policyWith('rest_for_one', [{ childId: 'c0', restartType: 'permanent' }]) const state = runningOf(coreWith(policy, [childOf('c0', generation, 'standby')], [])) - return isStale(decidedOf(subject, stepOf(state, startDeadlineTimerOf('c0', generation, at)))) + return isStale(decidedOf(subject, stepOf(state, startDeadlineTimerOf('c0', generation, 0, at)))) }, ) it.prop( - '∀a_Acquired_=StartingWithDeadline', + '∀a_Acquired_=StartingWithEpochDeadline', { of: [EventTime, Generation], subject: interpretSupervisionEvent }, (subject, [at, generation]) => { - const policy = policyWith([{ childId: 'c0', restartType: 'permanent' }]) + const policy = policyWith('rest_for_one', [{ childId: 'c0', restartType: 'permanent' }]) const state = runningOf(coreWith(policy, [childOf('c0', generation, 'standby')], [])) const decision = decidedOf(subject, stepOf(state, acquiredEventOf('c0', generation, at))) - return statusOf(decision, 'c0') === 'starting' && armedDeadlineFor(decision, 'c0', at) + return statusOf(decision, 'c0') === 'starting' && epochOf(decision, 'c0') === 1 && + armedDeadlineFor(decision, 'c0', 1, at) }, ) @@ -173,7 +211,7 @@ it.prop( '∀r_Released_=StandbyWithoutTermination', { of: [EventTime, Generation], subject: interpretSupervisionEvent }, (subject, [at, generation]) => { - const policy = policyWith([{ childId: 'c0', restartType: 'permanent' }]) + const policy = policyWith('rest_for_one', [{ childId: 'c0', restartType: 'permanent' }]) const state = runningOf(coreWith(policy, [childOf('c0', generation, 'ready')], [])) const decision = decidedOf(subject, stepOf(state, releasedEventOf('c0', generation, at))) return Match.value(decision).pipe( @@ -189,10 +227,97 @@ it.prop( ) it.prop( - '∀q_ReadyDrainsAwaiting_=StartNext', + '∀r_ReadyWhileStandby_=Held', + { of: [EventTime, Generation], subject: interpretSupervisionEvent }, + (subject, [at, generation]) => { + const policy = policyWith('rest_for_one', [{ childId: 'c0', restartType: 'permanent' }]) + const state = runningOf(coreWith(policy, [childOf('c0', generation, 'standby')], [])) + const decision = decidedOf(subject, stepOf(state, readyEventOf('c0', generation, at))) + return statusOf(decision, 'c0') === 'standby' && isStale(decision) === false + }, +) + +it.prop( + '∀p_AcquiredInRestarting_=KeepsPending', + { of: [EventTime, Generation], subject: interpretSupervisionEvent }, + (subject, [at, generation]) => { + const policy = policyWith('one_for_all', [ + { childId: 'c0', restartType: 'permanent' }, + { childId: 'c1', restartType: 'permanent' }, + ]) + const core = coreWith(policy, [ + childOf('c0', generation, 'standby'), + childOf('c1', generation, 'starting'), + ], []) + const state = new Restarting({ core, pending: [{ childId: 'c1', generation: generation + 1 }] }) + const decision = decidedOf(subject, stepOf(state, acquiredEventOf('c0', generation, at))) + return Result.match( + evolveSupervisor(new SupervisionEvolution({ state, decision })), + { + onFailure: () => false, + onSuccess: (evolved) => + Match.value(evolved).pipe( + Match.tag('Restarting', (restarting) => + restarting.pending.length === 1 && + Match.value(Arr.findFirst(restarting.core.children, (child) => child.childId === 'c0')).pipe( + Match.tag('Some', ({ value }) => value.status === 'starting' && value.epoch === 1), + Match.tag('None', () => false), + Match.exhaustive, + )), + Match.orElse(() => false), + ), + }, + ) + }, +) + +it.prop( + '∀e_StaleEpochDeadline_=Stale', + { of: [EventTime, Generation], subject: interpretSupervisionEvent }, + (subject, [at, generation]) => { + const policy = policyWith('one_for_one', [{ childId: 'c0', restartType: 'permanent' }]) + const state = runningOf(coreWith(policy, [childOf('c0', generation, 'starting', 2)], [])) + return isStale(decidedOf(subject, stepOf(state, startDeadlineTimerOf('c0', generation, 1, at)))) + }, +) + +it.prop( + '∀e_CurrentEpochDeadline_=Missed', + { of: [EventTime, Generation], subject: interpretSupervisionEvent }, + (subject, [at, generation]) => { + const policy = policyWith('one_for_one', [{ childId: 'c0', restartType: 'permanent' }]) + const state = runningOf(coreWith(policy, [childOf('c0', generation, 'starting', 2)], [])) + const decision = decidedOf(subject, stepOf(state, startDeadlineTimerOf('c0', generation, 2, at))) + return isStale(decision) === false && + Arr.some(startedIdsOf(decision), (childId) => childId === 'c0') + }, +) + +it.prop( + '∀o_OneForOneRestart_=Ungated', + { of: [EventTime, Generation], subject: interpretSupervisionEvent }, + (subject, [at, generation]) => { + const policy = policyWith('one_for_one', [ + { childId: 'c0', restartType: 'permanent' }, + { childId: 'c1', restartType: 'permanent' }, + ]) + const state = runningOf(coreWith(policy, [ + childOf('c0', generation, 'standby'), + childOf('c1', generation, 'ready'), + ], [])) + const decision = decidedOf( + subject, + stepOf(state, terminatedEventOf('c1', generation, at, abnormalOf('boom'))), + ) + return Arr.contains(startedIdsOf(decision), 'c1') + }, +) + +it.prop( + '∀q_AcquisitionBlocksAwaiting_=NoStart', { of: [EventTime, Generation], subject: interpretSupervisionEvent }, (subject, [at, generation]) => { - const policy = policyWith([ + const policy = policyWith('rest_for_one', [ { childId: 'c0', restartType: 'permanent' }, { childId: 'c1', restartType: 'permanent' }, ]) @@ -200,26 +325,26 @@ it.prop( childOf('c0', generation, 'standby'), childOf('c1', generation, 'starting'), ], [{ childId: 'c1', generation }])) - const decision = decidedOf(subject, stepOf(state, readyEventOf('c0', generation, at))) - return statusOf(decision, 'c0') === 'ready' && Arr.contains(startedIdsOf(decision), 'c1') && - armedDeadlineFor(decision, 'c1', at) + const acquired = decidedOf(subject, stepOf(state, acquiredEventOf('c0', generation, at))) + return statusOf(acquired, 'c0') === 'starting' && Arr.contains(startedIdsOf(acquired), 'c1') === false }, ) it.prop( - '∀q_StandbyBlocksAwaiting_=NoStart', + '∀q_ReadinessDrainsAwaiting_=StartNext', { of: [EventTime, Generation], subject: interpretSupervisionEvent }, (subject, [at, generation]) => { - const policy = policyWith([ + const policy = policyWith('rest_for_one', [ { childId: 'c0', restartType: 'permanent' }, { childId: 'c1', restartType: 'permanent' }, ]) const state = runningOf(coreWith(policy, [ - childOf('c0', generation, 'standby'), + childOf('c0', generation, 'starting', 1), childOf('c1', generation, 'starting'), ], [{ childId: 'c1', generation }])) - const decision = decidedOf(subject, stepOf(state, acquiredEventOf('c0', generation, at))) - return statusOf(decision, 'c0') === 'starting' && Arr.contains(startedIdsOf(decision), 'c1') === false + const readied = decidedOf(subject, stepOf(state, readyEventOf('c0', generation, at))) + return statusOf(readied, 'c0') === 'ready' && Arr.contains(startedIdsOf(readied), 'c1') && + armedDeadlineFor(readied, 'c1', 0, at) }, ) @@ -227,7 +352,7 @@ it.prop( '∀q_RemovedPredecessor_=UnblocksAwaiting', { of: [EventTime, Generation], subject: interpretSupervisionEvent }, (subject, [at, generation]) => { - const policy = policyWith([ + const policy = policyWith('rest_for_one', [ { childId: 'c0', restartType: 'temporary' }, { childId: 'c1', restartType: 'permanent' }, ]) @@ -242,3 +367,23 @@ it.prop( return Arr.contains(startedIdsOf(terminated), 'c1') }, ) + +it.prop( + '∀d_DeadEntry_=PrunedOnDrain', + { of: [EventTime, Generation], subject: interpretSupervisionEvent }, + (subject, [at, generation]) => { + const policy = policyWith('rest_for_one', [ + { childId: 'c0', restartType: 'permanent' }, + { childId: 'c1', restartType: 'permanent' }, + ]) + const state = runningOf(coreWith(policy, [ + childOf('c0', generation, 'starting'), + childOf('c1', generation, 'starting'), + ], [{ childId: 'gone', generation: 5 }, { childId: 'c1', generation }])) + const readied = decidedOf(subject, stepOf(state, readyEventOf('c0', generation, at))) + return Match.value(readied).pipe( + Match.tag('Continue', ({ core }) => Arr.contains(startedIdsOf(readied), 'c1') && core.awaiting.length === 0), + Match.orElse(() => false), + ) + }, +) diff --git a/packages/effect-daemon-spec/src/kernel/SupervisionEvent.schema.ts b/packages/effect-daemon-spec/src/kernel/SupervisionEvent.schema.ts index 095a582..7fd1c36 100644 --- a/packages/effect-daemon-spec/src/kernel/SupervisionEvent.schema.ts +++ b/packages/effect-daemon-spec/src/kernel/SupervisionEvent.schema.ts @@ -1,11 +1,15 @@ import { Schema } from 'effect' -import { ChildId, EventTime, Generation, RequestId } from './SupervisionLimits.schema.js' +import { ChildId, Epoch, EventTime, Generation, RequestId } from './SupervisionLimits.schema.js' import { TerminationReason } from './TerminationReport.schema.js' export const TimerKind = Schema.Literals(['start_deadline', 'backoff', 'liveness_tick', 'cool_down']) export type TimerKind = typeof TimerKind.Type -export const ChildTimerTarget = Schema.TaggedStruct('ChildTimer', { childId: ChildId, generation: Generation }) +export const ChildTimerTarget = Schema.TaggedStruct('ChildTimer', { + childId: ChildId, + generation: Generation, + epoch: Epoch, +}) export type ChildTimerTarget = typeof ChildTimerTarget.Type export const SupervisorTimerTarget = Schema.TaggedStruct('SupervisorTimer', {}) diff --git a/packages/effect-daemon-spec/src/kernel/SupervisionLimits.schema.ts b/packages/effect-daemon-spec/src/kernel/SupervisionLimits.schema.ts index c0f7e80..6a4e809 100644 --- a/packages/effect-daemon-spec/src/kernel/SupervisionLimits.schema.ts +++ b/packages/effect-daemon-spec/src/kernel/SupervisionLimits.schema.ts @@ -40,6 +40,9 @@ export type Deadline = typeof Deadline.Type export const Generation = NonNegativeSafeInt.pipe(Schema.check(generatedBetween(0, 1_024))) export type Generation = typeof Generation.Type +export const Epoch = NonNegativeSafeInt.pipe(Schema.check(generatedBetween(0, 1_024))) +export type Epoch = typeof Epoch.Type + /** OTP intensity (R3): any non-negative integer restarts per period. Generation stays small because laws fold intensity+1 restarts. */ export const Intensity = NonNegativeSafeInt.pipe(Schema.check(generatedBetween(0, 8))) export type Intensity = typeof Intensity.Type diff --git a/packages/effect-daemon-spec/src/kernel/SupervisorCommand.schema.ts b/packages/effect-daemon-spec/src/kernel/SupervisorCommand.schema.ts index fa1609c..fcafbca 100644 --- a/packages/effect-daemon-spec/src/kernel/SupervisorCommand.schema.ts +++ b/packages/effect-daemon-spec/src/kernel/SupervisorCommand.schema.ts @@ -1,6 +1,6 @@ import { Schema } from 'effect' import { TimerKind } from './SupervisionEvent.schema.js' -import { ChildId, Deadline, Generation, RequestId } from './SupervisionLimits.schema.js' +import { ChildId, Deadline, Epoch, Generation, RequestId } from './SupervisionLimits.schema.js' import { ShutdownMode } from './SupervisorPolicy.schema.js' import { SupervisorExit, TerminationReason } from './TerminationReport.schema.js' @@ -18,6 +18,7 @@ export const ArmChildTimer = Schema.TaggedStruct('ArmChildTimer', { kind: TimerKind, childId: ChildId, generation: Generation, + epoch: Epoch, deadline: Deadline, }) export type ArmChildTimer = typeof ArmChildTimer.Type diff --git a/packages/effect-daemon-spec/src/kernel/SupervisorState.schema.ts b/packages/effect-daemon-spec/src/kernel/SupervisorState.schema.ts index c44c2ef..dfdc3dd 100644 --- a/packages/effect-daemon-spec/src/kernel/SupervisorState.schema.ts +++ b/packages/effect-daemon-spec/src/kernel/SupervisorState.schema.ts @@ -3,6 +3,7 @@ import { dual } from 'effect/Function' import type { SupervisionDecision } from './interpret-supervision-event.workflow.js' import { ChildId, + Epoch, EventTime, Generation, Ordinal, @@ -22,6 +23,7 @@ export const ChildInstance = Schema.Struct({ childId: ChildId, generation: Generation, status: ChildStatus, + epoch: Epoch, consecutiveRestarts: RestartCount, probeFailures: ProbeFailures, }) @@ -84,6 +86,7 @@ export const initialStateOf = (policy: SupervisionPolicy): SupervisorCore => ({ childId: declaration.childId, generation: 0, status: 'starting', + epoch: 0, consecutiveRestarts: 0, probeFailures: 0, })), diff --git a/packages/effect-daemon-spec/src/kernel/interpret-supervision-event.workflow.ts b/packages/effect-daemon-spec/src/kernel/interpret-supervision-event.workflow.ts index abf24fd..9b67bbf 100644 --- a/packages/effect-daemon-spec/src/kernel/interpret-supervision-event.workflow.ts +++ b/packages/effect-daemon-spec/src/kernel/interpret-supervision-event.workflow.ts @@ -3,7 +3,7 @@ import { Array as Arr, Cause, Match, Number as Num, Option, Result, Schema } fro import { SupervisionEvent } from './SupervisionEvent.schema.js' import type { TimerKind, TimerTarget } from './SupervisionEvent.schema.js' import { Millis, PositiveMillis } from './SupervisionLimits.schema.js' -import type { EventTime, Generation, RequestId } from './SupervisionLimits.schema.js' +import type { Epoch, EventTime, Generation, RequestId } from './SupervisionLimits.schema.js' import { ChildId } from './SupervisionLimits.schema.js' import { StopChild, SupervisorCommands } from './SupervisorCommand.schema.js' import type { @@ -167,12 +167,15 @@ const drainAwaitingOf = ( at: EventTime, ): { readonly core: SupervisorCore; readonly commands: SupervisorCommands } => { const issuable = Arr.filter(core.awaiting, (start) => startIssues(core, start)) - const remaining = Arr.filter(core.awaiting, (start) => !Arr.contains(issuable, start)) + const remaining = Arr.filter( + core.awaiting, + (start) => [Arr.contains(issuable, start) === false, startIsLive(core, start)].every((holds) => holds), + ) return { core: { ...core, awaiting: remaining }, commands: commandsIn({ starts: Arr.map(issuable, startCommandOf), - arms: Arr.map(issuable, (start) => deadlineCommandOf(start, at + startTimeoutOf(core, start))), + arms: Arr.map(issuable, (start) => deadlineCommandOf(start, at + startTimeoutOf(core, start), 0)), }), } } @@ -279,6 +282,7 @@ const restartingOf = (child: ChildInstance, generation: Generation): ChildInstan childId: child.childId, generation, status: 'starting', + epoch: 0, consecutiveRestarts: child.consecutiveRestarts + 1, probeFailures: 0, }) @@ -286,6 +290,7 @@ const readiedOf = (child: ChildInstance): ChildInstance => ({ childId: child.childId, generation: child.generation, status: 'ready', + epoch: child.epoch, consecutiveRestarts: 0, probeFailures: 0, }) @@ -294,6 +299,7 @@ const acquiredOf = (child: ChildInstance): ChildInstance => ({ childId: child.childId, generation: child.generation, status: 'starting', + epoch: child.epoch + 1, consecutiveRestarts: child.consecutiveRestarts, probeFailures: 0, }) @@ -302,6 +308,7 @@ const standbyOf = (child: ChildInstance): ChildInstance => ({ childId: child.childId, generation: child.generation, status: 'standby', + epoch: child.epoch, consecutiveRestarts: child.consecutiveRestarts, probeFailures: child.probeFailures, }) @@ -361,14 +368,16 @@ const backoffCommandOf = (start: ChildStart, deadline: EventTime): ArmChildTimer kind: 'backoff', childId: start.childId, generation: start.generation, + epoch: 0, deadline, }) -const deadlineCommandOf = (start: ChildStart, deadline: EventTime): ArmChildTimer => ({ +const deadlineCommandOf = (start: ChildStart, deadline: EventTime, epoch: Epoch): ArmChildTimer => ({ _tag: 'ArmChildTimer', kind: 'start_deadline', childId: start.childId, generation: start.generation, + epoch, deadline, }) @@ -377,6 +386,7 @@ const tickCommandOf = (child: ChildInstance, deadline: EventTime): ArmChildTimer kind: 'liveness_tick', childId: child.childId, generation: child.generation, + epoch: child.epoch, deadline, }) @@ -585,18 +595,36 @@ type RestartOutcome = { const restartOutcomeOf = (core: SupervisorCore, scope: RestartScope, at: EventTime): RestartOutcome => { const nextCore = stampedCoreOf(core, scope, at) const phased = phasedCommands(core, at, scope.coming) - const queued: SupervisorCore = { - ...nextCore, - awaiting: enqueueStarts(nextCore.awaiting, phased.issuable), - } - const drained = drainAwaitingOf(queued, at) - return { - core: drained.core, - stops: scope.stops, - pending: phased.pending, - commands: appendCommands(commandsIn({ stops: scope.stops, arms: phased.arms }), drained.commands), - exhausted: Arr.length(nextCore.restartStamps) > nextCore.policy.intensity, - } + return Match.value(core.policy.strategy).pipe( + Match.when('one_for_one', () => ({ + core: { ...nextCore, awaiting: Arr.filter(nextCore.awaiting, (start) => startIsLive(nextCore, start)) }, + stops: scope.stops, + pending: phased.pending, + commands: commandsIn({ + stops: scope.stops, + arms: [ + ...phased.arms, + ...Arr.map(phased.issuable, (start) => deadlineCommandOf(start, at + startTimeoutOf(core, start), 0)), + ], + starts: Arr.map(phased.issuable, startCommandOf), + }), + exhausted: Arr.length(nextCore.restartStamps) > nextCore.policy.intensity, + })), + Match.orElse((): RestartOutcome => { + const queued: SupervisorCore = { + ...nextCore, + awaiting: enqueueStarts(nextCore.awaiting, phased.issuable), + } + const drained = drainAwaitingOf(queued, at) + return { + core: drained.core, + stops: scope.stops, + pending: phased.pending, + commands: appendCommands(commandsIn({ stops: scope.stops, arms: phased.arms }), drained.commands), + exhausted: Arr.length(nextCore.restartStamps) > nextCore.policy.intensity, + } + }), + ) } const restartedDecisionOf = ( @@ -786,7 +814,7 @@ const readyInRunning = ( const becomingReady = (core: SupervisorCore, child: ChildInstance, at: EventTime): SupervisionDecision => Match.value(child.status).pipe( Match.when('starting', () => readiedDecisionOf(core, child, at)), - Match.when('standby', () => readiedDecisionOf(core, child, at)), + Match.when('standby', () => holdOf(core)), Match.when('ready', () => holdOf(core)), Match.when('stopping', () => holdOf(core)), Match.exhaustive, @@ -882,15 +910,23 @@ const failedProbeOutcomeOf = (core: SupervisorCore, child: ChildInstance, at: Ev Match.exhaustive, ) +const epochedDecision = (child: ChildInstance, epoch: Epoch, decision: SupervisionDecision): SupervisionDecision => + Match.value(child.epoch === epoch).pipe( + Match.when(true, () => decision), + Match.when(false, () => new Stale({})), + Match.exhaustive, + ) + const namedTimerInRunning = ( core: SupervisorCore, child: ChildInstance, kind: TimerKind, at: EventTime, + epoch: Epoch, ): SupervisionDecision => Match.value(kind).pipe( - Match.when('start_deadline', () => deadlineInRunning(core, child, at)), - Match.when('liveness_tick', () => tickInRunning(core, child, at)), + Match.when('start_deadline', () => epochedDecision(child, epoch, deadlineInRunning(core, child, at))), + Match.when('liveness_tick', () => epochedDecision(child, epoch, tickInRunning(core, child, at))), Match.when('backoff', () => new Stale({})), Match.when('cool_down', () => new Stale({})), Match.exhaustive, @@ -936,10 +972,11 @@ const timerInRunning = ( generation: Generation, kind: TimerKind, at: EventTime, + epoch: Epoch, ): SupervisionDecision => Match.value(currentIncarnation(core, childId, generation)).pipe( Match.tag('None', () => new Stale({})), - Match.tag('Some', (found) => namedTimerInRunning(core, found.value, kind, at)), + Match.tag('Some', (found) => namedTimerInRunning(core, found.value, kind, at, epoch)), Match.exhaustive, ) @@ -952,15 +989,30 @@ const fireBackoffOf = ( ): SupervisionDecision => { const start: ChildStart = { childId, generation } const remaining = Arr.filter(pending, (plan) => planMatches(childId, generation)(plan) === false) - const queued: SupervisorCore = { ...core, awaiting: enqueueStarts(core.awaiting, [start]) } - const drained = drainAwaitingOf(queued, at) - return Match.value(Arr.length(remaining) === 0).pipe( - Match.when(true, () => new StartChildren({ core: drained.core, commands: drained.commands })), - Match.when( - false, - () => new RestartChildren({ core: drained.core, pending: remaining, commands: drained.commands }), - ), - Match.exhaustive, + return Match.value(core.policy.strategy).pipe( + Match.when('one_for_one', () => { + const commands: SupervisorCommands = commandsIn({ + starts: [startCommandOf(start)], + arms: [deadlineCommandOf(start, at + startTimeoutOf(core, start), 0)], + }) + return Match.value(Arr.length(remaining) === 0).pipe( + Match.when(true, () => new StartChildren({ core, commands })), + Match.when(false, () => new RestartChildren({ core, pending: remaining, commands })), + Match.exhaustive, + ) + }), + Match.orElse(() => { + const queued: SupervisorCore = { ...core, awaiting: enqueueStarts(core.awaiting, [start]) } + const drained = drainAwaitingOf(queued, at) + return Match.value(Arr.length(remaining) === 0).pipe( + Match.when(true, () => new StartChildren({ core: drained.core, commands: drained.commands })), + Match.when( + false, + () => new RestartChildren({ core: drained.core, pending: remaining, commands: drained.commands }), + ), + Match.exhaustive, + ) + }), ) } @@ -1020,6 +1072,7 @@ const dynamicInstanceOf = (core: SupervisorCore): { readonly core: SupervisorCor childId: start.childId, generation: start.generation, status: 'starting', + epoch: 0, consecutiveRestarts: 0, probeFailures: 0, } @@ -1035,18 +1088,14 @@ const allocatedDecisionOf = ( requestId: RequestId, at: EventTime, ): SupervisionDecision => { - const timeoutMillis = Match.value(dynamicKindOf(allocated.core.policy)).pipe( - Match.tag('Some', ({ value }) => value.startTimeoutMillis), - Match.tag('None', () => UNKNOWN_TIMEOUT_MILLIS), - Match.exhaustive, - ) + const queued: SupervisorCore = { + ...allocated.core, + awaiting: enqueueStarts(allocated.core.awaiting, [allocated.start]), + } + const drained = drainAwaitingOf(queued, at) return new StartChildren({ - core: allocated.core, - commands: commandsIn({ - starts: [startCommandOf(allocated.start)], - arms: [deadlineCommandOf(allocated.start, at + timeoutMillis)], - replies: [replyAcceptedOf(requestId, allocated.start)], - }), + core: drained.core, + commands: appendCommands(drained.commands, commandsIn({ replies: [replyAcceptedOf(requestId, allocated.start)] })), }) } @@ -1193,7 +1242,10 @@ const onTimerElapsed = ( Match.value(state).pipe( Match.tag('Running', ({ core }) => Match.value(target).pipe( - Match.tag('ChildTimer', (child) => timerInRunning(core, child.childId, child.generation, kind, at)), + Match.tag( + 'ChildTimer', + (child) => timerInRunning(core, child.childId, child.generation, kind, at, child.epoch), + ), Match.tag('SupervisorTimer', () => new Stale({})), Match.exhaustive, )), @@ -1317,6 +1369,7 @@ const acquiredOfChild = (core: SupervisorCore, child: ChildInstance, at: EventTi deadlineCommandOf( { childId: child.childId, generation: child.generation }, at + startTimeoutOf(core, { childId: child.childId, generation: child.generation }), + child.epoch + 1, ), ], }), @@ -1361,21 +1414,17 @@ const releasedInRunning = (core: SupervisorCore, childId: ChildId, generation: G Match.exhaustive, ) -const holdInOtherPhases = (state: SupervisorState): SupervisionDecision => +const coreOfLive = (state: SupervisorState): Option.Option => Match.value(state).pipe( - Match.tag('Running', ({ core }) => holdOf(core)), - Match.tag('Restarting', ({ core }) => holdOf(core)), - Match.tag('CoolingDown', ({ core }) => holdOf(core)), - Match.tag('ShuttingDown', ({ core }) => holdOf(core)), - Match.tag('Terminated', () => new Stale({})), - Match.exhaustive, + Match.tag('Terminated', () => Option.none()), + Match.orElse((live) => Option.some(live.core)), ) const onChildStandby = (state: SupervisorState, childId: ChildId, generation: Generation): SupervisionDecision => - Match.value(state).pipe( - Match.tag('Running', ({ core }) => standbyInRunning(core, childId, generation)), - Match.orElse(() => holdInOtherPhases(state)), - ) + Option.match(coreOfLive(state), { + onNone: () => new Stale({}), + onSome: (core) => standbyInRunning(core, childId, generation), + }) const onChildAcquired = ( state: SupervisorState, @@ -1383,16 +1432,16 @@ const onChildAcquired = ( generation: Generation, at: EventTime, ): SupervisionDecision => - Match.value(state).pipe( - Match.tag('Running', ({ core }) => acquiredInRunning(core, childId, generation, at)), - Match.orElse(() => holdInOtherPhases(state)), - ) + Option.match(coreOfLive(state), { + onNone: () => new Stale({}), + onSome: (core) => acquiredInRunning(core, childId, generation, at), + }) const onChildReleased = (state: SupervisorState, childId: ChildId, generation: Generation): SupervisionDecision => - Match.value(state).pipe( - Match.tag('Running', ({ core }) => releasedInRunning(core, childId, generation)), - Match.orElse(() => holdInOtherPhases(state)), - ) + Option.match(coreOfLive(state), { + onNone: () => new Stale({}), + onSome: (core) => releasedInRunning(core, childId, generation), + }) const decided = (state: SupervisorState, event: SupervisionEvent): SupervisionDecision => Match.value(event).pipe( From 194b859bc4c80625eec3824984c0b9f9dd2a5437 Mon Sep 17 00:00:00 2001 From: systemfsoftware-bot <318394648+systemfsoftware-bot@users.noreply.github.com> Date: Thu, 19 Nov 2026 14:32:30 -0500 Subject: [PATCH 3/3] fix(effect-daemon-spec): reset the acquisition epoch when a cool-down restarts children MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit cooledRestartOf spread the old child record, keeping a non-zero epoch from before the cool-down while drainAwaitingOf arms the fresh start_deadline at epoch 0, so the elapse would read as stale and a hung incarnation would never miss its deadline. A restart resets the epoch like restartingOf already does. New property law ∀c_CoolDown_=ResetsEpoch: a cooling-down child with a non-zero epoch comes back at epoch 0 with an epoch-0 start_deadline armed; it fails against the unfixed fold. --- ...upervision-event.workflow.property.test.ts | 49 ++++++++++++++++++- .../interpret-supervision-event.workflow.ts | 1 + 2 files changed, 48 insertions(+), 2 deletions(-) diff --git a/packages/effect-daemon-spec/src/__tests__/interpret-supervision-event.workflow.property.test.ts b/packages/effect-daemon-spec/src/__tests__/interpret-supervision-event.workflow.property.test.ts index e361fc2..4330eb1 100644 --- a/packages/effect-daemon-spec/src/__tests__/interpret-supervision-event.workflow.property.test.ts +++ b/packages/effect-daemon-spec/src/__tests__/interpret-supervision-event.workflow.property.test.ts @@ -82,11 +82,11 @@ const declaredChild = (childId: string, restartType: RestartType, significant: b probeFailureThreshold: 2, }) -const childOf = (childId: string, generation: number, status: ChildStatus): ChildInstance => ({ +const childOf = (childId: string, generation: number, status: ChildStatus, epoch = 0): ChildInstance => ({ childId, generation, status, - epoch: 0, + epoch, consecutiveRestarts: 0, probeFailures: 0, }) @@ -603,6 +603,51 @@ it.prop( }, ) +it.prop( + '∀c_CoolDown_=ResetsEpoch', + { of: [PositiveMillis, EventTime], subject: interpretSupervisionEvent }, + (subject, [millis, at0]) => { + const t0 = at0 + const policy = policyWith({ + strategy: 'one_for_one', + intensity: 1, + coolDown: { _tag: 'CoolDownAfter', millis }, + childDeclarations: [declaredChild('c0', 'permanent', false)], + }) + const cooled = new CoolingDown({ + core: coreWith(policy, [childOf('c0', 2, 'ready', 3)], []), + millis, + }) + const resumedAt = t0 + 1 + millis + const resumed = folded(subject, cooled, [supervisorTimerEvent('cool_down', resumedAt)]) + return Match.value(lastDecisionOf(resumed)).pipe( + Match.tag('StartChildren', (decision) => + Match.value(coreOf(decision)).pipe( + Match.tag('Some', ({ value }) => + Match.value(Arr.findFirst(value.children, (child) => + child.childId === 'c0')).pipe( + Match.tag('Some', ({ value: child }) => + child.epoch === 0 && + Arr.some(commandsOf(decision).arms, (arm) => + Match.value(arm).pipe( + Match.tag('ArmChildTimer', (timer) => + timer.kind === 'start_deadline' && timer.childId === 'c0' && timer.epoch === 0 && + timer.deadline === resumedAt + 50), + Match.tag('ArmSupervisorTimer', () => false), + Match.exhaustive, + ))), + Match.tag('None', () => false), + Match.exhaustive, + )), + Match.tag('None', () => + false), + Match.exhaustive, + )), + Match.orElse(() => false), + ) + }, +) + it.prop( '∀g_WindowPrune_≡Period', { of: [PositiveMillis, EventTime], subject: interpretSupervisionEvent }, diff --git a/packages/effect-daemon-spec/src/kernel/interpret-supervision-event.workflow.ts b/packages/effect-daemon-spec/src/kernel/interpret-supervision-event.workflow.ts index 9b67bbf..c2a0af3 100644 --- a/packages/effect-daemon-spec/src/kernel/interpret-supervision-event.workflow.ts +++ b/packages/effect-daemon-spec/src/kernel/interpret-supervision-event.workflow.ts @@ -1041,6 +1041,7 @@ const cooledRestartOf = (core: SupervisorCore, at: EventTime): SupervisionDecisi const fresh: ReadonlyArray = Arr.map(core.children, (child) => ({ ...child, status: 'starting', + epoch: 0, consecutiveRestarts: 0, probeFailures: 0, }))