Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions .changeset/standby-singleton-readiness.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
---
"@systemfsoftware/effect-daemon-spec": patch
"@systemfsoftware/effect-daemon-conformance": patch
"@systemfsoftware/effect-daemon-cluster": patch
---

Standby singletons no longer report ready before shard acquisition. A cluster singleton whose shard another runner owns now registers as a `standby`: it is not ready, children declared after it do not start until it owns its shard and its run signals ready, and `startTimeoutMillis` is measured from shard acquisition rather than registration. Losing the shard is a handoff, not a failure: the child returns to standby without a termination, restart, or intensity charge. The supervisor starts children in declared order, each gated on the readiness of the ones before it, and status and the trace expose `standby`, `acquired`, `ready` and `released` transitions for every child.
9 changes: 6 additions & 3 deletions packages/effect-daemon-cluster/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -21,9 +21,12 @@ Its peer dependencies, `effect@^4.1.2` and `typescript@>=7.0.0`, come from your
an inferred failure.
- **The declared loss**: the medium declares `{ reporting: 'inferred', groupStop: 'eventual' }`, so
the conformance kit holds it only to what it can observe and honour.
- **Standby and handoff**: a singleton whose shard another runner owns is announced ready at once
and waits; it starts when the shard arrives. A run interrupted because its shard moved away is a
handoff, not a termination: the child stays registered and nothing is reported.
- **Standby and handoff**: a singleton whose shard another runner owns registers as a standby: it is
not ready, it is not bound by the start deadline, and children declared after it do not start until
it owns its shard and its run signals ready. The start deadline is armed when the shard arrives, so
`startTimeoutMillis` bounds the real post-acquisition initialization. A run interrupted because its
shard moved away is a handoff, not a termination: the child returns to standby, stays registered,
and nothing is reported, restarted or charged against the restart intensity.
- **Singletons across processes over Postgres**: `ClusterMedium.sqlRunnerLayer` runs `Sharding`
over `SqlRunnerStorage` with no-op runner transport, health and message storage. The oldest live
runner owns every shard, so a joining process waits as a standby, and a runner whose sessions are
Expand Down
86 changes: 75 additions & 11 deletions packages/effect-daemon-cluster/src/ClusterMedium/medium.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,20 @@
import { Supervisor } from '@systemfsoftware/effect-daemon-spec'
import { Cause, Deferred, Duration, Effect, Exit, Layer, Match, Option, Ref, Result, Scope } from 'effect'
import { EntityId, Sharding } from 'effect/cluster'
import {
Cause,
Deferred,
Duration,
Effect,
Exit,
Layer,
Match,
Option,
Queue,
Ref,
Result,
Scope,
Stream,
} from 'effect'
import { Sharding } from 'effect/cluster'
import { absurd } from 'effect/Function'
import type { ClusterProgram, ClusterProgramRequirements, EntityChild, SingletonChild } from './ClusterProgram.js'

Expand All @@ -22,21 +36,33 @@ interface ClusterStarted extends Supervisor.Medium.Started {
readonly ended: Deferred.Deferred<Supervisor.Medium.TerminationReason>
readonly stopping: Ref.Ref<boolean>
readonly probe: Effect.Effect<boolean, never, ClusterMediumRequirement>
readonly activityQueue?: Queue.Queue<Supervisor.Medium.MediumActivity> | undefined
}

const activityExtras = (
activity?: Supervisor.Medium.StartedActivity,
activityQueue?: Queue.Queue<Supervisor.Medium.MediumActivity>,
): {
readonly activity?: Supervisor.Medium.StartedActivity
readonly activityQueue?: Queue.Queue<Supervisor.Medium.MediumActivity> | undefined
} => activity === undefined ? {} : { activity, activityQueue }

const clusterStarted = (
ready: Effect.Effect<void>,
registration: Scope.Closeable,
ended: Deferred.Deferred<Supervisor.Medium.TerminationReason>,
stopping: Ref.Ref<boolean>,
probe: Effect.Effect<boolean, never, ClusterMediumRequirement>,
activity?: Supervisor.Medium.StartedActivity,
activityQueue?: Queue.Queue<Supervisor.Medium.MediumActivity>,
): ClusterStarted => ({
...Supervisor.Medium.started(ready),
[ClusterStartedTypeId]: ClusterStartedTypeId,
registration,
ended,
stopping,
probe,
...activityExtras(activity, activityQueue),
})

const isClusterStarted = (evidence: Supervisor.Medium.Started): evidence is ClusterStarted =>
Expand Down Expand Up @@ -78,11 +104,39 @@ const reportedOnExit = (
lostItsShard(stopped, exit) ? Effect.void : Effect.asVoid(Deferred.succeed(ended, terminationOf(stopped, exit))),
)

const ownsSingleton = (sharding: Sharding.Sharding['Service'], name: string, shardGroup: string | undefined) =>
sharding.hasShardId(sharding.getShardId(EntityId.make(name), shardGroup ?? 'default'))
const announceAcquire = (changes: Queue.Queue<Supervisor.Medium.MediumActivity>): Effect.Effect<void> =>
Queue.offer(changes, { _tag: 'Acquired' }).pipe(Effect.asVoid)

const readyWhen = (owned: boolean, signalled: Deferred.Deferred<void>): Effect.Effect<void> =>
owned ? Deferred.await(signalled) : Effect.void
const singletonReady = (
signalled: Deferred.Deferred<void>,
changes: Queue.Queue<Supervisor.Medium.MediumActivity>,
): Effect.Effect<void> =>
Effect.flatMap(
Deferred.isDone(signalled),
(alreadySignalled) => alreadySignalled ? Effect.void : Effect.asVoid(Deferred.succeed(signalled, void 0)),
).pipe(
Effect.andThen(Queue.offer(changes, { _tag: 'Ready' })),
Effect.asVoid,
)

const singletonRun = (
program: SingletonChild<ClusterMediumRequirement>,
signalled: Deferred.Deferred<void>,
ended: Deferred.Deferred<Supervisor.Medium.TerminationReason>,
stopping: Ref.Ref<boolean>,
changes: Queue.Queue<Supervisor.Medium.MediumActivity>,
): Effect.Effect<void, never, ClusterMediumRequirement> =>
Effect.gen(function*() {
yield* announceAcquire(changes)
yield* program.run(singletonReady(signalled, changes))
}).pipe(
Effect.onExit((exit) =>
Effect.flatMap(Ref.get(stopping), (stopped) =>
lostItsShard(stopped, exit)
? Effect.asVoid(Queue.offer(changes, { _tag: 'Released' }))
: reportedOnExit(ended, stopping)(exit))
),
)

const startSingleton = (
program: SingletonChild<ClusterMediumRequirement>,
Expand All @@ -95,15 +149,21 @@ const startSingleton = (
const ended = yield* Deferred.make<Supervisor.Medium.TerminationReason>()
const stopping = yield* Ref.make(false)
const sharding = yield* Sharding.Sharding
const run = program.run(Deferred.succeed(signalled, void 0)).pipe(
Effect.onExit(reportedOnExit(ended, stopping)),
)
const changes = yield* Queue.unbounded<Supervisor.Medium.MediumActivity>()
const run = singletonRun(program, signalled, ended, stopping, changes)
yield* sharding.registerSingleton(program.name, run, { shardGroup }).pipe(
Effect.provideService(Scope.Scope, registration),
)
const ready = readyWhen(ownsSingleton(sharding, program.name, shardGroup), signalled)
const probe = Deferred.isDone(ended).pipe(Effect.map((done) => done === false))
return clusterStarted(ready, registration, ended, stopping, probe)
return clusterStarted(
Deferred.await(signalled),
registration,
ended,
stopping,
probe,
{ initiallyStandby: true, changes: Stream.fromQueue(changes) },
changes,
)
})

const startEntity = (
Expand Down Expand Up @@ -155,6 +215,9 @@ const probeOf = (
onSome: (self) => self.probe,
})

const shutdownActivityOf = (self: ClusterStarted): Effect.Effect<void> =>
self.activityQueue === undefined ? Effect.void : Queue.shutdown(self.activityQueue).pipe(Effect.ignore)

const stopOf = (
evidence: Supervisor.Medium.Started,
): Effect.Effect<Supervisor.Medium.Stopped, never, ClusterMediumRequirement> =>
Expand All @@ -163,6 +226,7 @@ const stopOf = (
onSome: (self) =>
Ref.set(self.stopping, true).pipe(
Effect.andThen(Scope.close(self.registration, Exit.void)),
Effect.andThen(shutdownActivityOf(self)),
Effect.andThen(Deferred.succeed(self.ended, shutdownTermination)),
Effect.as(Supervisor.Medium.stopped),
),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -114,13 +114,14 @@ const standbyThenOwned = Effect.gen(function*() {
const shards = yield* ShardScript
const { runs, program } = yield* countedSingleton
const evidence = yield* medium.start(program)
yield* evidence.ready
const readyWhileStandby = yield* Effect.timeoutOption(evidence.ready, Duration.millis(100))
const aliveAsStandby = yield* medium.probe(evidence)
const runsAsStandby = yield* Ref.get(runs)
yield* shards.grant(SINGLETON)
yield* evidence.ready
const startedOnceOwned = yield* runsReach(runs, 1)
const reported = yield* reportedWithin(medium.report(evidence))
return { aliveAsStandby, runsAsStandby, startedOnceOwned, reported }
return { readyWhileStandby, aliveAsStandby, runsAsStandby, startedOnceOwned, reported }
})

const handedOffAndBack = Effect.gen(function*() {
Expand All @@ -141,7 +142,6 @@ const standbyStopped = Effect.gen(function*() {
const { medium } = yield* ClusterMedium.port
const { program } = yield* countedSingleton
const evidence = yield* medium.start(program)
yield* evidence.ready
yield* medium.stop(evidence, { _tag: 'Brutal' })
const reported = yield* reportedWithin(medium.report(evidence))
yield* nothingLeftRegistered
Expand All @@ -163,13 +163,15 @@ Feature('Supervising children hosted by the cluster', { timeout: 120_000 })
'observed',
(s) => Effect.provide(standbyThenOwned, s.environment),
),
Then('it was ready and alive without running, ran once its shard arrived, and nothing ended')((s, expect) =>
expect(s.observed).toEqual({
aliveAsStandby: true,
runsAsStandby: 0,
startedOnceOwned: true,
reported: Option.none(),
})
Then('it was not ready yet stayed alive without running, ran once its shard arrived, and nothing ended')(
(s, expect) =>
expect(s.observed).toEqual({
readyWhileStandby: Option.none(),
aliveAsStandby: true,
runsAsStandby: 0,
startedOnceOwned: true,
reported: Option.none(),
}),
),
),
)
Expand Down
Loading