diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml new file mode 100644 index 0000000..d81d201 --- /dev/null +++ b/.github/workflows/go.yml @@ -0,0 +1,33 @@ +name: Go + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + +permissions: + contents: read + +jobs: + test: + name: Vet and Test + runs-on: ubuntu-24.04 + + steps: + - name: Check out code + uses: actions/checkout@v4 + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version-file: 'go.mod' + + - name: Vet + run: go vet ./... + + - name: Test Race Conditions + run: go test -race -v ./... + + - name: Build + run: go build ./... diff --git a/.golangci.yml b/.golangci.yml new file mode 100644 index 0000000..f9c537e --- /dev/null +++ b/.golangci.yml @@ -0,0 +1,87 @@ +version: "2" + +linters: + default: all + disable: + - inamedparam + - gochecknoinits + - wsl + - wsl_v5 + - nlreturn + - whitespace + - wrapcheck + - exhaustruct + - exhaustruct_v5 + - err113 + - testpackage + - depguard + - paralleltest + - varnamelen + - forbidigo + - funlen + - gochecknoglobals + - godox + - lll + - mnd + - noinlineerr + # funcorder demands constructor-then-exported-then-unexported ordering, + # which conflicts with the convention here (shared with dioad/net) of + # placing an unexported helper immediately after the exported method that + # calls it: readability comes from proximity to the call site. Pure + # code-organization preference, consistent with the other style linters + # disabled above. + - funcorder + settings: + ireturn: + allow: + - anon + - error + - empty + - stdlib + # Generic functions must return their type parameter; that isn't + # the "unnecessary interface" smell ireturn looks for. + - generic + exhaustive: + # A switch that deliberately dispatches over a small, bounded subset + # of an enum's members and handles everything else in "default" is + # exhaustive by design; the default case is the intended catch-all, + # not a sign the switch forgot members. + default-signifies-exhaustive: true + tagliatelle: + case: + rules: + json: snake + exclusions: + rules: + # Table-driven tests naturally accumulate cognitive/nesting + # "complexity" from many small, independent per-case closures + # (checkFunc, subtests); that's the idiomatic Go testing pattern, not a + # sign the test itself is hard to follow. funlen is disabled repo-wide + # for the same reason. + - path: '_test\.go' + linters: + - cyclop + - gocognit + - gocyclo + - maintidx + - nestif + # Repeated literals in tests are almost always fixture/expectation + # data (a hostname, a header name, an error string) restated across + # otherwise-independent table-driven cases, not a shared business + # constant that drifted. Forcing every one into a package-level + # const would couple unrelated test cases together for no benefit; + # a magic number/string inside one test's own table is still + # self-documenting in context. 96 of this repo's 97 goconst + # findings are in _test.go files. + - path: '_test\.go' + linters: + - goconst + +issues: + max-issues-per-linter: 0 + max-same-issues: 0 + +formatters: + enable: + - gofmt + - goimports diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..4ca6609 --- /dev/null +++ b/Makefile @@ -0,0 +1,21 @@ + +# This module is verified against the versions its go.mod declares, not against +# whatever a go.work above it resolves. +export GOWORK=off + +default: verify + +build: + go build . + +lint: + go fmt ./... + go fix ./... + golangci-lint run ./... + gosec ./... + go vet ./... + +test: + go test -race ./... + +verify: lint test build diff --git a/README.md b/README.md index 8194b8a..8d960df 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,41 @@ # lifecycle + Process lifecycle for Dioad daemons: ordered start with rollback, staged stop with budgets, failure fan-in, readiness and drain. + +```go +app := lifecycle.New( + lifecycle.WithDrainDelay(5*time.Second), + lifecycle.WithStopStages( + lifecycle.Stage{Names: []string{"api"}, Budget: 10 * time.Second}, + lifecycle.Stage{Names: []string{"worker"}}, + ), +) +if err := app.Add("store", store); err != nil { + return err +} +if err := app.Add("worker", worker); err != nil { + return err +} +if err := app.Add("api", apiServer); err != nil { + return err +} + +// Report readiness wherever the platform looks for it, for example +// dioad/net's http.Server.AddHealthSource. +app.Readiness().AddCheck("store", store.Ping, lifecycle.WithCacheTTL(5*time.Second)) + +return app.Run(ctx) // returns when ctx is done or a service fails +``` + +| | | +|---|---| +| **Start** | In registration order. A failure stops the services that had started, in the same way Stop would, and is returned joined with any stop errors. | +| **Stop** | In stages: the services of a stage stop concurrently, each stage optionally within a budget, all within one stop timeout. Without stages, in reverse registration order. Errors are joined. | +| **Failure** | A service that implements `Failed() <-chan error` has its failure reported on `App.Failed` as a `*ServiceError`. `Run` stops at once on a failure, without draining. | +| **Readiness** | Not ready until started, not ready once draining, and in between ready unless a named check fails. A check can be cached for a TTL, and can be `Degraded`: reported, but still ready. It is a health source for `dioad/net`'s HTTP server, without importing it. | +| **Drain** | When `Run`'s context is done, readiness reports not ready and `Run` waits for the drain delay before it stops the services, so the load balancer stops first. | + +A `Service` is `Start(ctx) error` and `Shutdown(ctx) error`: the same method set as `dioad/net`'s `http.Server`, so +one is a `Service` with no adapter. `Failed`, `Live` and `Ready` are optional, found by interface assertion. + +See the package documentation for the details, and the examples for runnable versions. diff --git a/app.go b/app.go new file mode 100644 index 0000000..1033d7a --- /dev/null +++ b/app.go @@ -0,0 +1,399 @@ +package lifecycle + +import ( + "context" + "errors" + "fmt" + "slices" + "sync" + "time" +) + +// defaultStopTimeout is the overall stop deadline when WithStopTimeout is not +// given. +const defaultStopTimeout = 30 * time.Second + +// Stage is a set of services that Stop stops concurrently. +type Stage struct { + // Names are the names the services were registered under. + Names []string + // Budget, if positive, is how long the stage may take. A service that has + // not returned by then has its Shutdown context cancelled, and the next + // stage starts. It is inside the overall stop deadline. + Budget time.Duration +} + +// Option configures an App. +type Option func(*App) + +// WithDrainDelay sets how long Drain waits after reporting not ready, before +// the services are stopped. The default is no wait. +func WithDrainDelay(d time.Duration) Option { + return func(a *App) { a.drainDelay = d } +} + +// WithStopTimeout sets the deadline for draining and stopping together. The +// default is 30 seconds. Zero or less means no deadline other than the +// context given to Stop. +func WithStopTimeout(d time.Duration) Option { + return func(a *App) { a.stopTimeout = d } +} + +// WithStopStages sets the stages Stop stops services in. See the package +// documentation for what stops a service no stage names. +func WithStopStages(stages ...Stage) Option { + return func(a *App) { a.stages = stages } +} + +// App starts, watches and stops a set of services. See the package +// documentation. +type App struct { + drainDelay time.Duration + stopTimeout time.Duration + stages []Stage + readiness *Readiness + + // op serialises Start and Stop; mu guards the fields below. + op sync.Mutex + + mu sync.Mutex + entries []*entry + state appState + started []*entry + failed chan error + done chan struct{} + watchers sync.WaitGroup +} + +type appState int + +const ( + appNew appState = iota + appStarted + appStopped +) + +type entry struct { + name string + service Service +} + +// New returns an App with no services. +func New(opts ...Option) *App { + a := &App{ + stopTimeout: defaultStopTimeout, + readiness: NewReadiness(), + done: make(chan struct{}), + } + for _, opt := range opts { + opt(a) + } + + return a +} + +// Readiness returns the App's readiness, for adding checks and for reporting +// as a health source. +func (a *App) Readiness() *Readiness { + return a.readiness +} + +// Add registers a service under name. Services start in the order they are +// added. It returns an error for an empty or repeated name, a nil service, or +// once the App has been started. +func (a *App) Add(name string, service Service) error { + a.mu.Lock() + defer a.mu.Unlock() + + switch { + case a.state != appNew: + return ErrAlreadyStarted + case name == "": + return errors.New("lifecycle: service name is empty") + case service == nil: + return fmt.Errorf("lifecycle: service %q is nil", name) + case slices.ContainsFunc(a.entries, func(e *entry) bool { return e.name == name }): + return fmt.Errorf("lifecycle: service %q is already added", name) + } + + a.entries = append(a.entries, &entry{name: name, service: service}) + if checker, ok := service.(readyChecker); ok { + a.readiness.AddCheck(name, func(ctx context.Context) error { + _, err := checker.Ready(ctx) + + return err + }) + } + + return nil +} + +// Start starts the services in the order they were added, then opens +// Readiness. If a service fails to start, the ones already started are +// stopped and Start returns the failure joined with any stop errors; the App +// cannot then be started again. An App can be started once. +func (a *App) Start(ctx context.Context) error { + a.op.Lock() + defer a.op.Unlock() + + a.mu.Lock() + if a.state != appNew { + a.mu.Unlock() + + return ErrAlreadyStarted + } + entries := slices.Clone(a.entries) + if err := a.validateStages(entries); err != nil { + a.mu.Unlock() + + return err + } + a.failed = make(chan error, len(entries)) + a.mu.Unlock() + + started := make([]*entry, 0, len(entries)) + for _, e := range entries { + if err := e.service.Start(ctx); err != nil { + a.mu.Lock() + a.state = appStopped + a.mu.Unlock() + close(a.done) + + // ctx may be done, which may be why Start failed, and the + // rollback must not inherit that. + rollbackCtx, cancel := a.stopContext(context.WithoutCancel(ctx)) + defer cancel() + + return errors.Join(fmt.Errorf("start %s: %w", e.name, err), a.stopEntries(rollbackCtx, started)) + } + started = append(started, e) + } + + a.mu.Lock() + a.started = started + a.state = appStarted + a.mu.Unlock() + + a.watch(started) + a.readiness.MarkStarted() + + return nil +} + +// watch fans the failures of the services that can fail into a.failed. Each +// watcher ends when its service fails or when the App stops, since a Failed +// channel is not required to close. +func (a *App) watch(started []*entry) { + for _, e := range started { + failer, ok := e.service.(Failer) + if !ok { + continue + } + failed := failer.Failed() + a.watchers.Go(func() { + select { + case err := <-failed: + // a.failed has room for every service, and each fails once. + a.failed <- &ServiceError{Service: e.name, Err: err} + case <-a.done: + } + }) + } +} + +// Failed returns a channel that receives the failure of any service that +// implements Failer, as a *ServiceError, once Start has returned. It is nil +// before then, and never closes. +func (a *App) Failed() <-chan error { + a.mu.Lock() + defer a.mu.Unlock() + + return a.failed +} + +// Live returns nil while every started service that implements Liveness +// reports live, and otherwise the errors of those that do not, each naming its +// service. +func (a *App) Live(ctx context.Context) error { + a.mu.Lock() + started := slices.Clone(a.started) + a.mu.Unlock() + + var errs []error + for _, e := range started { + live, ok := e.service.(Liveness) + if !ok { + continue + } + if err := live.Live(ctx); err != nil { + errs = append(errs, fmt.Errorf("%s: %w", e.name, err)) + } + } + + return errors.Join(errs...) +} + +// Drain makes Readiness report not ready, then waits for the drain delay or +// for ctx to be done, so that traffic stops before the services do. +func (a *App) Drain(ctx context.Context) { + a.readiness.BeginDrain() + if a.drainDelay <= 0 { + return + } + + timer := time.NewTimer(a.drainDelay) + defer timer.Stop() + select { + case <-timer.C: + case <-ctx.Done(): + } +} + +// Stop stops the started services in stages, within the stop timeout and +// ctx's deadline, whichever is sooner, and returns the errors joined. It does +// not drain; see Drain. It does nothing for an App that was never started, +// that failed to start, or that has already stopped. +func (a *App) Stop(ctx context.Context) error { + a.op.Lock() + defer a.op.Unlock() + + a.mu.Lock() + if a.state != appStarted { + a.mu.Unlock() + + return nil + } + a.state = appStopped + started := a.started + a.mu.Unlock() + + a.readiness.BeginDrain() + close(a.done) + + ctx, cancel := a.stopContext(ctx) + defer cancel() + + err := a.stopEntries(ctx, started) + a.watchers.Wait() + + return err +} + +// Run starts the services and blocks until ctx is done or a service fails, +// then stops them. When ctx is done it drains first and returns nil if the +// stop was clean. When a service fails it stops at once, without draining, and +// returns the *ServiceError joined with any stop errors. The drain and the +// stop share one stop timeout, and neither is cut short by ctx being done. +func (a *App) Run(ctx context.Context) error { + if err := a.Start(ctx); err != nil { + return err + } + + select { + case <-ctx.Done(): + stopCtx, cancel := a.stopContext(context.WithoutCancel(ctx)) + defer cancel() + a.Drain(stopCtx) + + return a.Stop(stopCtx) + case failure := <-a.Failed(): + stopCtx, cancel := a.stopContext(context.WithoutCancel(ctx)) + defer cancel() + + return errors.Join(failure, a.Stop(stopCtx)) + } +} + +// stopContext returns ctx with the stop timeout applied. +func (a *App) stopContext(ctx context.Context) (context.Context, context.CancelFunc) { + if a.stopTimeout <= 0 { + return ctx, func() {} + } + + return context.WithTimeout(ctx, a.stopTimeout) +} + +// validateStages checks that every stage names registered services, once. +func (a *App) validateStages(entries []*entry) error { + seen := map[string]bool{} + for _, stage := range a.stages { + for _, name := range stage.Names { + if !slices.ContainsFunc(entries, func(e *entry) bool { return e.name == name }) { + return fmt.Errorf("lifecycle: stop stage names %q, which is not a service", name) + } + if seen[name] { + return fmt.Errorf("lifecycle: service %q is in more than one stop stage", name) + } + seen[name] = true + } + } + + return nil +} + +type plannedStage struct { + entries []*entry + budget time.Duration +} + +// plan returns the stages that stop started. +func (a *App) plan(started []*entry) []plannedStage { + var stages []plannedStage + planned := map[*entry]bool{} + + for _, stage := range a.stages { + var entries []*entry + for _, name := range stage.Names { + i := slices.IndexFunc(started, func(e *entry) bool { return e.name == name }) + if i < 0 { + continue + } + entries = append(entries, started[i]) + planned[started[i]] = true + } + if len(entries) > 0 { + stages = append(stages, plannedStage{entries: entries, budget: stage.Budget}) + } + } + + for _, e := range slices.Backward(started) { + if !planned[e] { + stages = append(stages, plannedStage{entries: []*entry{e}}) + } + } + + return stages +} + +// stopEntries stops started, a stage at a time. +func (a *App) stopEntries(ctx context.Context, started []*entry) error { + stages := a.plan(started) + errs := make([]error, 0, len(stages)) + for _, stage := range stages { + errs = append(errs, stopStage(ctx, stage)) + } + + return errors.Join(errs...) +} + +// stopStage stops the services of one stage concurrently. +func stopStage(ctx context.Context, stage plannedStage) error { + if stage.budget > 0 { + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(ctx, stage.budget) + defer cancel() + } + + errs := make([]error, len(stage.entries)) + var wg sync.WaitGroup + for i, e := range stage.entries { + wg.Go(func() { + if err := e.service.Shutdown(ctx); err != nil { + errs[i] = fmt.Errorf("stop %s: %w", e.name, err) + } + }) + } + wg.Wait() + + return errors.Join(errs...) +} diff --git a/app_test.go b/app_test.go new file mode 100644 index 0000000..6baa633 --- /dev/null +++ b/app_test.go @@ -0,0 +1,405 @@ +package lifecycle + +import ( + "context" + "errors" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestApp_Start_StartsInRegistrationOrder(t *testing.T) { + t.Parallel() + log := &eventLog{} + app := newApp(t, log, nil, "store", "worker", "api") + + require.NoError(t, app.Start(t.Context())) + + assert.Equal(t, []string{"start store", "start worker", "start api"}, log.snapshot()) +} + +func TestApp_Start_RollsBackOnlyWhatStarted(t *testing.T) { + t.Parallel() + log := &eventLog{} + app := New() + require.NoError(t, app.Add("store", &fakeService{name: "store", log: log})) + require.NoError(t, app.Add("worker", &fakeService{name: "worker", log: log, shutdownErr: errors.New("worker stuck")})) + require.NoError(t, app.Add("api", &fakeService{name: "api", log: log, startErr: errBoom})) + require.NoError(t, app.Add("never", &fakeService{name: "never", log: log})) + + err := app.Start(t.Context()) + + require.ErrorIs(t, err, errBoom) + require.ErrorContains(t, err, "start api") + require.ErrorContains(t, err, "stop worker: worker stuck", "a rollback error is joined with the start error") + assert.Equal(t, []string{ + "start store", "start worker", "start-failed api", + "stop worker", "stop store", + }, log.snapshot(), "only started services are stopped, in reverse, and the failed one is not") + _, readyErr := app.Readiness().Ready(t.Context()) + require.Error(t, readyErr, "an App that failed to start never becomes ready") +} + +func TestApp_Start_RollsBackWhenItsContextIsDone(t *testing.T) { + t.Parallel() + log := &eventLog{} + ctx, cancel := context.WithCancel(t.Context()) + app := New() + require.NoError(t, app.Add("store", &fakeService{name: "store", log: log, onShutdown: func(ctx context.Context) error { + return ctx.Err() + }})) + require.NoError(t, app.Add("api", &fakeService{name: "api", log: log, startErr: context.Canceled})) + cancel() + + err := app.Start(ctx) + + require.ErrorIs(t, err, context.Canceled) + assert.NotContains(t, err.Error(), "stop store", "the rollback does not inherit the cancelled context") +} + +func TestApp_Start_OnlyOnce(t *testing.T) { + t.Parallel() + app := newApp(t, &eventLog{}, nil, "api") + require.NoError(t, app.Start(t.Context())) + + require.ErrorIs(t, app.Start(t.Context()), ErrAlreadyStarted) + require.ErrorIs(t, app.Add("late", &fakeService{log: &eventLog{}}), ErrAlreadyStarted) +} + +func TestApp_Add_RejectsBadServices(t *testing.T) { + t.Parallel() + app := New() + require.NoError(t, app.Add("api", &fakeService{log: &eventLog{}})) + + require.ErrorContains(t, app.Add("", &fakeService{log: &eventLog{}}), "name is empty") + require.ErrorContains(t, app.Add("nil", nil), "is nil") + require.ErrorContains(t, app.Add("api", &fakeService{log: &eventLog{}}), "already added") +} + +func TestApp_Start_RejectsBadStages(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + stages []Stage + wantErr string + }{ + {"unknown service", []Stage{{Names: []string{"missing"}}}, `"missing", which is not a service`}, + {"service in two stages", []Stage{{Names: []string{"api"}}, {Names: []string{"api"}}}, "more than one stop stage"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + log := &eventLog{} + app := newApp(t, log, []Option{WithStopStages(tt.stages...)}, "api") + + require.ErrorContains(t, app.Start(t.Context()), tt.wantErr) + assert.Empty(t, log.snapshot(), "nothing is started for a configuration that is wrong") + }) + } +} + +func TestApp_Stop_DefaultsToReverseRegistrationOrder(t *testing.T) { + t.Parallel() + log := &eventLog{} + app := newApp(t, log, nil, "store", "worker", "api") + require.NoError(t, app.Start(t.Context())) + + require.NoError(t, app.Stop(t.Context())) + + assert.Equal(t, []string{"stop api", "stop worker", "stop store"}, log.snapshot()[3:]) +} + +func TestApp_Stop_RunsStagesInOrderAndAStageConcurrently(t *testing.T) { + t.Parallel() + log := &eventLog{} + // Each service in the first stage waits for the other to have begun + // stopping, which only happens if they are stopped concurrently. + var begun sync.WaitGroup + begun.Add(2) + meet := func(ctx context.Context) error { + begun.Done() + done := make(chan struct{}) + go func() { begun.Wait(); close(done) }() + select { + case <-done: + return nil + case <-ctx.Done(): + return ctx.Err() + } + } + app := New(WithStopStages( + Stage{Names: []string{"api", "grpc"}}, + Stage{Names: []string{"worker"}}, + )) + require.NoError(t, app.Add("store", &fakeService{name: "store", log: log})) + require.NoError(t, app.Add("worker", &fakeService{name: "worker", log: log})) + require.NoError(t, app.Add("api", &fakeService{name: "api", log: log, onShutdown: meet})) + require.NoError(t, app.Add("grpc", &fakeService{name: "grpc", log: log, onShutdown: meet})) + require.NoError(t, app.Start(t.Context())) + + within(t, func() { require.NoError(t, app.Stop(t.Context())) }) + + stops := log.snapshot()[4:] + assert.ElementsMatch(t, []string{"stop api", "stop grpc"}, stops[:2], "the first stage stops first") + assert.Equal(t, []string{"stop worker", "stop store"}, stops[2:], "then the next stage, then what no stage names") +} + +func TestApp_Stop_AStageBudgetCutsOffASlowServiceAndTheNextStageRuns(t *testing.T) { + t.Parallel() + log := &eventLog{} + app := New(WithStopStages( + Stage{Names: []string{"slow"}, Budget: 20 * time.Millisecond}, + Stage{Names: []string{"store"}}, + )) + require.NoError(t, app.Add("store", &fakeService{name: "store", log: log})) + require.NoError(t, app.Add("slow", &fakeService{name: "slow", log: log, onShutdown: func(ctx context.Context) error { + <-ctx.Done() + + return ctx.Err() + }})) + require.NoError(t, app.Start(t.Context())) + + var err error + within(t, func() { err = app.Stop(t.Context()) }) + + require.ErrorIs(t, err, context.DeadlineExceeded) + require.ErrorContains(t, err, "stop slow") + assert.Equal(t, []string{"stop slow", "stop store"}, log.snapshot()[2:]) +} + +func TestApp_Stop_TheStopTimeoutBoundsEveryStage(t *testing.T) { + t.Parallel() + log := &eventLog{} + blockUntilDone := func(ctx context.Context) error { + <-ctx.Done() + + return ctx.Err() + } + app := New(WithStopTimeout(30 * time.Millisecond)) + require.NoError(t, app.Add("a", &fakeService{name: "a", log: log, onShutdown: blockUntilDone})) + require.NoError(t, app.Add("b", &fakeService{name: "b", log: log, onShutdown: blockUntilDone})) + require.NoError(t, app.Start(t.Context())) + + var err error + within(t, func() { err = app.Stop(t.Context()) }) + + require.ErrorIs(t, err, context.DeadlineExceeded) + require.ErrorContains(t, err, "stop a") + require.ErrorContains(t, err, "stop b", "a later stage still runs, and finds the deadline passed") +} + +func TestApp_Stop_JoinsTheErrorsOfEveryService(t *testing.T) { + t.Parallel() + log := &eventLog{} + app := New() + require.NoError(t, app.Add("a", &fakeService{name: "a", log: log, shutdownErr: errors.New("a failed")})) + require.NoError(t, app.Add("b", &fakeService{name: "b", log: log, shutdownErr: errors.New("b failed")})) + require.NoError(t, app.Start(t.Context())) + + err := app.Stop(t.Context()) + + require.ErrorContains(t, err, "stop a: a failed") + require.ErrorContains(t, err, "stop b: b failed") +} + +func TestApp_Stop_IsANoOpUnlessStarted(t *testing.T) { + t.Parallel() + log := &eventLog{} + app := newApp(t, log, nil, "api") + + require.NoError(t, app.Stop(t.Context()), "never started") + assert.Empty(t, log.snapshot()) + + require.NoError(t, app.Start(t.Context())) + require.NoError(t, app.Stop(t.Context())) + require.NoError(t, app.Stop(t.Context()), "already stopped") + assert.Equal(t, []string{"start api", "stop api"}, log.snapshot(), "the service is stopped once") +} + +func TestApp_Stop_ReturnsWhenNoServiceEverFails(t *testing.T) { + t.Parallel() + // A Failed channel does not receive or close for a service that does not + // fail. Stop must still return, which means the watcher on that channel + // has ended: a leaked watcher would hang the wait. + app := New() + svc := newFailingService("worker", &eventLog{}) + require.NoError(t, app.Add("worker", svc)) + require.NoError(t, app.Start(t.Context())) + + within(t, func() { require.NoError(t, app.Stop(t.Context())) }) +} + +func TestApp_Failed_ReportsAServiceErrorByName(t *testing.T) { + t.Parallel() + app := New() + svc := newFailingService("worker", &eventLog{}) + require.NoError(t, app.Add("worker", svc)) + require.NoError(t, app.Add("api", &fakeService{name: "api", log: &eventLog{}})) + require.NoError(t, app.Start(t.Context())) + t.Cleanup(func() { _ = app.Stop(context.Background()) }) + + svc.failed <- errBoom + + select { + case err := <-app.Failed(): + failure, ok := errors.AsType[*ServiceError](err) + require.True(t, ok) + assert.Equal(t, "worker", failure.Service) + require.ErrorIs(t, err, errBoom) + case <-time.After(5 * time.Second): + t.Fatal("the failure was not reported") + } +} + +func TestApp_Live_NamesTheServicesThatAreNotLive(t *testing.T) { + t.Parallel() + app := New() + good := newFailingService("good", &eventLog{}) + bad := newFailingService("bad", &eventLog{}) + bad.live = errBoom + require.NoError(t, app.Add("good", good)) + require.NoError(t, app.Add("bad", bad)) + require.NoError(t, app.Add("plain", &fakeService{name: "plain", log: &eventLog{}})) + require.NoError(t, app.Start(t.Context())) + t.Cleanup(func() { _ = app.Stop(context.Background()) }) + + err := app.Live(t.Context()) + + require.ErrorIs(t, err, errBoom) + require.ErrorContains(t, err, "bad: boom") + assert.NotContains(t, err.Error(), "good") +} + +func TestApp_ServiceReadinessIsACheckOfThatName(t *testing.T) { + t.Parallel() + app := New() + svc := newFailingService("store", &eventLog{}) + svc.ready = errors.New("replica behind") + require.NoError(t, app.Add("store", svc)) + require.NoError(t, app.Start(t.Context())) + t.Cleanup(func() { _ = app.Stop(context.Background()) }) + + status, err := app.Readiness().Ready(t.Context()) + + require.ErrorContains(t, err, "store: replica behind") + assert.Equal(t, map[string]string{"store": "failed: replica behind"}, status) +} + +func TestApp_Run_OnCancelDrainsThenStopsAndReturnsNil(t *testing.T) { + t.Parallel() + log := &eventLog{} + app := New(WithDrainDelay(20 * time.Millisecond)) + var readyWhenStopping error + require.NoError(t, app.Add("api", &fakeService{name: "api", log: log, onShutdown: func(ctx context.Context) error { + _, readyWhenStopping = app.Readiness().Ready(ctx) + + return nil + }})) + ctx, cancel := context.WithCancel(t.Context()) + result := make(chan error, 1) + go func() { result <- app.Run(ctx) }() + + require.Eventually(t, func() bool { + _, err := app.Readiness().Ready(t.Context()) + + return err == nil + }, 5*time.Second, time.Millisecond, "the App becomes ready once started") + cancel() + + select { + case err := <-result: + require.NoError(t, err) + case <-time.After(5 * time.Second): + t.Fatal("Run did not return") + } + assert.Equal(t, []string{"start api", "stop api"}, log.snapshot()) + require.ErrorIs(t, readyWhenStopping, ErrDraining) +} + +func TestApp_Run_OnFailureStopsAtOnceWithoutDraining(t *testing.T) { + t.Parallel() + log := &eventLog{} + // A drain delay this long would fail the test if the failure were drained. + app := New(WithDrainDelay(time.Hour)) + svc := newFailingService("worker", log) + svc.shutdownErr = errors.New("worker stuck") + require.NoError(t, app.Add("worker", svc)) + require.NoError(t, app.Add("api", &fakeService{name: "api", log: log})) + result := make(chan error, 1) + go func() { result <- app.Run(t.Context()) }() + require.Eventually(t, func() bool { return len(log.snapshot()) == 2 }, 5*time.Second, time.Millisecond) + + svc.failed <- errBoom + + select { + case err := <-result: + failure, ok := errors.AsType[*ServiceError](err) + require.True(t, ok, "Run returns the failure") + assert.Equal(t, "worker", failure.Service) + require.ErrorIs(t, err, errBoom) + require.ErrorContains(t, err, "stop worker: worker stuck", "joined with the stop errors") + case <-time.After(5 * time.Second): + t.Fatal("Run did not return: the failure was drained") + } + assert.Equal(t, []string{"start worker", "start api", "stop api", "stop worker"}, log.snapshot()) +} + +func TestApp_Run_ReturnsAStartFailure(t *testing.T) { + t.Parallel() + log := &eventLog{} + app := New() + require.NoError(t, app.Add("store", &fakeService{name: "store", log: log})) + require.NoError(t, app.Add("api", &fakeService{name: "api", log: log, startErr: errBoom})) + + err := app.Run(t.Context()) + + require.ErrorIs(t, err, errBoom) + assert.Equal(t, []string{"start store", "start-failed api", "stop store"}, log.snapshot()) +} + +func TestApp_Run_StopIsNotCutShortByACancelledContext(t *testing.T) { + t.Parallel() + log := &eventLog{} + app := New() + var stopCtxErr error + require.NoError(t, app.Add("api", &fakeService{name: "api", log: log, onShutdown: func(ctx context.Context) error { + stopCtxErr = ctx.Err() + + return nil + }})) + ctx, cancel := context.WithCancel(t.Context()) + cancel() + + require.NoError(t, app.Run(ctx)) + + require.NoError(t, stopCtxErr, "Shutdown gets a live context although Run's was cancelled") +} + +func TestApp_Drain_WaitsForTheDelayOrTheContext(t *testing.T) { + t.Parallel() + + t.Run("the delay", func(t *testing.T) { + t.Parallel() + app := New(WithDrainDelay(30 * time.Millisecond)) + + start := time.Now() + app.Drain(t.Context()) + + assert.GreaterOrEqual(t, time.Since(start), 30*time.Millisecond) + _, err := app.Readiness().Ready(t.Context()) + require.ErrorIs(t, err, ErrDraining) + }) + + t.Run("a context that is done", func(t *testing.T) { + t.Parallel() + app := New(WithDrainDelay(time.Hour)) + ctx, cancel := context.WithCancel(t.Context()) + cancel() + + within(t, func() { app.Drain(ctx) }) + }) +} diff --git a/doc.go b/doc.go new file mode 100644 index 0000000..c948abf --- /dev/null +++ b/doc.go @@ -0,0 +1,65 @@ +// Package lifecycle runs the services of a Dioad daemon: it starts them in +// order, rolling back on a failure; fans their failures into one channel; +// reports readiness; drains; and stops them in stages within a deadline. +// +// A daemon registers each service with an App, then calls Run: +// +// app := lifecycle.New(lifecycle.WithDrainDelay(5 * time.Second)) +// if err := app.Add("store", store); err != nil { +// return err +// } +// if err := app.Add("api", apiServer); err != nil { +// return err +// } +// return app.Run(ctx) +// +// # Services +// +// A Service has Start and Shutdown. A service that can also fail after it has +// started implements Failer, and one that can report that it has stopped +// working implements Liveness. A service that implements Ready adds a check +// of that name to the App's Readiness. All three are discovered by interface +// assertion, so a service implements only what it has. +// +// The method set matches the one dioad/net's http.Server has (Start, +// Shutdown, Failed and Live), so a *http.Server is a Service with no adapter. +// It is Shutdown, not Stop, for that reason. A service's name is given when it +// is registered, not by the service. +// +// # Starting +// +// Start starts services in registration order. If one fails, the services +// that had already started are stopped, in the same way Stop would stop them, +// and Start returns the start error joined with any stop errors. A service +// whose Start fails is not stopped, so Start must clean up after itself. +// +// # Stopping +// +// Stop stops services in stages. A stage is a set of services stopped +// concurrently, optionally within a budget; the stages run one after another, +// all within one overall deadline (WithStopTimeout). With no stages +// configured every service is its own stage, stopped in reverse registration +// order. A service that no stage names is stopped after the stages, in reverse +// registration order. Errors are combined with errors.Join. +// +// A service's Shutdown must return when its context is done. The App cannot +// stop a Shutdown that ignores it. +// +// # Failure and drain +// +// A failure of a service that implements Failer is reported on App.Failed, +// wrapped in a ServiceError. Run, on a failure, stops the services at once; +// when its context is done instead, it first drains: Readiness reports not +// ready, and Run waits for the drain delay (WithDrainDelay) so that a load +// balancer stops sending requests before the listeners close. +// +// # Readiness +// +// Readiness is a health source: it has the Ready method that dioad/net's +// http.Server.AddHealthSource looks for, so it is reported by /health/ready +// without this package importing dioad/net. It is not ready until Start has +// returned (the start gate) and not ready once draining. In between it runs +// its named checks. A check can be cached for a time to live, and can report +// itself degraded, in which case the process stays ready and the check is +// reported. +package lifecycle diff --git a/example_test.go b/example_test.go new file mode 100644 index 0000000..6bf60a9 --- /dev/null +++ b/example_test.go @@ -0,0 +1,127 @@ +package lifecycle_test + +import ( + "context" + "errors" + "fmt" + "time" + + "github.com/dioad/lifecycle" +) + +// printer is a Service that prints what it is asked to do. +type printer struct { + name string + startErr error +} + +func (p printer) Start(context.Context) error { + if p.startErr != nil { + fmt.Println("start", p.name, "failed") + + return p.startErr + } + fmt.Println("start", p.name) + + return nil +} + +func (p printer) Shutdown(context.Context) error { + fmt.Println("stop", p.name) + + return nil +} + +// Services start in the order they are added. When one fails to start, the +// services that had already started are stopped, newest first, and Start +// returns the failure. +func ExampleApp() { + app := lifecycle.New() + for _, service := range []printer{ + {name: "store"}, + {name: "queue"}, + {name: "api", startErr: errors.New("address in use")}, + {name: "metrics"}, + } { + if err := app.Add(service.name, service); err != nil { + fmt.Println(err) + + return + } + } + + fmt.Println("result:", app.Start(context.Background())) + // Output: + // start store + // start queue + // start api failed + // stop queue + // stop store + // result: start api: address in use +} + +// Stop stops the services in stages. The api stops first, so that nothing is +// sending the worker work; the store, which no stage names, stops last. +func ExampleApp_Stop() { + app := lifecycle.New(lifecycle.WithStopStages( + lifecycle.Stage{Names: []string{"api"}, Budget: time.Second}, + lifecycle.Stage{Names: []string{"worker"}}, + )) + for _, name := range []string{"store", "worker", "api"} { + if err := app.Add(name, printer{name: name}); err != nil { + fmt.Println(err) + + return + } + } + if err := app.Start(context.Background()); err != nil { + fmt.Println(err) + + return + } + + fmt.Println("result:", app.Stop(context.Background())) + // Output: + // start store + // start worker + // start api + // stop api + // stop worker + // stop store + // result: +} + +// A process is not ready until it has started, stays ready while a check is +// degraded, is not ready when a check fails, and is not ready once it is +// draining, so that traffic stops before its listeners close. +func ExampleReadiness() { + readiness := lifecycle.NewReadiness() + var searchErr, dbErr error + readiness.AddCheck("db", func(context.Context) error { return dbErr }) + readiness.AddCheck("search", func(context.Context) error { return lifecycle.Degraded(searchErr) }) + + report := func(when string) { + status, err := readiness.Ready(context.Background()) + fmt.Printf("%s: status=%v err=%v\n", when, status, err) + } + + report("before start") + readiness.MarkStarted() + report("started") + + searchErr = errors.New("slow") + report("search degraded") + + dbErr = errors.New("connection refused") + report("db down") + + dbErr = nil + readiness.BeginDrain() + report("draining") + // Output: + // before start: status= err=starting + // started: status=map[db:ok search:ok] err= + // search degraded: status=map[db:ok search:degraded: slow] err= + // db down: status=map[db:failed: connection refused search:degraded: slow] err=db: connection refused + // draining: status= err=draining +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..77e1a38 --- /dev/null +++ b/go.mod @@ -0,0 +1,7 @@ +module github.com/dioad/lifecycle + +go 1.27.0 + +require github.com/stretchr/testify v1.12.1 + +require go.yaml.in/yaml/v3 v3.0.5 // indirect diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..c233683 --- /dev/null +++ b/go.sum @@ -0,0 +1,4 @@ +github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE= +github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg= +go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= +go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= diff --git a/helpers_test.go b/helpers_test.go new file mode 100644 index 0000000..558c122 --- /dev/null +++ b/helpers_test.go @@ -0,0 +1,109 @@ +package lifecycle + +import ( + "context" + "errors" + "slices" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +// eventLog records what the fake services were asked to do, in order. +type eventLog struct { + mu sync.Mutex + events []string +} + +func (l *eventLog) add(event string) { + l.mu.Lock() + defer l.mu.Unlock() + l.events = append(l.events, event) +} + +func (l *eventLog) snapshot() []string { + l.mu.Lock() + defer l.mu.Unlock() + + return slices.Clone(l.events) +} + +// fakeService is a Service that records its calls and can be told to fail. +type fakeService struct { + name string + log *eventLog + startErr error + shutdownErr error + // onShutdown, if set, runs inside Shutdown, with the context it was given. + onShutdown func(ctx context.Context) error +} + +func (s *fakeService) Start(context.Context) error { + if s.startErr != nil { + s.log.add("start-failed " + s.name) + + return s.startErr + } + s.log.add("start " + s.name) + + return nil +} + +func (s *fakeService) Shutdown(ctx context.Context) error { + s.log.add("stop " + s.name) + if s.onShutdown != nil { + return s.onShutdown(ctx) + } + + return s.shutdownErr +} + +// failingService is a fakeService that can fail after it has started, and +// can report liveness and readiness. +type failingService struct { + fakeService + + failed chan error + live error + ready error +} + +func newFailingService(name string, log *eventLog) *failingService { + return &failingService{name: name, log: log, failed: make(chan error, 1)} +} + +func (s *failingService) Failed() <-chan error { return s.failed } +func (s *failingService) Live(context.Context) error { return s.live } +func (s *failingService) Ready(context.Context) (any, error) { return "checked", s.ready } + +// newApp returns an App with the named fake services added. +func newApp(t *testing.T, log *eventLog, opts []Option, names ...string) *App { + t.Helper() + + app := New(opts...) + for _, name := range names { + require.NoError(t, app.Add(name, &fakeService{name: name, log: log})) + } + + return app +} + +// within fails the test if fn does not return in time. +func within(t *testing.T, fn func()) { + t.Helper() + + done := make(chan struct{}) + go func() { + defer close(done) + fn() + }() + select { + case <-done: + case <-time.After(5 * time.Second): + t.Fatal("did not return in time") + } +} + +var errBoom = errors.New("boom") diff --git a/readiness.go b/readiness.go new file mode 100644 index 0000000..a7b60ca --- /dev/null +++ b/readiness.go @@ -0,0 +1,211 @@ +package lifecycle + +import ( + "context" + "errors" + "slices" + "strings" + "sync" + "time" +) + +// CheckFunc reports whether one thing a process depends on is working. It +// returns nil when it is, and an error saying why not when it is not. Wrap the +// error in Degraded to report a problem that does not stop the process taking +// traffic. +type CheckFunc func(ctx context.Context) error + +// Degraded marks err as a degradation: the check that returned it is reported +// by Readiness, but the process stays ready. It returns nil if err is nil. +func Degraded(err error) error { + if err == nil { + return nil + } + + return °radedError{err: err} +} + +type degradedError struct{ err error } + +func (e *degradedError) Error() string { return e.err.Error() } +func (e *degradedError) Unwrap() error { return e.err } + +// Readiness reports whether a process should be sent traffic. It is not ready +// until MarkStarted, not ready once BeginDrain, and in between ready unless a +// check fails. +// +// It has the Ready method that dioad/net's http.Server.AddHealthSource looks +// for. The zero value is not usable; call NewReadiness. +type Readiness struct { + now func() time.Time + + mu sync.Mutex + state readinessState + checks []*check +} + +type readinessState int + +const ( + stateStarting readinessState = iota + stateReady + stateDraining +) + +// ReadinessOption configures a Readiness. +type ReadinessOption func(*Readiness) + +// WithClock sets the clock that check results are cached against. It is for +// tests. +func WithClock(now func() time.Time) ReadinessOption { + return func(r *Readiness) { r.now = now } +} + +// NewReadiness returns a Readiness that is not ready until MarkStarted. +func NewReadiness(opts ...ReadinessOption) *Readiness { + r := &Readiness{now: time.Now} + for _, opt := range opts { + opt(r) + } + + return r +} + +// CheckOption configures a check added with AddCheck. +type CheckOption func(*check) + +// WithCacheTTL caches the check's result for ttl, so that a health probe that +// arrives every second does not query a database every second. A result is +// not cached when the probe's context is done, since that says nothing about +// the thing checked. Without it every Ready runs the check. +func WithCacheTTL(ttl time.Duration) CheckOption { + return func(c *check) { c.ttl = ttl } +} + +// AddCheck adds a named check. Checks run in the order they were added. +func (r *Readiness) AddCheck(name string, fn CheckFunc, opts ...CheckOption) { + c := &check{name: name, fn: fn} + for _, opt := range opts { + opt(c) + } + + r.mu.Lock() + defer r.mu.Unlock() + r.checks = append(r.checks, c) +} + +// MarkStarted opens the start gate: from now on Ready runs the checks. App +// does this when every service has started. +func (r *Readiness) MarkStarted() { + r.mu.Lock() + defer r.mu.Unlock() + if r.state == stateStarting { + r.state = stateReady + } +} + +// BeginDrain makes Ready report not ready from now on, so that traffic stops +// before listeners close. App does this when it drains or stops. +func (r *Readiness) BeginDrain() { + r.mu.Lock() + defer r.mu.Unlock() + r.state = stateDraining +} + +// Ready reports whether the process should be sent traffic. The first result +// is a status for each check, by name: "ok", "degraded: " or +// "failed: ". The error is non-nil before MarkStarted (ErrStarting), +// after BeginDrain (ErrDraining), and when a check has failed; a degraded +// check does not make it non-nil. +func (r *Readiness) Ready(ctx context.Context) (any, error) { + r.mu.Lock() + state := r.state + checks := slices.Clone(r.checks) + r.mu.Unlock() + + switch state { + case stateStarting: + return nil, ErrStarting + case stateDraining: + return nil, ErrDraining + case stateReady: + } + + status := make(map[string]string, len(checks)) + var failures checkErrors + for _, c := range checks { + err := c.run(ctx, r.now) + switch { + case err == nil: + status[c.name] = "ok" + case isDegraded(err): + status[c.name] = "degraded: " + err.Error() + default: + status[c.name] = "failed: " + err.Error() + failures = append(failures, &namedError{name: c.name, err: err}) + } + } + if len(failures) > 0 { + return status, failures + } + + return status, nil +} + +func isDegraded(err error) bool { + _, ok := errors.AsType[*degradedError](err) + + return ok +} + +// check is one named CheckFunc and its cached result. +type check struct { + name string + fn CheckFunc + ttl time.Duration + + mu sync.Mutex + cached bool + at time.Time + err error +} + +// run returns the check's result, from the cache while it is fresh. It holds +// the check's lock while the check runs, so concurrent probes share one run. +func (c *check) run(ctx context.Context, now func() time.Time) error { + c.mu.Lock() + defer c.mu.Unlock() + + if c.ttl > 0 && c.cached && now().Sub(c.at) < c.ttl { + return c.err + } + + err := c.fn(ctx) + if c.ttl > 0 && ctx.Err() == nil { + c.cached, c.at, c.err = true, now(), err + } + + return err +} + +type namedError struct { + name string + err error +} + +func (e *namedError) Error() string { return e.name + ": " + e.err.Error() } +func (e *namedError) Unwrap() error { return e.err } + +// checkErrors is the error Ready returns when checks have failed. +type checkErrors []error + +func (f checkErrors) Error() string { + messages := make([]string, len(f)) + for i, err := range f { + messages[i] = err.Error() + } + + return strings.Join(messages, "; ") +} + +func (f checkErrors) Unwrap() []error { return f } diff --git a/readiness_test.go b/readiness_test.go new file mode 100644 index 0000000..23e32e3 --- /dev/null +++ b/readiness_test.go @@ -0,0 +1,149 @@ +package lifecycle + +import ( + "context" + "errors" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// fakeClock is a clock tests move by hand. +type fakeClock struct{ now atomic.Pointer[time.Time] } + +func newFakeClock() *fakeClock { + c := &fakeClock{} + c.set(time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)) + + return c +} + +func (c *fakeClock) set(t time.Time) { c.now.Store(&t) } +func (c *fakeClock) Now() time.Time { return *c.now.Load() } +func (c *fakeClock) advance(d time.Duration) { c.set(c.Now().Add(d)) } + +func TestReadiness_IsNotReadyUntilStartedAndNotReadyOnceDraining(t *testing.T) { + t.Parallel() + r := NewReadiness() + + _, err := r.Ready(t.Context()) + require.ErrorIs(t, err, ErrStarting, "the start gate") + + r.MarkStarted() + _, err = r.Ready(t.Context()) + require.NoError(t, err) + + r.BeginDrain() + _, err = r.Ready(t.Context()) + require.ErrorIs(t, err, ErrDraining) + + r.MarkStarted() + _, err = r.Ready(t.Context()) + require.ErrorIs(t, err, ErrDraining, "a drain is not undone") +} + +func TestReadiness_ChecksRunInOrderAndAFailureMakesItNotReady(t *testing.T) { + t.Parallel() + r := NewReadiness() + r.AddCheck("db", func(context.Context) error { return nil }) + r.AddCheck("cache", func(context.Context) error { return errors.New("connection refused") }) + r.AddCheck("queue", func(context.Context) error { return errors.New("full") }) + r.MarkStarted() + + status, err := r.Ready(t.Context()) + + require.EqualError(t, err, "cache: connection refused; queue: full") + assert.Equal(t, map[string]string{"db": "ok", "cache": "failed: connection refused", "queue": "failed: full"}, status) +} + +func TestReadiness_ADegradedCheckIsReportedButTheProcessStaysReady(t *testing.T) { + t.Parallel() + r := NewReadiness() + r.AddCheck("db", func(context.Context) error { return nil }) + r.AddCheck("search", func(context.Context) error { return Degraded(errors.New("slow")) }) + r.MarkStarted() + + status, err := r.Ready(t.Context()) + + require.NoError(t, err) + assert.Equal(t, map[string]string{"db": "ok", "search": "degraded: slow"}, status) +} + +func TestDegraded(t *testing.T) { + t.Parallel() + + require.NoError(t, Degraded(nil)) + + err := Degraded(errBoom) + require.ErrorIs(t, err, errBoom) + assert.EqualError(t, err, "boom") +} + +func TestReadiness_CachesAResultForItsTTL(t *testing.T) { + t.Parallel() + clock := newFakeClock() + r := NewReadiness(WithClock(clock.Now)) + var runs atomic.Int32 + var failure atomic.Pointer[error] + r.AddCheck("db", func(context.Context) error { + runs.Add(1) + if err := failure.Load(); err != nil { + return *err + } + + return nil + }, WithCacheTTL(10*time.Second)) + r.MarkStarted() + + _, err := r.Ready(t.Context()) + require.NoError(t, err) + failing := errors.New("down") + failure.Store(&failing) + clock.advance(9 * time.Second) + _, err = r.Ready(t.Context()) + require.NoError(t, err, "the cached result is served inside the TTL") + assert.Equal(t, int32(1), runs.Load()) + + clock.advance(time.Second) + _, err = r.Ready(t.Context()) + require.ErrorContains(t, err, "db: down", "the TTL has passed") + assert.Equal(t, int32(2), runs.Load()) + + failure.Store(nil) + _, err = r.Ready(t.Context()) + require.ErrorContains(t, err, "db: down", "a failure is cached too") + assert.Equal(t, int32(2), runs.Load()) +} + +func TestReadiness_WithoutATTLEveryReadyRunsTheCheck(t *testing.T) { + t.Parallel() + r := NewReadiness() + var runs atomic.Int32 + r.AddCheck("db", func(context.Context) error { runs.Add(1); return nil }) + r.MarkStarted() + + _, _ = r.Ready(t.Context()) + _, _ = r.Ready(t.Context()) + + assert.Equal(t, int32(2), runs.Load()) +} + +func TestReadiness_DoesNotCacheAResultFromACancelledProbe(t *testing.T) { + t.Parallel() + r := NewReadiness() + var runs atomic.Int32 + r.AddCheck("db", func(ctx context.Context) error { runs.Add(1); return ctx.Err() }, WithCacheTTL(time.Hour)) + r.MarkStarted() + cancelled, cancel := context.WithCancel(t.Context()) + cancel() + + _, err := r.Ready(cancelled) + require.ErrorIs(t, err, context.Canceled) + _, err = r.Ready(t.Context()) + + require.NoError(t, err, "the next probe runs the check again") + assert.Equal(t, int32(2), runs.Load()) +} diff --git a/service.go b/service.go new file mode 100644 index 0000000..8b711e7 --- /dev/null +++ b/service.go @@ -0,0 +1,65 @@ +package lifecycle + +import ( + "context" + "errors" + "fmt" +) + +// Service is something an App starts and stops: an HTTP server, a worker, a +// publisher, a store. +type Service interface { + // Start brings the service up and returns once it is serving. It must not + // block for the life of the service. + Start(ctx context.Context) error + // Shutdown stops the service. It must return when ctx is done. + Shutdown(ctx context.Context) error +} + +// Failer is implemented by a Service that can stop working after Start has +// returned. Failed returns a channel that receives, at most once, the error +// that ended the service. It need not close it. +type Failer interface { + Failed() <-chan error +} + +// Liveness is implemented by a Service that can report that it has stopped +// working. Live returns nil while the service is working. +type Liveness interface { + Live(ctx context.Context) error +} + +// readyChecker is implemented by a Service that can report whether it is +// ready. It is the method dioad/net calls a readiness resource. +type readyChecker interface { + Ready(ctx context.Context) (any, error) +} + +var ( + // ErrAlreadyStarted is returned by Start, and by Add, once the App has + // been started. + ErrAlreadyStarted = errors.New("lifecycle: app already started") + // ErrStarting is the reason Readiness reports before the App has started. + ErrStarting = errors.New("starting") + // ErrDraining is the reason Readiness reports once the App is draining or + // stopping. + ErrDraining = errors.New("draining") +) + +// ServiceError is a failure of a service, as reported on App.Failed. +type ServiceError struct { + // Service is the name the service was registered under. + Service string + // Err is the error the service reported. + Err error +} + +// Error implements error. +func (f *ServiceError) Error() string { + return fmt.Sprintf("service %s failed: %v", f.Service, f.Err) +} + +// Unwrap returns the error the service reported. +func (f *ServiceError) Unwrap() error { + return f.Err +}