From 281e52333aab96ed5a05ba2df0bb8293ca0f5d56 Mon Sep 17 00:00:00 2001 From: Antoine Toussaint <312536060+antoinetoussaint-byte@users.noreply.github.com> Date: Mon, 12 Oct 2026 21:05:39 -0400 Subject: [PATCH] fix: a boot waits for a projection that is not there yet; the uninstalled asker survives a transient answer A projected token that is absent ended the process on the broker path. The platform projects the token and a process can be running before it has; nothing is sent without one, so an absent projection is now unavailable and a boot waits for it inside its window. One that is present and unusable stays terminal. awaitInstallation stopped asking after a single transient answer, leaving the first credential to the host's next install delivery. It now asks for as long as no credential is held and the host has said the solution is installed nowhere. Whether the process may serve uninstalled is unchanged and stricter. Review follow-ups from #65: displaced doc comments, a README sentence that overstated the no-fallback rule, a CHANGELOG entry. chore(scout): the test listener binds the loopback address the tests dial. A wildcard bind succeeded on a port a neighbour held on 127.0.0.1 and the client reached the neighbour. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 14 ++++++++ README.md | 25 ++++++++------ boot_test.go | 6 +++- broker_credential.go | 3 ++ credential.go | 23 +++++++++++-- mint_errors.go | 34 ++++++++++++++++--- mint_migration_test.go | 9 ++++- solution.go | 34 ++++++++++++------- uninstalled_test.go | 74 ++++++++++++++++++++++++++++++++++++++++++ 9 files changed, 192 insertions(+), 30 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0ed0fa1..f8c047d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,19 @@ # Changelog +## Unreleased — the mint reports the registration's dynamic half; the runtime serves uninstalled + +The mint body carries `upstream` (the advertised https origin, never the listen +address) and `manifest` beside `prefix`, bounded at 64 KiB at boot. A deployed +runtime context refuses a loopback origin. + +`412` from the broker is "admitted, installed nowhere": the runtime serves its +install step and `/health` holding no credential, and asks again until one is +issued. A viewer's route still answers 503. + +A projected token that is not there yet is unavailable, not terminal: a boot +waits for it inside the two-minute window, and a renewal keeps an unexpired +credential. One that is present and unusable stays terminal. + ## Unreleased — native module-broker execution mint Breaking API change: `WorkContextRefusal.Code` and `WorkContextRefusal.Message` diff --git a/README.md b/README.md index c2db60d..a25d7fe 100644 --- a/README.md +++ b/README.md @@ -82,20 +82,24 @@ endpoint — the `CODEFLY__SELF_ENDPOINT______HTTP__HTTP` carri `CODEFLY__ENDPOINT__` carrier to `localhost`, so reporting that would have the gateway proxy the solution to the gateway's own loopback. With no carrier, a process on a developer machine reports the loopback port it actually binds -(`PORT` included). In a deployed runtime context a loopback origin, a missing -carrier and a carrier that does not hold exactly a host and a port each fail the -boot; a present carrier is never treated as absent. +(`PORT` included). In a deployed runtime context a loopback origin fails the +boot — which is what a missing carrier resolves to in a rendered cell — and so +does a carrier that does not hold exactly a host and a port; a present carrier +is never treated as absent. A host that admits this workload but has not been asked to install it anywhere answers `412`. That is not a refusal. The runtime serves holding no credential: `/health` answers `200`, and the [install step](#install-step) runs, because it acts under the capability the host carries in and the host issues this execution's credential only after that step is acknowledged. Neither asks the -host again. The runtime asks every 20 seconds (jittered), and at once — then a -few times in quick succession — after it acknowledges an install step, and holds -the first credential issued. A viewer's route still answers `503` until then. A +host again. The runtime asks every 20 seconds (jittered) for as long as it holds +no credential — through a transient answer too — and at once, then a few times +in quick succession, after it acknowledges an install step. It holds the first +credential issued. A viewer's route still answers `503` until then. A `401`/`403` -stays what it was: a judgement, which fails the boot and is never retried. The answer carries `token`, `expiresAt`, +stays what it was: a judgement, which fails the boot and is never retried. + +The answer carries `token`, `expiresAt`, `principalId`, and `tenant`. Core alone reads the returned capability, through its structural `Decode` (named `Inspect` before core v0.15); the runtime does not implement a signed encoding or verify an issuer's authority, and that read @@ -555,13 +559,14 @@ is waited for, within a bound.** The two are different answers: - A **refusal** (`401` or `403`, wrapping `ErrMintRefused`) is terminal and retained. It is never retried. - An **unavailable** mint (`503`, timeout, connection refused or connection - reset, wrapping `ErrMintUnavailable`) is retried with backoff inside the + reset, or a projected token that is not there yet, wrapping + `ErrMintUnavailable`) is retried with backoff inside the **two-minute** boot deadline. Renewal uses the same classification and keeps an unexpired held credential available during transient failures. - Every other HTTP status, including `400`, `404`, `429`, `500`, and redirects, is a **terminal error**. TLS, permanent DNS, cancellation, unclassified - transport failures, invalid responses and unusable projections are terminal - too. They stop renewal and fail health. Status, status class and transport + transport failures, invalid responses and a projection that is present but + unusable are terminal too. They stop renewal and fail health. Status, status class and transport category survive; raw body, headers, parser diagnostics and causes do not. Every mint makes a fresh TLS 1.3 handshake, presents this workload's identity, diff --git a/boot_test.go b/boot_test.go index 30f7648..bd2df16 100644 --- a/boot_test.go +++ b/boot_test.go @@ -961,7 +961,11 @@ func TestServedContractReportsWhatThisProcessHoldsItselfTo(t *testing.T) { // the port is released by construction. Nothing is released here. func boundPort(t *testing.T) (net.Listener, string) { t.Helper() - ln, err := net.Listen("tcp", ":0") + // On the loopback address the tests dial, not the wildcard. A wildcard + // bind succeeds on a port another local process holds on 127.0.0.1, and + // the test's own client then reaches that process instead: "first record + // does not look like a TLS handshake", from a neighbour. + ln, err := net.Listen("tcp", "127.0.0.1:0") if err != nil { t.Fatal(err) } diff --git a/broker_credential.go b/broker_credential.go index 5610c39..7393932 100644 --- a/broker_credential.go +++ b/broker_credential.go @@ -145,6 +145,9 @@ func (c *brokerCredentialSource) obtain(ctx context.Context) (executionCredentia if err != nil { var projection *projectedTokenError if errors.As(err, &projection) { + if projection.absent { + return workcontext.Credential{}, &brokerMintError{class: brokerProjectionAbsent} + } return workcontext.Credential{}, &brokerMintError{class: brokerProjection} } return workcontext.Credential{}, classifyBrokerTransport(err) diff --git a/credential.go b/credential.go index 1985664..47fa2c6 100644 --- a/credential.go +++ b/credential.go @@ -450,8 +450,6 @@ func (s *Server) openCredential(ctx context.Context) error { } } -// mustDeadline is the deadline of a context that has one. openCredential sets -// it, so the fallback is only there to keep the arithmetic total. // manifestJSON is the frontend manifest this runtime serves, reported with // the mint so the host's registration carries it without a self-registration. func (s *Server) manifestJSON() string { @@ -500,7 +498,7 @@ func (s *Server) awaitInstallation(ctx context.Context) { followUps = awaitInstallationFollowUps case <-idle.C: } - for ask := 0; s.uninstalled(); ask++ { + for ask := 0; s.awaitsInstallation(); ask++ { credential, err := s.askOnce(ctx, platformRequestTimeout) if err == nil { seal := credential.Seal() @@ -576,9 +574,26 @@ func (s *Server) uninstalled() bool { return awaitingInstallation(s.credentialAskErr) } +// awaitsInstallation reports that this process holds no credential and the +// host has said, at some point since it last held one, that nothing has +// installed the solution. +// +// Wider than uninstalled on purpose. One transient answer after a 412 — the +// host restarting, a pass in flight — leaves the solution exactly as +// uninstalled as it was, and an asker that stopped there would leave the +// first credential waiting for the host's next install delivery. +func (s *Server) awaitsInstallation() bool { + if s.terminalErr.Load() != nil || !s.awaiting.Load() { + return false + } + _, held := s.heldCredential() + return !held +} + // noteCredentialAnswer records an answer the controller did not itself // receive — the boot's — so every reader sees the same last answer. func (s *Server) noteCredentialAnswer(err error) { + s.awaiting.Store(awaitingInstallation(err)) s.credentialMu.Lock() s.credentialAskErr = err s.credentialMu.Unlock() @@ -594,6 +609,8 @@ func (s *Server) installAcknowledged() { } } +// mustDeadline is the deadline of a context that has one. openCredential sets +// it, so the fallback is only there to keep the arithmetic total. func mustDeadline(ctx context.Context) time.Time { if at, ok := ctx.Deadline(); ok { return at diff --git a/mint_errors.go b/mint_errors.go index 6eeb86a..e961e92 100644 --- a/mint_errors.go +++ b/mint_errors.go @@ -6,8 +6,10 @@ import ( "crypto/x509" "errors" "fmt" + "io/fs" "net" "net/http" + "os" "syscall" "github.com/codefly-dev/sdk-go/workcontext" @@ -44,12 +46,20 @@ type projectedTokenFile string func (path projectedTokenFile) ProjectedToken() (string, error) { token, err := workcontext.ProjectedTokenFile(path).ProjectedToken() if err != nil { - return "", &projectedTokenError{} + // Absent is told apart from unusable, and only by asking the + // filesystem the one question: is there anything at this path. A + // token that is not there has not been projected yet; one that is + // there and cannot be used is a projection somebody got wrong. + _, statErr := os.Lstat(string(path)) + return "", &projectedTokenError{absent: errors.Is(statErr, fs.ErrNotExist)} } return token, nil } -type projectedTokenError struct{} +type projectedTokenError struct { + // absent: nothing exists at the projection path. + absent bool +} func (*projectedTokenError) Error() string { return "projected service-account token is missing, empty, unreadable or exceeds 65536 bytes" @@ -68,6 +78,10 @@ const ( brokerTransport brokerInvalidAnswer brokerProjection + // brokerProjectionAbsent: there is no token at the projection path. The + // platform has not projected it yet; nothing is sent, and the ask is + // retried. + brokerProjectionAbsent // brokerAwaitingInstallation: the host admits this workload and has not // been asked to install it anywhere yet. The install step arrives on this // runtime's own listener; until then there is nothing to mint for. @@ -102,6 +116,8 @@ func (e *brokerMintError) Error() string { return fmt.Sprintf("module broker: HTTP %d: request failed", e.status) case brokerInvalidAnswer: return "module broker: invalid credential response" + case brokerProjectionAbsent: + return "projected service-account token is missing: it has not been projected yet" case brokerProjection: return "projected service-account token is missing, empty, unreadable or exceeds 65536 bytes" } @@ -118,9 +134,19 @@ func (e *brokerMintError) Error() string { return "module broker: transport or response read failed" } } + +// retryable reports an answer that judges nothing about this build. +// +// A projection that is not there yet is one of them. The platform projects the +// token, and a process can be running before it has; nothing is sent without +// one, so waiting costs no authority: a boot waits inside its bounded window +// and exits as unavailable if the projection never arrives. A projection that +// is there and unusable — empty, oversized, not a file — is a different thing +// and stays terminal: no amount of waiting repairs it. func (e *brokerMintError) retryable() bool { - return e.class == brokerNotEstablished || e.class == brokerAwaitingInstallation || e.class == brokerTransport && - (e.transport == brokerTransportTimeout || e.transport == brokerTransportRefused || e.transport == brokerTransportReset) + return e.class == brokerNotEstablished || e.class == brokerAwaitingInstallation || e.class == brokerProjectionAbsent || + e.class == brokerTransport && + (e.transport == brokerTransportTimeout || e.transport == brokerTransportRefused || e.transport == brokerTransportReset) } func (e *brokerMintError) Unwrap() error { if e.class == brokerRefused { diff --git a/mint_migration_test.go b/mint_migration_test.go index acb6ab9..74aefa3 100644 --- a/mint_migration_test.go +++ b/mint_migration_test.go @@ -95,9 +95,16 @@ func TestUnusableProjectedTokenNeverReachesTheMint(t *testing.T) { mint := newHostMint(t, &hostMint{}) source := productionMintSource(t, mint, path) credential, err := source.Credential(context.Background()) - if credential.Token() != "" || !terminalCredentialFailure(err) { + if credential.Token() != "" || err == nil { t.Fatalf("unusable projection did not fail closed: %v", err) } + // Not there yet is waited for; there and unusable is not. + if wantTerminal := kind != "missing"; terminalCredentialFailure(err) != wantTerminal { + t.Fatalf("a %s projection: terminal = %v, want %v (%v)", kind, !wantTerminal, wantTerminal, err) + } + if kind == "missing" && !errors.Is(err, workcontext.ErrMintUnavailable) { + t.Fatalf("a projection that is not there yet must read as unavailable: %v", err) + } if !strings.Contains(err.Error(), "projected service-account token") || strings.Contains(err.Error(), path) { t.Fatalf("projection error is unclear or discloses its raw path: %v", err) } diff --git a/solution.go b/solution.go index 36037c8..b6640bc 100644 --- a/solution.go +++ b/solution.go @@ -326,7 +326,12 @@ type Server struct { terminalOnce sync.Once // installed signals awaitInstallation that an install step was just // acknowledged. One slot: a pending signal is as good as several. - installed chan struct{} + installed chan struct{} + // awaiting is set once the host has answered that nothing has installed + // this solution, and cleared when a credential is held. It is what keeps + // awaitInstallation asking through a transient answer in between; whether + // the process may SERVE uninstalled is a stricter question (uninstalled). + awaiting atomic.Bool terminalErr atomic.Pointer[error] // The one in-flight credential ask and what it last produced. See askOnce: // without single-flight, a credential entering its renewal window turns @@ -519,15 +524,6 @@ func env(key, fallback string) string { return fallback } -// address returns a resolved endpoint's "scheme://host:port", or "" if the SDK -// could not resolve it. Using the SDK keeps addresses and ports out of the -// runtime: they come from Codefly's runtime-injected endpoint map (or, in a -// local run, its deterministic native workspace map) — never a hardcoded port. -// listenPort is the port of this service's own endpoint address. Codefly hands a -// service its own endpoint as a bare "host:port" (a rendered cell writes -// "localhost:8080"), and other endpoints as URLs; url.Parse reads the bare form -// as scheme "localhost" with no port, which left every rendered solution -// refusing to boot with an empty listen port. Both forms resolve here. // advertisedOrigin is the https origin this runtime reports to the host as // where it serves: what the gateway dials to proxy to it. // @@ -656,6 +652,11 @@ func loopbackOrigin(origin string) bool { return ip != nil && ip.IsLoopback() } +// listenPort is the port of this service's own endpoint address. Codefly hands a +// service its own endpoint as a bare "host:port" (a rendered cell writes +// "localhost:8080"), and other endpoints as URLs; url.Parse reads the bare form +// as scheme "localhost" with no port, which left every rendered solution +// refusing to boot with an empty listen port. Both forms resolve here. func listenPort(self string) string { if strings.Contains(self, "://") { if u, err := url.Parse(self); err == nil { @@ -669,6 +670,10 @@ func listenPort(self string) string { return "" } +// address returns a resolved endpoint's "scheme://host:port", or "" if the SDK +// could not resolve it. Using the SDK keeps addresses and ports out of the +// runtime: they come from Codefly's runtime-injected endpoint map (or, in a +// local run, its deterministic native workspace map) — never a hardcoded port. func address(ctx context.Context, module, service, endpoint, api string) string { q := codefly.For(ctx).Endpoint(endpoint) if module != "" { @@ -1113,7 +1118,7 @@ func (c config) validate() error { // wrong origin is a solution that installs and then answers nothing. if c.upstreamErr != nil { if deployedRuntimeContext(c.runtimeContext) { - return fmt.Errorf("no origin to report to the host in the deployed runtime context %q: %w — Codefly's render injects the address peers reach this service at as %s______HTTP__HTTP, and a deployed runtime never falls back to the address it listens on", + return fmt.Errorf("no origin to report to the host in the deployed runtime context %q: %w — Codefly's render injects the address peers reach this service at as %s______HTTP__HTTP, and a deployed runtime never reports a loopback address it only listens on", c.runtimeContext, c.upstreamErr, resources.SelfEndpointPrefix) } return fmt.Errorf("no origin to report to the host: %w", c.upstreamErr) @@ -1991,6 +1996,7 @@ func (s *Server) credentialWithin(ctx context.Context, d time.Duration) (executi // holdCredential records a credential the controller did not itself acquire — // the boot's — so the first caller after boot sees a process that holds one. func (s *Server) holdCredential(credential executionCredential) { + s.awaiting.Store(false) s.credentialMu.Lock() s.credentialHeld = credential s.credentialQuietUntil = time.Time{} @@ -2111,6 +2117,12 @@ func (s *Server) resolveCredential(ctx context.Context, flight *credentialFlight // former and never the latter. flight.credential, flight.err = credential, err + switch { + case err == nil: + s.awaiting.Store(false) + case awaitingInstallation(err): + s.awaiting.Store(true) + } s.credentialMu.Lock() s.credentialAskErr = err if err != nil { diff --git a/uninstalled_test.go b/uninstalled_test.go index 716e556..ba990d2 100644 --- a/uninstalled_test.go +++ b/uninstalled_test.go @@ -7,6 +7,7 @@ import ( "io" "net/http" "net/http/httptest" + "os" "strings" "sync/atomic" "testing" @@ -302,3 +303,76 @@ func TestAnOversizeMintRequestIsRefusedAtBoot(t *testing.T) { t.Fatalf("a mint request inside the bound was refused: %v", err) } } + +// TestTheAskerSurvivesATransientAnswerWhileUninstalled: one answer that +// judges nothing — the host restarting, a pass in flight — leaves the solution +// exactly as uninstalled as it was. An asker that stopped there left the first +// credential waiting for the host's next install delivery. +func TestTheAskerSurvivesATransientAnswerWhileUninstalled(t *testing.T) { + var phase atomic.Int32 // 0: installed nowhere, 1: one transient answer, 2: issuing + host := newHostMint(t, &hostMint{answer: func() int { + switch phase.Load() { + case 0: + return http.StatusPreconditionFailed + case 1: + phase.Store(2) + return http.StatusServiceUnavailable + } + return 0 + }}) + s := New(Manifest{ID: testSolutionID}).OnInstall(func(context.Context, Installation) error { return nil }) + b := boot(t, s, host) + t.Cleanup(b.client.CloseIdleConnections) + b.client.Timeout = 5 * time.Second + + if status := postInstall(t, b); status != http.StatusNoContent { + t.Fatalf("install step = %d, want 204", status) + } + phase.Store(1) + deadline := time.Now().Add(awaitInstallationEvery / 2) + for { + if _, held := s.heldCredential(); held { + return + } + if time.Now().After(deadline) { + t.Fatalf("no credential held %s after one transient answer: the asker stopped, and nothing but the host's next install delivery would start it again", awaitInstallationEvery/2) + } + time.Sleep(20 * time.Millisecond) + } +} + +// TestABootWaitsForAProjectionThatIsNotThereYet: the platform projects the +// token, and a process can be running before it has. Nothing is sent without +// one, so the boot waits inside its window instead of ending on a state the +// platform is about to leave. +func TestABootWaitsForAProjectionThatIsNotThereYet(t *testing.T) { + host := newHostMint(t, &hostMint{}) + s := New(Manifest{ID: testSolutionID}) + bootEnvironment(t, host) + tokenFile := os.Getenv(ProjectedTokenFileEnvironmentVariable) + projected, err := os.ReadFile(tokenFile) + if err != nil { + t.Fatal(err) + } + if err := os.Remove(tokenFile); err != nil { + t.Fatal(err) + } + go func() { + time.Sleep(300 * time.Millisecond) + _ = os.WriteFile(tokenFile, projected, 0o600) + }() + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + ln, err := takeListener(s).start(ctx) + if err != nil { + t.Fatalf("a boot whose projected token arrived a moment late was refused: %v", err) + } + _ = ln.Close() + if _, held := s.heldCredential(); !held { + t.Fatal("the boot came up holding no credential") + } + if got := host.count(); got != 1 { + t.Fatalf("the host was asked %d time(s); nothing is sent without a token, so exactly one ask reaches it", got) + } +}