diff --git a/CHANGELOG.md b/CHANGELOG.md index 0ed0fa1..f8c047d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,19 @@ # Changelog +## Unreleased — the mint reports the registration's dynamic half; the runtime serves uninstalled + +The mint body carries `upstream` (the advertised https origin, never the listen +address) and `manifest` beside `prefix`, bounded at 64 KiB at boot. A deployed +runtime context refuses a loopback origin. + +`412` from the broker is "admitted, installed nowhere": the runtime serves its +install step and `/health` holding no credential, and asks again until one is +issued. A viewer's route still answers 503. + +A projected token that is not there yet is unavailable, not terminal: a boot +waits for it inside the two-minute window, and a renewal keeps an unexpired +credential. One that is present and unusable stays terminal. + ## Unreleased — native module-broker execution mint Breaking API change: `WorkContextRefusal.Code` and `WorkContextRefusal.Message` diff --git a/README.md b/README.md index c2db60d..a25d7fe 100644 --- a/README.md +++ b/README.md @@ -82,20 +82,24 @@ endpoint — the `CODEFLY__SELF_ENDPOINT______HTTP__HTTP` carri `CODEFLY__ENDPOINT__` carrier to `localhost`, so reporting that would have the gateway proxy the solution to the gateway's own loopback. With no carrier, a process on a developer machine reports the loopback port it actually binds -(`PORT` included). In a deployed runtime context a loopback origin, a missing -carrier and a carrier that does not hold exactly a host and a port each fail the -boot; a present carrier is never treated as absent. +(`PORT` included). In a deployed runtime context a loopback origin fails the +boot — which is what a missing carrier resolves to in a rendered cell — and so +does a carrier that does not hold exactly a host and a port; a present carrier +is never treated as absent. A host that admits this workload but has not been asked to install it anywhere answers `412`. That is not a refusal. The runtime serves holding no credential: `/health` answers `200`, and the [install step](#install-step) runs, because it acts under the capability the host carries in and the host issues this execution's credential only after that step is acknowledged. Neither asks the -host again. The runtime asks every 20 seconds (jittered), and at once — then a -few times in quick succession — after it acknowledges an install step, and holds -the first credential issued. A viewer's route still answers `503` until then. A +host again. The runtime asks every 20 seconds (jittered) for as long as it holds +no credential — through a transient answer too — and at once, then a few times +in quick succession, after it acknowledges an install step. It holds the first +credential issued. A viewer's route still answers `503` until then. A `401`/`403` -stays what it was: a judgement, which fails the boot and is never retried. The answer carries `token`, `expiresAt`, +stays what it was: a judgement, which fails the boot and is never retried. + +The answer carries `token`, `expiresAt`, `principalId`, and `tenant`. Core alone reads the returned capability, through its structural `Decode` (named `Inspect` before core v0.15); the runtime does not implement a signed encoding or verify an issuer's authority, and that read @@ -555,13 +559,14 @@ is waited for, within a bound.** The two are different answers: - A **refusal** (`401` or `403`, wrapping `ErrMintRefused`) is terminal and retained. It is never retried. - An **unavailable** mint (`503`, timeout, connection refused or connection - reset, wrapping `ErrMintUnavailable`) is retried with backoff inside the + reset, or a projected token that is not there yet, wrapping + `ErrMintUnavailable`) is retried with backoff inside the **two-minute** boot deadline. Renewal uses the same classification and keeps an unexpired held credential available during transient failures. - Every other HTTP status, including `400`, `404`, `429`, `500`, and redirects, is a **terminal error**. TLS, permanent DNS, cancellation, unclassified - transport failures, invalid responses and unusable projections are terminal - too. They stop renewal and fail health. Status, status class and transport + transport failures, invalid responses and a projection that is present but + unusable are terminal too. They stop renewal and fail health. Status, status class and transport category survive; raw body, headers, parser diagnostics and causes do not. Every mint makes a fresh TLS 1.3 handshake, presents this workload's identity, diff --git a/boot_test.go b/boot_test.go index 30f7648..bd2df16 100644 --- a/boot_test.go +++ b/boot_test.go @@ -961,7 +961,11 @@ func TestServedContractReportsWhatThisProcessHoldsItselfTo(t *testing.T) { // the port is released by construction. Nothing is released here. func boundPort(t *testing.T) (net.Listener, string) { t.Helper() - ln, err := net.Listen("tcp", ":0") + // On the loopback address the tests dial, not the wildcard. A wildcard + // bind succeeds on a port another local process holds on 127.0.0.1, and + // the test's own client then reaches that process instead: "first record + // does not look like a TLS handshake", from a neighbour. + ln, err := net.Listen("tcp", "127.0.0.1:0") if err != nil { t.Fatal(err) } diff --git a/broker_credential.go b/broker_credential.go index 5610c39..7393932 100644 --- a/broker_credential.go +++ b/broker_credential.go @@ -145,6 +145,9 @@ func (c *brokerCredentialSource) obtain(ctx context.Context) (executionCredentia if err != nil { var projection *projectedTokenError if errors.As(err, &projection) { + if projection.absent { + return workcontext.Credential{}, &brokerMintError{class: brokerProjectionAbsent} + } return workcontext.Credential{}, &brokerMintError{class: brokerProjection} } return workcontext.Credential{}, classifyBrokerTransport(err) diff --git a/credential.go b/credential.go index 1985664..47fa2c6 100644 --- a/credential.go +++ b/credential.go @@ -450,8 +450,6 @@ func (s *Server) openCredential(ctx context.Context) error { } } -// mustDeadline is the deadline of a context that has one. openCredential sets -// it, so the fallback is only there to keep the arithmetic total. // manifestJSON is the frontend manifest this runtime serves, reported with // the mint so the host's registration carries it without a self-registration. func (s *Server) manifestJSON() string { @@ -500,7 +498,7 @@ func (s *Server) awaitInstallation(ctx context.Context) { followUps = awaitInstallationFollowUps case <-idle.C: } - for ask := 0; s.uninstalled(); ask++ { + for ask := 0; s.awaitsInstallation(); ask++ { credential, err := s.askOnce(ctx, platformRequestTimeout) if err == nil { seal := credential.Seal() @@ -576,9 +574,26 @@ func (s *Server) uninstalled() bool { return awaitingInstallation(s.credentialAskErr) } +// awaitsInstallation reports that this process holds no credential and the +// host has said, at some point since it last held one, that nothing has +// installed the solution. +// +// Wider than uninstalled on purpose. One transient answer after a 412 — the +// host restarting, a pass in flight — leaves the solution exactly as +// uninstalled as it was, and an asker that stopped there would leave the +// first credential waiting for the host's next install delivery. +func (s *Server) awaitsInstallation() bool { + if s.terminalErr.Load() != nil || !s.awaiting.Load() { + return false + } + _, held := s.heldCredential() + return !held +} + // noteCredentialAnswer records an answer the controller did not itself // receive — the boot's — so every reader sees the same last answer. func (s *Server) noteCredentialAnswer(err error) { + s.awaiting.Store(awaitingInstallation(err)) s.credentialMu.Lock() s.credentialAskErr = err s.credentialMu.Unlock() @@ -594,6 +609,8 @@ func (s *Server) installAcknowledged() { } } +// mustDeadline is the deadline of a context that has one. openCredential sets +// it, so the fallback is only there to keep the arithmetic total. func mustDeadline(ctx context.Context) time.Time { if at, ok := ctx.Deadline(); ok { return at diff --git a/mint_errors.go b/mint_errors.go index 6eeb86a..e961e92 100644 --- a/mint_errors.go +++ b/mint_errors.go @@ -6,8 +6,10 @@ import ( "crypto/x509" "errors" "fmt" + "io/fs" "net" "net/http" + "os" "syscall" "github.com/codefly-dev/sdk-go/workcontext" @@ -44,12 +46,20 @@ type projectedTokenFile string func (path projectedTokenFile) ProjectedToken() (string, error) { token, err := workcontext.ProjectedTokenFile(path).ProjectedToken() if err != nil { - return "", &projectedTokenError{} + // Absent is told apart from unusable, and only by asking the + // filesystem the one question: is there anything at this path. A + // token that is not there has not been projected yet; one that is + // there and cannot be used is a projection somebody got wrong. + _, statErr := os.Lstat(string(path)) + return "", &projectedTokenError{absent: errors.Is(statErr, fs.ErrNotExist)} } return token, nil } -type projectedTokenError struct{} +type projectedTokenError struct { + // absent: nothing exists at the projection path. + absent bool +} func (*projectedTokenError) Error() string { return "projected service-account token is missing, empty, unreadable or exceeds 65536 bytes" @@ -68,6 +78,10 @@ const ( brokerTransport brokerInvalidAnswer brokerProjection + // brokerProjectionAbsent: there is no token at the projection path. The + // platform has not projected it yet; nothing is sent, and the ask is + // retried. + brokerProjectionAbsent // brokerAwaitingInstallation: the host admits this workload and has not // been asked to install it anywhere yet. The install step arrives on this // runtime's own listener; until then there is nothing to mint for. @@ -102,6 +116,8 @@ func (e *brokerMintError) Error() string { return fmt.Sprintf("module broker: HTTP %d: request failed", e.status) case brokerInvalidAnswer: return "module broker: invalid credential response" + case brokerProjectionAbsent: + return "projected service-account token is missing: it has not been projected yet" case brokerProjection: return "projected service-account token is missing, empty, unreadable or exceeds 65536 bytes" } @@ -118,9 +134,19 @@ func (e *brokerMintError) Error() string { return "module broker: transport or response read failed" } } + +// retryable reports an answer that judges nothing about this build. +// +// A projection that is not there yet is one of them. The platform projects the +// token, and a process can be running before it has; nothing is sent without +// one, so waiting costs no authority: a boot waits inside its bounded window +// and exits as unavailable if the projection never arrives. A projection that +// is there and unusable — empty, oversized, not a file — is a different thing +// and stays terminal: no amount of waiting repairs it. func (e *brokerMintError) retryable() bool { - return e.class == brokerNotEstablished || e.class == brokerAwaitingInstallation || e.class == brokerTransport && - (e.transport == brokerTransportTimeout || e.transport == brokerTransportRefused || e.transport == brokerTransportReset) + return e.class == brokerNotEstablished || e.class == brokerAwaitingInstallation || e.class == brokerProjectionAbsent || + e.class == brokerTransport && + (e.transport == brokerTransportTimeout || e.transport == brokerTransportRefused || e.transport == brokerTransportReset) } func (e *brokerMintError) Unwrap() error { if e.class == brokerRefused { diff --git a/mint_migration_test.go b/mint_migration_test.go index acb6ab9..74aefa3 100644 --- a/mint_migration_test.go +++ b/mint_migration_test.go @@ -95,9 +95,16 @@ func TestUnusableProjectedTokenNeverReachesTheMint(t *testing.T) { mint := newHostMint(t, &hostMint{}) source := productionMintSource(t, mint, path) credential, err := source.Credential(context.Background()) - if credential.Token() != "" || !terminalCredentialFailure(err) { + if credential.Token() != "" || err == nil { t.Fatalf("unusable projection did not fail closed: %v", err) } + // Not there yet is waited for; there and unusable is not. + if wantTerminal := kind != "missing"; terminalCredentialFailure(err) != wantTerminal { + t.Fatalf("a %s projection: terminal = %v, want %v (%v)", kind, !wantTerminal, wantTerminal, err) + } + if kind == "missing" && !errors.Is(err, workcontext.ErrMintUnavailable) { + t.Fatalf("a projection that is not there yet must read as unavailable: %v", err) + } if !strings.Contains(err.Error(), "projected service-account token") || strings.Contains(err.Error(), path) { t.Fatalf("projection error is unclear or discloses its raw path: %v", err) } diff --git a/solution.go b/solution.go index 36037c8..b6640bc 100644 --- a/solution.go +++ b/solution.go @@ -326,7 +326,12 @@ type Server struct { terminalOnce sync.Once // installed signals awaitInstallation that an install step was just // acknowledged. One slot: a pending signal is as good as several. - installed chan struct{} + installed chan struct{} + // awaiting is set once the host has answered that nothing has installed + // this solution, and cleared when a credential is held. It is what keeps + // awaitInstallation asking through a transient answer in between; whether + // the process may SERVE uninstalled is a stricter question (uninstalled). + awaiting atomic.Bool terminalErr atomic.Pointer[error] // The one in-flight credential ask and what it last produced. See askOnce: // without single-flight, a credential entering its renewal window turns @@ -519,15 +524,6 @@ func env(key, fallback string) string { return fallback } -// address returns a resolved endpoint's "scheme://host:port", or "" if the SDK -// could not resolve it. Using the SDK keeps addresses and ports out of the -// runtime: they come from Codefly's runtime-injected endpoint map (or, in a -// local run, its deterministic native workspace map) — never a hardcoded port. -// listenPort is the port of this service's own endpoint address. Codefly hands a -// service its own endpoint as a bare "host:port" (a rendered cell writes -// "localhost:8080"), and other endpoints as URLs; url.Parse reads the bare form -// as scheme "localhost" with no port, which left every rendered solution -// refusing to boot with an empty listen port. Both forms resolve here. // advertisedOrigin is the https origin this runtime reports to the host as // where it serves: what the gateway dials to proxy to it. // @@ -656,6 +652,11 @@ func loopbackOrigin(origin string) bool { return ip != nil && ip.IsLoopback() } +// listenPort is the port of this service's own endpoint address. Codefly hands a +// service its own endpoint as a bare "host:port" (a rendered cell writes +// "localhost:8080"), and other endpoints as URLs; url.Parse reads the bare form +// as scheme "localhost" with no port, which left every rendered solution +// refusing to boot with an empty listen port. Both forms resolve here. func listenPort(self string) string { if strings.Contains(self, "://") { if u, err := url.Parse(self); err == nil { @@ -669,6 +670,10 @@ func listenPort(self string) string { return "" } +// address returns a resolved endpoint's "scheme://host:port", or "" if the SDK +// could not resolve it. Using the SDK keeps addresses and ports out of the +// runtime: they come from Codefly's runtime-injected endpoint map (or, in a +// local run, its deterministic native workspace map) — never a hardcoded port. func address(ctx context.Context, module, service, endpoint, api string) string { q := codefly.For(ctx).Endpoint(endpoint) if module != "" { @@ -1113,7 +1118,7 @@ func (c config) validate() error { // wrong origin is a solution that installs and then answers nothing. if c.upstreamErr != nil { if deployedRuntimeContext(c.runtimeContext) { - return fmt.Errorf("no origin to report to the host in the deployed runtime context %q: %w — Codefly's render injects the address peers reach this service at as %s______HTTP__HTTP, and a deployed runtime never falls back to the address it listens on", + return fmt.Errorf("no origin to report to the host in the deployed runtime context %q: %w — Codefly's render injects the address peers reach this service at as %s______HTTP__HTTP, and a deployed runtime never reports a loopback address it only listens on", c.runtimeContext, c.upstreamErr, resources.SelfEndpointPrefix) } return fmt.Errorf("no origin to report to the host: %w", c.upstreamErr) @@ -1991,6 +1996,7 @@ func (s *Server) credentialWithin(ctx context.Context, d time.Duration) (executi // holdCredential records a credential the controller did not itself acquire — // the boot's — so the first caller after boot sees a process that holds one. func (s *Server) holdCredential(credential executionCredential) { + s.awaiting.Store(false) s.credentialMu.Lock() s.credentialHeld = credential s.credentialQuietUntil = time.Time{} @@ -2111,6 +2117,12 @@ func (s *Server) resolveCredential(ctx context.Context, flight *credentialFlight // former and never the latter. flight.credential, flight.err = credential, err + switch { + case err == nil: + s.awaiting.Store(false) + case awaitingInstallation(err): + s.awaiting.Store(true) + } s.credentialMu.Lock() s.credentialAskErr = err if err != nil { diff --git a/uninstalled_test.go b/uninstalled_test.go index 716e556..ba990d2 100644 --- a/uninstalled_test.go +++ b/uninstalled_test.go @@ -7,6 +7,7 @@ import ( "io" "net/http" "net/http/httptest" + "os" "strings" "sync/atomic" "testing" @@ -302,3 +303,76 @@ func TestAnOversizeMintRequestIsRefusedAtBoot(t *testing.T) { t.Fatalf("a mint request inside the bound was refused: %v", err) } } + +// TestTheAskerSurvivesATransientAnswerWhileUninstalled: one answer that +// judges nothing — the host restarting, a pass in flight — leaves the solution +// exactly as uninstalled as it was. An asker that stopped there left the first +// credential waiting for the host's next install delivery. +func TestTheAskerSurvivesATransientAnswerWhileUninstalled(t *testing.T) { + var phase atomic.Int32 // 0: installed nowhere, 1: one transient answer, 2: issuing + host := newHostMint(t, &hostMint{answer: func() int { + switch phase.Load() { + case 0: + return http.StatusPreconditionFailed + case 1: + phase.Store(2) + return http.StatusServiceUnavailable + } + return 0 + }}) + s := New(Manifest{ID: testSolutionID}).OnInstall(func(context.Context, Installation) error { return nil }) + b := boot(t, s, host) + t.Cleanup(b.client.CloseIdleConnections) + b.client.Timeout = 5 * time.Second + + if status := postInstall(t, b); status != http.StatusNoContent { + t.Fatalf("install step = %d, want 204", status) + } + phase.Store(1) + deadline := time.Now().Add(awaitInstallationEvery / 2) + for { + if _, held := s.heldCredential(); held { + return + } + if time.Now().After(deadline) { + t.Fatalf("no credential held %s after one transient answer: the asker stopped, and nothing but the host's next install delivery would start it again", awaitInstallationEvery/2) + } + time.Sleep(20 * time.Millisecond) + } +} + +// TestABootWaitsForAProjectionThatIsNotThereYet: the platform projects the +// token, and a process can be running before it has. Nothing is sent without +// one, so the boot waits inside its window instead of ending on a state the +// platform is about to leave. +func TestABootWaitsForAProjectionThatIsNotThereYet(t *testing.T) { + host := newHostMint(t, &hostMint{}) + s := New(Manifest{ID: testSolutionID}) + bootEnvironment(t, host) + tokenFile := os.Getenv(ProjectedTokenFileEnvironmentVariable) + projected, err := os.ReadFile(tokenFile) + if err != nil { + t.Fatal(err) + } + if err := os.Remove(tokenFile); err != nil { + t.Fatal(err) + } + go func() { + time.Sleep(300 * time.Millisecond) + _ = os.WriteFile(tokenFile, projected, 0o600) + }() + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + ln, err := takeListener(s).start(ctx) + if err != nil { + t.Fatalf("a boot whose projected token arrived a moment late was refused: %v", err) + } + _ = ln.Close() + if _, held := s.heldCredential(); !held { + t.Fatal("the boot came up holding no credential") + } + if got := host.count(); got != 1 { + t.Fatalf("the host was asked %d time(s); nothing is sent without a token, so exactly one ask reaches it", got) + } +}