diff --git a/AGENTS.md b/AGENTS.md index 0af8a53..99214c6 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -3,22 +3,150 @@ This repository owns the generic Go runtime for **codefly solutions** — modules deployed independently of the host they extend, with no build-time coupling. It owns what every solution needs identically: configuration resolution through the -codefly SDK, self-registration with the host frontend and the gateway (the -credential exchanges and their heartbeats), CORS, Module Federation asset -serving, the capability handshake, the manifest, serving the solution's MCP -server to agent clients, and the gateway client a handler uses to read composed -modules on the viewer's behalf. - -It owns none of the counterparts it talks to. The registration surfaces and -their admission rules belong to the host (`codefly-dev/module-saas-starter`) and -the gateway; endpoint, port and secret resolution to `codefly-dev/sdk-go`; -workspace and manifest types to `codefly-dev/core`; running a composition and -provisioning its secrets to `codefly-dev/cli`. Everything here is a library — -one Go package at the repository root, consumed as the Go module +codefly SDK, obtaining this execution's credential once from the projected +service-account token, a listener that presents the workload's own X.509-SVID, +CORS, Module Federation asset serving, the capability handshake, the manifest, +the published authority contract, serving the solution's MCP server to agent +clients, and the gateway client a handler uses to read composed modules on the +viewer's behalf. + +**A runtime does not register itself.** Presence is delivered — a signed +presence document declares which solution runs on which host, and the host +reconciles towards it — and authority is delivered the same way, bound to one +approved build. The heartbeat, both self-registrations, the per-beat single-use +token exchange, the consumed-module registrations and every key that fed them +are deleted, not fenced; the host's endpoints for them are deleted in the same +cutover, so a runtime of the previous generation gets 404 and that is the +intended outcome. Health is answered, never pushed. If you are about to add +something that tells the host this process exists, you are rebuilding what this +runtime removed. + +It owns none of the counterparts it talks to. The mint endpoint and the +admission rules belong to the host (the composition's own host module) and the +gateway; the mint-once client, the authority reader and the TLS reloader to +`codefly-dev/sdk-go`; endpoint, port and secret resolution to `sdk-go` as well; +workspace, manifest and presence/authority document types to `codefly-dev/core`; +running a composition, provisioning its secrets and rendering the authority +document to `codefly-dev/cli`. Everything here is a library — one Go package at +the repository root, consumed as the Go module `github.com/codefly-dev/solution-runtime-go` at a `vX.Y.Z` tag, plus -`passthroughtest`, the test seam a consumer runs the root package's -passthrough under against a fake host. It holds no runtime behaviour of its -own: it builds on `Server.PassthroughHandler`, the handler `Serve` mounts. +`passthroughtest`, the test seam a consumer runs the root package's passthrough +under against a fake host. It holds no runtime behaviour of its own: it builds +on the handler `Serve` mounts, reached through `internal/seam`. + +**That seam is not public API, and must not become one again.** +`Server.PassthroughHandler` was exported, and a deployment calling it completed +a viewer mint and a module call over plaintext, 200, having skipped +`validate()`, the mTLS boot, the caller allow-list, the ceiling and +authenticated outbound — with the viewer's bearer and this workload's own +credential on the wire, and supplying a credential source defeated the "no +source, nothing to mint with" mitigation it relied on. Go's internal-package +rule is the gate, which is a compiler rather than a naming convention; +`TestNoExportedPathBuildsACredentialBearingHandlerWithoutTheBoot` fails if an +exported path reappears. + +Beside it, the handler-export, capability-decoder and seal-construction gates +are **built on `go/types`**, in `typed_boundary_test.go`, and the fixtures that +drive them are **compiled** into this package through a `packages.Load` +overlay. Two rules follow from resolution that no list could express: +"servable" means **implements `net/http.Handler`**, not a set of types that +happen to; and "core's wire message" means **from core's generated package +path**. What resolution still cannot decide is in that file: a decode +destination whose static type is an interface (`proto.Message`, `any`) is not a +capability by its type, which is the same residual core's own verifier has. + +**There is ONE traversal, `typeWalk`, and every rule asks its question through +it.** Four rounds of findings were the same defect — a walk that enumerated the +carriers it knew: a struct, then a map, then a signature result, then an +interface-typed field, then a generic's type argument, then a concrete wrapper +around a capability. Each fix was correct and the next carrier arrived, because +the author picks the carrier. So the walk reaches a **fixed point** over +resolved types: pointer, slice, array, map key and element, channel, struct +field, signature result, interface method result, exported method result, a +named type's underlying type, a generic's type arguments, a type parameter's +constraint. There is no list to extend because there is no list. + +It **fails closed in both directions a traversal can go wrong.** Cycles are +guarded by identity, not by a depth cap: the two walks this replaced returned +false past depth 8 and depth 6, so nine nested arrays around an `http.Handler` +escaped both — a traversal that gives up quietly is a gate that can be +exhausted by nesting. And what it cannot resolve is **reported**, not skipped; +every caller turns `unresolved` into a finding, because a gate that cannot see +is not a gate that passes. `TestTheWalkRefusesWhatItCannotFinish` asks that +branch directly, since no fixture can exhaust a 50,000-node budget. + +Two distinctions in it are load-bearing, and both were found by the walk's own +first run flagging real code: + +- **What a value HANDS BACK is not what its memory HOLDS.** Construction + follows struct fields and array elements — inline memory — and nothing else: + a method result, and what a pointer, slice, map or channel refers to, are not + allocated by allocating the value. Expanding every carrier reported + `&authorityHeldSource{…}` as constructing a capability, because its + `CredentialSource` field is an interface whose method returns the SDK's + credential, which is an alias for one of core's wire messages. A **decode + destination** does reach through indirection, because a decoder allocates + through it. +- **A foreign package's type is its own API.** What this boundary forbids is + THIS package building a carrier for core's wire message; the SDK's + `Credential` transitively contains them by design. So a named type from + another package is not expanded into its fields — while a core wire type is + still matched wherever it appears, however deep, because the match runs + before any expansion. + +The value-flow half is the same walk with a different predicate. A handler +handed out through `struct{ H any }` or `func() any` cannot be decided from the +result type — and refusing every interface-typed field would refuse +`Operation.Request`/`Response`, which are `any` by design — so the rule asks +whether an exported function whose results carry an interface **a handler could +be stored in** also has a mountable value in hand. `error` is an interface and +every function returns one; a handler cannot be stored in it, and asking +whether `http.Handler` implements the interface is what tells the two apart. A +generic is the same shape one level up: `alloc[SealedValues]()` sees a type +PARAMETER at its `new(T)`, so the question is asked at the INSTANTIATION, where +the concrete type exists (`types.Info.Instances`) — and the type ARGUMENT is +then walked, which is how `alloc[box, *box]()` is caught through `box`'s own +field, two type parameters and an alias deep. Indirection is not a shape +either: a `**SealedValues` destination is dereferenced all the way down, and +the RECEIVER of a `Decode`/`Unmarshal` method is a destination as much as its +arguments are. + +They were syntactic until round sixteen, and that history is the reason for the +rewrite — the old rule was falsified by a shape nobody had written down, six +times over: +`Routes() func(http.ResponseWriter, *http.Request)`, a named collection, an +import-aliased `nh.Handler`, this package's own bare `Handler` (which the +servable set had always named and never matched, because the renderer had no +identifier case at all), an unexported alias, an unexported struct field, an +anonymous struct result, a supplied decode destination, a zero-value +declaration, an assignment alias, a function-local alias, and `Decode` instead +of `Unmarshal`. Every fix was correct and the next spelling arrived anyway. A +rule that reads syntax is a rule about names, and the author picks the names — +so the rules read types now and there is no list to extend. + +Worse than any single shape: two probes asserted against +`workcontext.WorkContextV1`, which the SDK **does not export**, and because a +probe only PARSED its fixture, a name that cannot compile supplied three rounds +of evidence that the whole-capability rule worked. The fixtures compile now. + +A gate cannot be mutation-tested — the harness asks "mutate the code, does the +suite fail?", and a gate only fails when the counterexample it exists to refuse +is in the tree — so its acceptance criterion is the compiled fixture set, +driving the rule itself rather than a copy of it. A copy is not evidence: the +signing-relaxation test checked its own copy of the condition and survived the +real condition being removed. The same applies to the seam's refusal: the +defeat check (`recover()`, a branch that cannot be taken) ran only where +`mustBeATest()` was called directly, so the identical defeat one hop away was +*inherited* as a refusal and passed. + +`passthroughtest` is itself importable by any module, so that rule alone did not +close this: `passthroughtest.Handler` was an exported, production-importable +path to the same handler, and the gate skipped the package by path while +matching only receiver methods named `*Passthrough*`. Its constructors now panic +outside a test binary (`testing.Testing()`), and the gate reads every caller of +the seam in the module and requires that refusal. A `testing.TB` parameter is +not a gate: the unexported method stops a type declaring the interface, not one +embedding it. ## Boundaries @@ -32,48 +160,369 @@ own: it builds on `Server.PassthroughHandler`, the handler `Serve` mounts. change a signature, a struct field or an exported constant; a change that breaks a consumer carries a `!` in the commit type and says so in the PR body. - Boot configuration is resolved in one place, `loadConfig`, and checked in one - place, `validate()`: resolved through the SDK, refused loudly when unresolved. - New configuration follows both rather than reading the environment where it is - used. Most values also take an explicit env override, for an operator whose - deployment the resolver cannot see — but an override is not how a *composition* - supplies a value, because a Codefly render cannot set a bare environment - variable on a service. Anything the composition has to supply is a declared - workspace-configuration group read with `WorkspaceConfiguration` / - `WorkspaceSecret` (`solution-registration/SECRET`, `mcp/issuer-url`, - `mcp/public-url`), and a refusal names that group/key, never a variable name. - `MCP_PUBLIC_URL` and `HOST_ISSUER_URL` were bare variables for one release and - the boot refused with the name of something nobody could set (#48). -- The exception is the consumed-API projection, read at serve time in - `registerConsumedAPIs` and never seen by `validate()`. A malformed one is not - refused: the runtime logs that the federation is disabled and serves on, so - every consumed facade 404s at the gateway while the solution looks healthy. - Anything you add on that path inherits that property — it degrades silently - unless you make it loud, and a log line is the only signal there is. Both - halves of this bullet are pinned by `environment_boundary_test.go`, so a - second exception fails the suite until this file describes it. + place, `validate()`: resolved through the SDK, with an explicit env override, + refused loudly when unresolved. New configuration follows both rather than + reading the environment where it is used. **There is no exception.** There used + to be one — the consumed-API projection was read at serve time to register each + consumed module's upstream, and a malformed one disabled the whole federation + with a log line while the solution served on — and it went with the + registrations. Every environment read now sits in `loadConfig`'s call tree, + which `environment_boundary_test.go` pins: a new read outside it fails the + suite until this file describes the exception. +- **Never sign, parse or verify a Work Context here.** There is one + implementation, Core's; this runtime obtains a credential through the SDK's + mint client, carries it as the string it travels as, and lets the far end + decide. It declares none of a capability's fields and holds no verifier — + core's needs the issuer's live revision, replay, grant and seal sources, which + a solution does not have. `work_context_boundary_test.go` fails on a signing + primitive, on a `WorkContext`-named declaration of this package's own, on a + call to a verifier and on an import of core's generated wire types — the + syntactic half. `typed_boundary_test.go` fails on **constructing** one of + core's wire messages and on **decoding into** one, by RESOLVED TYPE: that is + the escape the import and verifier rules cannot close, because + `proto.Unmarshal(raw, &workcontext.SealedValues{})` needs no generated + import, no signer and no verifier — `SealedValues` is a type *alias* for + core's `WorkSealV1`, so the one SDK import this package already has is enough + to allocate one and decode into it. The rule is on allocation and on decode + DESTINATIONS rather than on the type appearing, because the type appears + legitimately: `holdSealedIdentity` takes a `*SealedValues` and reads it + through `GetInstallationId`. It keys on core's generated PACKAGE PATH rather + than a set of message names, so an alias — the SDK's `Claims`, a + function-local one, anything — resolves to the same type and a message core + adds is covered the day it exists. This is not a style rule: a second signed encoding grew + inside an SDK beside core's once, with its own payload struct, signer, + verifier and error taxonomy, and 3,532 lines had to be deleted to get back to + one. Every step of that was locally reasonable. +- The authority-bearing values are a narrower case still: the principal, the + mint audience and the projected token's own audience are read **once**, through + the SDK's authority reader, and frozen. The credential this process holds is + sealed to the values it was minted under, so a value that resolves differently + later is an error and never a reload — the reader rechecks them before every + renewal, which is the one moment a drifted value would otherwise be laundered + into a credential nobody approved. Do not add an ordinary accessor for one of + those three. + Most values also take an explicit env override, for an operator whose + deployment the resolver cannot see — but an override is not how a + *composition* supplies a value, because a Codefly render cannot set a bare + environment variable on a service. Anything the composition has to supply is + a declared workspace-configuration group read with `WorkspaceConfiguration` / + `WorkspaceSecret` (`mcp/issuer-url`, `mcp/public-url`), and a refusal names + that group/key, never a variable name. - Validate at the boundary — boot, and the headers a request arrives with — not between internal callers. `validate()` is the model: each refusal names the variable or the provisioning path that fixes it. -- A registration that cannot attest which publisher it speaks for is not a - registration. There is no fallback to the shared cluster-internal token and no - downgrade path when an exchange answers `404`. -- The MCP surface verifies no token, and must not start: the gateway runs +- The listener presents this workload's own identity and there is no plain-HTTP + listener. A configuration that cannot produce one is refused at boot naming the + material, because a solution serving plain HTTP is refused at the edge instead, + for a reason only the edge can see. Where the identity comes from is the + platform's: `IdentitySource` is the hook, and the default reads the pair the + platform projects through the SDK's reloader. A source's configuration is + checked for what it must *have* (a certificate, a TLS 1.3 floor, + `RequireAndVerifyClientCert`, a named anchor) **and refused for what it must + not replace**: `Time`, `Rand`, `KeyLogWriter`, `WrapSession`, `UnwrapSession`, + `InsecureSkipVerify`. Each has a safe zero value, so refusing a non-zero one + is exact — and resumption is the sharpest, because a resumed connection + presents no certificate, so a source encoding its own tickets admits callers + without any check here seeing a handshake. That list is a denylist over a + struct this package does not own, and it is **not** complete. That claim was + made here and falsified twice: first by the session-ticket keys, then by + `VerifyPeerCertificate`. A `Clone()` copies 29 of 32 fields unchanged, so + whatever this list does not name, a source keeps. Where a property is + load-bearing, **take** it rather than check it — resumption is disabled + outright, and a caller's identity is read from a fresh parse of the raw DER + rather than from the `*x509.Certificate` a source's own verifier was handed a + pointer to. This list is what *tells* a source it has gone wrong; it is not + what makes the listener safe. The per-connection answer runs through the same + function rather than a copy of its reasoning. `SessionTicketKey` + is on that list for the same reason as the callbacks — choosing the key tickets + are sealed with is the same capability as encoding them — and + `SetSessionTicketKeys` is the one hole a check cannot close, because it is a + method and the keys it installs cannot be read back off the configuration. +- **This listener does not resume TLS sessions, and that is a control rather + than a check.** A resumed connection presents no certificate, so anything + able to forge a ticket is admitted as whoever it was issued to — demonstrated + end to end with a source-supplied ticket key, against a listener whose + posture checks all passed. A denylist cannot close it (`SetSessionTicketKeys` + installs keys that cannot be read back), so `SessionTicketsDisabled` is set on + the served configuration and on every per-connection clone. Where a property + cannot be verified on a value a consumer hands over, take it rather than + inspect it. +- **A source's anchor is asked for, never fabricated.** A consumer source that + resolves its anchor per handshake implements `PeerAnchorSource`; one that can + only answer through a `GetConfigForClient` call is refused at boot, because + obtaining the anchor would mean passing a `ClientHelloInfo` nobody sent — the + same synthetic-probe defect this cutover was blocked on for certificates, and + worse, since an anchor has no check behind it. The outbound *certificate* is + still taken through an empty hello, which is sound only because the frozen + principal hold catches a wrong answer at the moment it is presented. +- The credential is obtained **once**, before the listener exists. A *refusal* + fails the boot and is never retried — the host is saying this build is not the + one its presence document approved, which no number of attempts changes, and a + loop around it is the audited-mint cost this runtime was changed to remove. An + *unavailable* mint is not a judgement (a presence generation not yet applied, + an issuer that cannot reach its own dependencies), so the boot waits inside a + bounded window, with the window as a deadline on the operation, and then exits + non-zero for the orchestrator. Those two answers must stay distinguishable: + conflating them was a review blocker in both directions. +- **Credential-bearing traffic never follows a redirect and never leaves the + gateway's origin.** Go copies a request's headers onto a redirected one and + strips only `Authorization`, `WWW-Authenticate` and `Cookie`, a 307 re-sends + the body, and this runtime sets the bearer per round trip — so a client that + merely inherits a transport, as every gateway client once did, hands a `307 + Location: http://…` the viewer's bearer, their capability and this workload's + credential in cleartext. Build clients through `Gateway.platformClient`, which + carries `ErrUseLastResponse`, and leave the transport's origin pin in place. +- **Peer trust is re-read per connection in BOTH directions, and established + connections are re-verified.** Inbound, per handshake through + `GetConfigForClient`; outbound, per dial through `DialTLSContext`. Per-dial + and per-handshake bound nothing about a connection that already exists, which + an idle timeout cannot fix because it only bounds a connection nobody is + using: a busy outbound connection answered 31s after its root was removed, and + the listener had no recheck, no `IdleTimeout` and no `ReadHeaderTimeout` at + all. Both directions now re-verify the established peer against current trust + once a second and close what stops verifying. The recheck reads only the + anchor and the admitted set — never this workload's own leaf, so a + half-written rotation of our own key pair cannot tear down connections whose + peers are fine — and the outbound leaf comes from ONE long-lived SDK reloader, + which keeps the last good pair; building one per dial threw that away. The outbound half went + unanswered through two review rounds while the inbound argument — judging by + a stale anchor admits whoever should be refused — was already written down + here. It applies harder outbound: those destinations receive the projected + token, the viewer's bearer and this workload's credential. +- **Authentication is not authorisation.** Every workload in the trust domain + holds a certificate from the same anchor, the consumed modules included, so + the admitted caller set is provisioned + (`workload-identity`/`ALLOWED_CALLERS_FILE`) and refused at boot when absent. + Outbound there is **one set per destination** (`MINT_PEERS_FILE`, + `GATEWAY_PEERS_FILE`): the mint and the gateway are two parties, and a single + set spanning both authorises each to stand in for the other at the other's + address — the mint being the one that receives the projected token. + `MINT_PEERS_FILE` governs the mint hop again, per handshake, through the + SDK's `AdmittedPeers` reader (sdk-go#51) — so the question it was kept open + for is settled: it is the authorization set for the destination that receives + the projected service-account token. It also still narrows the shared-address + case, where the mint and the gateway answer at one address and a dial cannot + be attributed, and what is admitted there is the intersection of both sets. A dial to + a third address has no set and is refused, because this runtime talks to + exactly two destinations. + An admission set answered by **both** the env override and the platform's + provisioning is refused rather than ranked — two sources for one + authorization fact, which is the stance the SDK already takes on a value + delivered inline and by carrier. The other paths in that group still take the + override first, because the worst case there is this process reading its own + material from somewhere else, not admitting a caller. + Both admission sets are **paths this process reads itself**, per handshake and + per dial, not configuration values: a value is fixed at process start, so + "re-resolved per handshake" through the SDK's accessor returned the boot answer + forever while the code and the README both said revocation needed no restart. + Anything in `workload-identity` the platform can change is a path or it is a + snapshot pretending otherwise. A + listener that verifies every caller and admits all of them lets a consumed + module set its own `x-org-id`/`x-session-id` and drive mints under this + workload's attestation. +- **Every mint names the installation it acts under, and it comes from the + header the gateway stamps.** The host requires it (core's + `StartInput.InstallationID`) and refuses a mint naming none; an organization + is not an installation, since one org may hold several. + `x-codefly-installation-id` is the source, on the same footing as `x-org-id` + and `x-session-id`: the gateway stamps all three from verified claims and + replaces whatever a caller sent. A request naming none is refused, and not as + a `ClientError` — no caller can stamp one, so it is not theirs to fix. + **This paragraph said the opposite for two rounds.** It described reading the + installation from the *seal* of the carried capability, with the header as a + fallback, which is the SDK's rule for the direction where a capability IS the + authority being carried — outbound, where `Attach` puts the seal's own + installation beside it and the far end cross-checks the two. Inbound that + carrier is caller-controlled: a browser can put a capability in + `x-codefly-work-context`, and this runtime's standing property — with its own + test, `TestBrowserSuppliedWorkContextIsNeverForwarded` — is that such a + capability authenticates nothing. Reading the installation out of it would + let a caller name the deployment their own mint is attributed to, and because + `FromHeaders` refuses a capability whose installation carriers are + incomplete, a caller sending a malformed one would decide whether the mint + happens at all. The code has read only the stamped header since round ten; + these two documents did not follow it, which is the failure this file warns + about in its own first line. + A background or delegated call would take the installation from the parent + capability's seal on the held credential; no such path exists here, because + every mint is viewer-driven and `ForModule` refuses a caller without an org + and a session. +- **This execution's own credential is held to what it was FIRST sealed to.** + The host seals it from the projected token — `MintOptions` carries no + installation field — so what this runtime owns is refusing one sealed to no + installation, refusing one sealed to no **execution** (image digest and build + incarnation, which core seals as a pair), and refusing a renewal sealed to a + *different* installation, digest or incarnation. The last is one value, not + three checks: a renewal that changed any part of it is a different + execution's credential arriving through the renewal path, and every work + context this process mints for a viewer is attested by it — so the mints + either side would name a different deployment or a different build, + including one the host's approved-build ceiling was never applied to. The + no-execution case is the one the sdk-go v0.3.0 migration opened: the previous + SDK's reader refused a zero incarnation, so a credential arriving here + carried one by construction, and core legitimately mints execution-free + credentials for a principal recorded as bearing none. The guarantee was + inherited and the inheritance ended. +- **Every surface that acts for a viewer is held to this execution's + credential, and the MCP methods are held by an allowlist.** A plain handler + and a `ViewerBearer` passthrough were gated and a tool call was not, so an + MCP tool received a gateway carrying the viewer's bearer and ran while the + issuer had withdrawn this process's credential — the same fail-closed defect, + arriving with a surface rather than being missed in one. `actsForNobody` + lists the methods that run nothing for anyone (the handshake, the keepalive, + the listings, notifications) and **everything else** goes through + `actingForAViewer`: a method the MCP SDK adds is gated by default, because a + denylist over a set this package does not own is the mistake the TLS posture + made three times. The refusal carries the sanitized sentence, never the + source's error — an MCP client is a viewer's channel like a browser is. It + carries a JSON-RPC code, `-32001`: this file said the SDK's wire-error type + was internal and that no code could be set, which was false — + `jsonrpc.Error` is an exported alias for it, mutant `R10-8` holds the code, + and the claim survived here for two rounds after the code landed. +- **What a composition supplies is declared configuration, and a published URL + is held to what a dialled one is.** A Codefly render cannot set a bare + environment variable on a service, so anything the composition owns is a + workspace-configuration group read through the SDK (`mcp/issuer-url`, + `mcp/public-url`) and a refusal names that group/key. In a deployed runtime + context the MCP resource identifier must be **declared, not derived**: the + derivation builds `/solutions//mcp`, which is this runtime + encoding the route the *host* serves it on — the assumption the Module + Federation manifest URL was changed to stop making, for the same reason. A + derived value satisfied the older "a public URL is set" check, so nothing + refused it in the one case that mattered. Both MCP URLs are refused for + userinfo, a query, a fragment, and plaintext in a deployment, and those + refusals redact: the pairing refusal printed an identifier's `?token=` into + the boot log. +- **Nothing mints per request.** One credential per execution plus the renewals + its own expiry dictates, and that has to survive the checks added in front of + it: a route gate that asked the source on every viewer request turned a + 503-ing issuer into one audited mint per request, which is worse than the + heartbeat this runtime deleted, because the heartbeat at least minted on a + timer. Acquisition is single-flight with a backoff, a credential already in + hand and unexpired serves while the issuer is unavailable, and a terminal + answer is recorded **where the answer lands** rather than by whichever caller + was still waiting — the ask is detached, so it can finish with nobody + listening. +- **The mint hop is held to the same three things the gateway hop is, and the + SDK owns the transport that holds it.** `MintOptions` takes no pool, no + client and no dialler: it takes `TrustAnchor`, `ClientCertificate` and + `AdmittedPeers` as **readers**, each consulted during the handshake + (sdk-go#51). So the mint request presents this workload's X.509-SVID, the + endpoint is admitted by the SPIFFE ID in its single URI SAN against + `MINT_PEERS_FILE`, and the anchor is re-read before and after every + handshake and never cached. Withdrawal of a root or of a peer takes effect on + the next handshake rather than on the next process. + The SDK still refuses a caller-supplied `*http.Client`, and that reasoning is + sound and worth keeping in mind: a supplied client is a hole it cannot + inspect — a nil `Transport` means the global mutable default, a + `DialTLSContext` bypasses `TLSClientConfig` entirely, and a caller holding + the same `*http.Transport` can turn verification off after construction. The + answer was never for this runtime to take the transport back; it was for the + transport to take readers instead of values, which is what it now does. + **This paragraph said the opposite, and said it was unreachable.** It + recorded that the mint presented no certificate, that `MINT_PEERS_FILE` did + not govern that hop, and that the rest of the posture could not be restored + from outside the SDK — accurate at `v0.3.0`, and the reason it was an sdk-go + issue rather than a local workaround. What was wrong was not the diagnosis + but treating the resulting gap as a property of this runtime: a reviewer + pointed out that the fixtures had been weakened to accommodate it + (`VerifyClientCertIfGiven`, the mint caller's identity expected to be empty), + which establishes a changed behaviour rather than closing a finding. The + fixtures demand the certificate again, and + `TestTheMintHopIsHeldToTheSameThingsTheGatewayHopIs` drives the negative + controls through a real handshake: a peer the anchor signed but the set does + not name, peer withdrawal with the anchor unchanged, an unreadable anchor, + and a certificate reader that cannot answer. + **There is no established mint connection to bound, and this file said there + was.** It recorded a residual — "the client keeps idle connections, so a + withdrawal takes effect on the next handshake and not on a connection already + open" — which was true of `v0.3.0` and is not true of the pinned leaf. That + transport sets `DisableKeepAlives`, `MaxIdleConnsPerHost: 0` and + `ForceAttemptHTTP2: false`, so **every mint is a fresh handshake**; and after + the handshake completes it re-reads the anchor and the peer set and verifies + the peer again, because `GetClientCertificate` and the server's own + processing run *after* the first admission decision and a withdrawal landing + in that window must not authorize the HTTP write. It also refuses an endpoint + that did not request a client certificate, so this hop cannot silently become + anonymous. The residual to carry is narrower and belongs to the SDK: the + interval between that post-handshake recheck and the write itself. +- **Only 401 and 403 latch a credential refusal.** The rule was "429 and 5xx + retry, everything else is terminal", and sdk-go v0.3.0 inverted it after + measuring the cost: a 408 from a proxy, with a valid credential in hand, + refused and latched for the life of the process, and a 404 from an ingress + mid-rollout would do the same. Those two are the statuses the host signs — + the projected token is not acceptable, or the build is not approved. **A 404 + on the mint is now an outage**, so an old runtime against a host of this + generation waits out its bounded window and exits non-zero for the + orchestrator rather than failing on the first answer. It still does not + recover; what changed is the shape, not the outcome. +- **A terminal credential refusal ends the process; a transient one does not.** + `ErrMintRefused` at renewal fails `/health` and ends `serve` with the reason. + `ErrMintUnavailable` must not: it is transient by construction and exiting on + it is a crash loop. `ErrRevoked` is not in either set, and three review rounds + found this file still saying it was: the mint client does not return it at all + (see `credential.go`) — it comes back from a *callee* rejecting a capability + this runtime minted for a viewer, where it is a 409 on that request and no + statement about this execution's own credential. + **Nothing is inferred from that 409, and the code now matches what this file + has said all along.** A cached capability was dropped when a module answered + 409 with an installation header, and that was wrong three times over: any 409 + with any header; then any 409 whose header named the installation the + capability is sealed to — and a module answering an ordinary business + conflict IS in that installation, so a duplicate or a lost update evicted a + valid capability and the next call minted again. 409 means "the state you + were sealed to has moved" and it means "that page already exists"; nothing on + the wire says which, and matching installation metadata identifies the SCOPE + of a conflict rather than establishing supersession. A genuinely superseded + capability is refused call by call by the far end, correct if noisy, until its + own expiry. A sound inference needs a host-side discriminator, which is + nobody's to invent here. The boot and the run have to + classify these the same way — they did not, and the result was a solution + answering 503 forever while reporting itself healthy. +- **Fail closed, with no exception for a counterpart's current state.** A mint + this runtime cannot attest for is not sent; a listener that cannot + authenticate its callers does not start; a credential-bearing destination that + is not authenticated https is refused at boot. "The gateway presents no client + certificate today" and "the host does not require the attestation yet" are + statements about someone else's deployment, and leaving a hole open for as long + as the counterpart takes is the concession this cutover exists to stop making. +- The published contract is **declared, not derived**. A scope ceiling computed + from what the code asks for would be satisfied by construction — a method that + asked for one more action would widen the ceiling meant to refuse it — so the + author declares the ceiling per profile, the boot refuses a declaration that + exceeds it, and a deployed profile missing from the contract is a refusal + rather than a silent read of the local one. +- **The contract this runtime publishes is not the one the renderer reads, and + must never claim its schema string.** The renderer reads exactly one file, + `module.contract.codefly.yaml` — YAML, strict-decoded, at the module directory + the composition resolved — under `codefly/module-contract/v1`, whose audiences + are `{from: /}` slots a composition resolves per environment + (`codefly-dev/cli#855`); it never reads a running process or a build artifact. + This runtime publishes JSON, keyed by profile, under its own + `codefly/solution-runtime-contract/v1`, answering what a process holds *itself* + to. Both surfaces claimed the renderer's string once, which is worse than a + mismatch: strict decoding refuses the unknown fields, and a matching string + makes that a *malformed* module contract rather than somebody else's document — + a version skew that is not one, pointing at the wrong owner. + `TestThisRuntimesContractDoesNotClaimTheRenderersSchema` fails if the two ever + converge again. Writing the renderer's file would be a generator in this repo + and nothing here does it today. +- **The MCP surface verifies no token, and must not start.** The gateway runs ext_authz and stamps the identity, so a check here would be a second, divergent copy of the host's admission rules. What `requireStampedViewer` - enforces is that the gateway did it — and the refusal for a credential with no - session is at that boundary, not inside a tool, because there it reads as a - broken tool. A tool's arguments are written by a model: no identity ever comes - from them. + enforces is that the gateway did it, and the refusal for a credential with no + session is at that boundary rather than inside a tool, where it would read as + a broken tool. A tool's arguments are written by a model: no identity ever + comes from them. - Tests live beside the code in the same package. Every behaviour change brings one, and the counterpart it exercises is an `httptest` server, never a real host. ## How to behave when something does not work -Fleet standard, governed by the handbook's agent-context track -([obin-ai/handbook#68](https://github.com/obin-ai/handbook/issues/68)). These -are not style preferences; each is a rule an agent broke at real cost, and this -repository is the runtime that cost was measured against. +Fleet standard. These are not style preferences; each is a rule an agent broke +at real cost, and this repository is the runtime that cost was measured +against. 1. **A gap in the tooling is a bug in the tooling** — never a reason to reach around it. Not as a "workaround", not "just this once", not "until the verb @@ -101,20 +550,25 @@ repository is the runtime that cost was measured against. at the place that owns the behaviour, or a *hack*. A hack does not become a fix by working, by being small, by being local, or by the real fix belonging to core, the SDK or the host. -4. **Never hardcode what the system resolves.** This package resolves every - address, port and secret through the SDK; the single `localhost` literal in - it is the self upstream's listen-address fallback, built from the *resolved* - port and refused by `validate()` in a deployed runtime context. Both token-exchange URLs - are derived from their register URLs by swapping the path suffix, which is - why an override that drops the documented suffix cannot be paired and is - refused at boot rather than guessed at. The MCP resource identifier is - derived the same way — from the resolved `PUBLIC_URL` and the solution id — - and is the one place the gateway's `/solutions/` route is encoded, because - a client's token is audience-bound to that exact string and nothing - downstream can resolve it later the way the host resolves a relative manifest - URL. README says so where it says the route layout is not encoded. If you are typing an address, a port - or a credential, you are encoding something true only on your machine for the - next ten minutes. +4. **Never hardcode what the system resolves, and never guess what it cannot.** + This package resolves every address, port and secret through the SDK; the + single `localhost` literal in it is the self upstream's listen-address + fallback, built from the *resolved* port and refused by `validate()` in a + deployed runtime context. The credential mint URL has **no default**: it is + the address the projected service-account token goes to, nothing resolves it + yet, and a derived path on the resolved gateway was carried for two rounds as + a "labelled stopgap" while the README described it as settled — a guess a + label does not make safe. It is refused at boot until something resolves it. + If you are typing an address, a port or a credential, you are encoding + something true only on your machine for the next ten minutes; if you are + deriving one the owner has not published, you are encoding a guess about + somebody else's deployment. + The MCP resource identifier is derived the same way — from the resolved + `PUBLIC_URL` and the solution id — and is the one place the gateway's + `/solutions/` route is encoded here. `PUBLIC_URL` itself went with the + manifest registration it fed and came back for this one consumer: whether a + runtime should derive the host's route at all is an open follow-up, not a + settled answer. 5. **Diagnose, do not pattern-match.** "It started working when I set X" is not a diagnosis — set X back and confirm it breaks. Do not trust an error message before checking it: in the session above, *"the provisioned secret does not @@ -147,8 +601,13 @@ test -z "$(gofmt -l .)" Go comes from `go.mod` (1.27). Nothing else is needed: no Docker, no credentials, no running composition. -- The suite takes ~20s, most of it two heartbeat tests that wait on real wall - time (10s and 5s). That is not a hang. +- The suite takes about a minute under `-race` and is hermetic: the host's mint, its gateway + and accounts are all `httptest` servers on `127.0.0.1`, the workload identity + is a key pair the test generates into a temp directory, and a stand-in + capability is minted by **core's** authority from core's published fixture + identities — never by a signer of this repo's own. The two + heartbeat tests that waited on 10s and 5s of real wall time are gone with the + heartbeat. - The formatting gate runs over the whole tree (`gofmt -l .`), so a package added in a subdirectory is format-checked as well as vetted and tested. - Dependency pins are how fixes from `core` and `sdk-go` reach consumers: @@ -166,15 +625,15 @@ credentials, no running composition. Keep this file short; add depth to the file that owns the subject. - [README.md](README.md) — the consumer-facing contract: handler errors, the - full configuration and env-override table, self-registration and its - provisioning, consumed-module federation, reading a Work-Context-authenticated - module, and the MCP surface (what is served, every refusal, and why the - resource identifier cannot be resolved here). -- The package doc comment and the comments in [`solution.go`](solution.go) and - [`mcp.go`](mcp.go) — why a refusal, a backoff cap or a cache lifetime is what - it is. Several - record a failure mode that is not obvious from the code; read the one next to - what you are changing before you change it. + full configuration and env-override table, the workload identity and the one + mint, the published contract, consumed-module federation, and reading a + Work-Context-authenticated module. It also carries the migration note: an old + runtime against a new host gets 404, and nothing bridges the two. +- The package doc comment and the comments in [`solution.go`](solution.go), + [`credential.go`](credential.go), [`identity.go`](identity.go) and + [`contract.go`](contract.go) and [`mcp.go`](mcp.go) — why a refusal, a renewal point or a cache + lifetime is what it is. Several record a failure mode that is not obvious from + the code; read the one next to what you are changing before you change it. - `.claude/skills/` — procedures an agent repeats, loaded only when relevant. None exist yet, because nothing here repeats beyond the four commands above. Add one instead of growing this file. diff --git a/README.md b/README.md index 4ddbe73..696c703 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,11 @@ # solution-runtime-go Generic Go runtime for **codefly solutions** — independently deployed modules -that plug into a host at runtime with no build-time coupling. Owns registration -(host + gateway, with heartbeat), CORS, Module Federation asset serving, the -capability handshake, the manifest, serving the solution's MCP server to agent -clients, and the gateway client each handler uses to read composed modules on -the viewer's behalf. +that plug into a host at runtime with no build-time coupling. Owns configuration +resolution, this execution's one credential, a listener that presents the +workload's own identity, CORS, Module Federation asset serving, the capability +handshake, the manifest, the published authority contract, and the gateway +client each handler uses to read composed modules on the viewer's behalf. A solution author writes a manifest and one handler: @@ -20,6 +20,48 @@ solution.New(solution.Manifest{ID: "my-solution", Title: "My Solution"}). The gateway URL, the caller's bearer, and the wire protocol are hidden. +## A runtime does not register itself + +**If you are upgrading from a runtime that did, read this first.** Nothing +bridges the two models, and the cutover is deliberate: + +- An **old runtime against a new host** gets `404` on the endpoints it + registers with — they are deleted, in the same cutover, by design. It will + log a failed registration forever and the host will never learn of it. It + must be redeployed on this runtime. There is no compatibility mode, no + fallback credential, and no version of the host that serves both. +- A **new runtime against an old host** registers nothing, because it has + nothing to register with, and mints nothing, because the old host serves no + mint endpoint: the boot fails, naming the endpoint it could not reach, and + the process exits non-zero. + +What changed, and why. A solution used to become present by announcing itself: +two self-registrations — the manifest to the host frontend, the dialable +upstream to the gateway — on a 15s heartbeat, each beat burning a single-use +credential the issuer audited as a mint. A solution that had not changed in a +week announced itself roughly 11,500 times a day, the audit log recorded every +one, and presence came from a process being up, which means a process could +claim it. + +Presence is **delivered** now. A signed presence document declares which +solution runs on which host, at which generation and build, under which +workload identity; the host reconciles towards it. Authority is delivered the +same way, by a signed document bound to one approved build. This runtime's only +outbound act at boot is to obtain its own credential once, from the +service-account token the platform projects for it; it renews that credential +when it expires and otherwise says nothing to anyone. Health is **answered**, +never pushed: the host probes the destination its own presence document names, +and nothing this process answers can make it present. + +Four things follow, and each is a boot failure rather than a degraded run: + +| | | +|---|---| +| The listener presents this workload's X.509-SVID over TLS | there is no plain-HTTP listener, because the host refuses a plain-HTTP destination | +| The credential is obtained once, before the listener exists | a refusal is terminal and never retried; an unavailable mint is waited out inside a bounded deadline — see [One credential per execution](#one-credential-per-execution) | +| The authority-bearing values are read once and frozen | the credential is sealed to the values it was minted under, so a value that drifts is an error, never a reload | +| The contract is published, and the declaration is held to it | an ask outside its declared ceiling fails the boot — see [The published contract](#the-published-contract) | + ## Handler errors Return `ClientError` for an explicitly public validation message. Connect errors @@ -41,7 +83,7 @@ or flattening failures into strings must adopt these typed errors to benefit. The runtime hardcodes nothing. On boot `Serve` calls `codefly.LoadEnvironmentVariables()` to load Codefly's injected carriers, then -`loadConfig` resolves every address, port, and secret through the codefly Go SDK +`loadConfig` resolves every address, port, and path through the codefly Go SDK (`github.com/codefly-dev/sdk-go`), falling back to the local native workspace map when not running under the runtime. Most values have an explicit env override for an operator with a deployment the resolver cannot see; the @@ -57,68 +99,627 @@ identifier and the issuer](#the-resource-identifier-and-the-issuer). | What | SDK resolution | Env override (default) | |---|---|---| | Own listen port | `codefly.For(ctx).Endpoint("http").NetworkInstance()` — the Codefly-assigned port, not a fixed default | `PORT` | -| Public URL | none — the manifest is then registered as the root-relative path `/assets/mf-manifest.json` on this backend (see [Where the host reaches the solution](#where-the-host-reaches-the-solution)) | `PUBLIC_URL` | -| Gateway URL (auth-gateway `rest`) | resolved by role — the single module owning the `auth-gateway` `rest`/`rest` endpoint, discovered from the injected carriers (or the workspace, run locally) | `GATEWAY_URL` | -| Host frontend URL | resolved by role — the single module owning the `frontend` `http`/`http` endpoint, discovered the same way | — (feeds the host register URL) | -| Host register URL | `/api/solutions/register` | `HOST_REGISTER_URL` | -| Gateway register URL | `/solutions/_register` | `GATEWAY_REGISTER_URL` (must end in `/solutions/_register`, see below) | -| Gateway module register URL | `/modules/_register` | `GATEWAY_MODULE_REGISTER_URL` | -| Gateway module token URL | the module register URL above with `/modules/_register` swapped for `/modules/_registration-token`, so it keeps that gateway's base path | `GATEWAY_MODULE_REGISTRATION_TOKEN_URL` | -| Gateway solution token URL | the gateway register URL above with `/solutions/_register` swapped for `/solutions/_registration-token`, same reasoning | `GATEWAY_SOLUTION_REGISTRATION_TOKEN_URL` | -| Internal-auth token | `codefly.For(ctx).WorkspaceSecret("internal-auth", "CODEFLY_INTERNAL_TOKEN")` — the namespaced secret Codefly injects | `CODEFLY_INTERNAL_TOKEN` | -| Solution registration secret | `codefly.For(ctx).WorkspaceSecret("solution-registration", "SECRET")` — see [Self-registration](#self-registration) | `CODEFLY__SOLUTION_REGISTRATION_SECRET` | -| Registration beat interval | `15s` | `CODEFLY__SOLUTION_REGISTRATION_INTERVAL` | -| Self upstream | the reachable self endpoint Codefly injects as `CODEFLY__SELF_ENDPOINT______HTTP__HTTP` (core ≥ v0.5.6); without it, the listen address `http://localhost:` | `SELF_UPSTREAM` | -| Deployed or local | `CODEFLY__RUNTIME_CONTEXT`, injected by Codefly: `native`/`nix`/`container`/`free` (or unset) is a local run, anything else (a GitOps render's `kubernetes`, core ≥ v0.5.6) a deployment | — | +| Gateway URL (auth-gateway `rest`) | resolved by role — the single module owning the `auth-gateway` `rest`/`rest` endpoint, discovered from the injected carriers (or the workspace, run locally). Must be `https` | `GATEWAY_URL` | +| Credential mint URL | **not resolved — required explicitly.** Must be `https`. Nothing derives it: whether the host mints through the gateway or the workload posts straight to the issuer is unsettled and neither endpoint is deployed, and this is the address the projected service-account token goes to, so an assumed one is refused at boot. Every boot logs where it will mint, and a 404 there means that endpoint does not exist on that host — not that this build was refused | `CODEFLY__CREDENTIAL_MINT_URL` (**required**) | +| Projected service-account token | `codefly.For(ctx).WorkspaceConfiguration("workload-identity", "TOKEN_FILE")` — a **path**, re-read at every mint | `CODEFLY__WORKLOAD_TOKEN_FILE` | +| Workload identity certificate | `workload-identity`/`CERT_FILE` | `CODEFLY__WORKLOAD_IDENTITY_CERT_FILE` | +| Workload identity private key | `workload-identity`/`KEY_FILE` | `CODEFLY__WORKLOAD_IDENTITY_KEY_FILE` | +| Peer trust anchor | `workload-identity`/`TRUST_BUNDLE_FILE` — **required**: the listener requires and verifies a caller's certificate against it, and the outbound client verifies the platform against it | `CODEFLY__WORKLOAD_IDENTITY_TRUST_BUNDLE_FILE` | +| Allowed callers | `workload-identity`/`ALLOWED_CALLERS_FILE` — **required**, a **path** to a file of identities, one per line (normally the gateway's); re-read per handshake. Verifying against the anchor says a caller holds an identity the platform issued; this says which of them this solution serves | `CODEFLY__WORKLOAD_IDENTITY_ALLOWED_CALLERS_FILE` | +| Credential mint peer | `workload-identity`/`MINT_PEERS_FILE` — **required**, a **path** to a file of identities; re-read per handshake. This is the set the credential mint endpoint is admitted by: the SDK's mint client reads it during the handshake and refuses a peer whose single URI SAN is not in it, even when the certificate was signed by the accepted anchor. It also narrows the shared-address case, where the mint and the gateway answer at one address and a dial cannot be attributed: what is admitted there is the intersection of this set and the gateway's | `CODEFLY__WORKLOAD_IDENTITY_MINT_PEERS_FILE` | +| Gateway peer | `workload-identity`/`GATEWAY_PEERS_FILE` — **required**, same shape. One set **per destination**: the mint and the gateway are two parties, and a single set spanning both authorises each to stand in for the other at the other's address — and the mint is the destination that receives the projected service-account token. A dial to an address that is neither has no set and is refused. When the two are at the **same** host and port — the brokered shape — a dial there cannot be attributed to one of them, so it is held to the **intersection** of both sets, and an empty intersection is refused at boot | `CODEFLY__WORKLOAD_IDENTITY_GATEWAY_PEERS_FILE` | +| Principal this workload runs as | `module-authority`/`PRINCIPAL`, read once and frozen | — | +| Audience it mints against | `module-authority`/`AUDIENCE`, read once and frozen | — | +| Audience of its own projected token | `module-authority`/`PROJECTION_AUDIENCE`, read once and frozen | — | +| Contract profile | the Codefly environment's own name, which is how Core resolves a profile for an environment that declares none | `CODEFLY__CONTRACT_PROFILE` | | MF assets | `Manifest.Assets` when set (see below), else the `../fe-remote/dist` directory | `ASSETS_DIR` (directory only) | | Host issuer (MCP) | `codefly.For(ctx).WorkspaceConfiguration("mcp", "issuer-url")` — the declared value the composition supplies; without one, the resolved host frontend origin (see [Exposing an MCP server](#exposing-an-mcp-server)). Checked at boot only when `ServeMCP` is declared | — (declared configuration, no env override) | -| Public MCP URL | derived by construction: `/solutions//mcp`. Overridden by `codefly.For(ctx).WorkspaceConfiguration("mcp", "public-url")` (must end in `/mcp`). With neither, the resource identifier is reconstructed per request from `x-forwarded-proto` / `x-forwarded-host` / `x-forwarded-prefix` | — (declared configuration, no env override) | +| Public MCP URL | **declared** (`mcp/public-url`) in a deployed runtime context, where a derived value is refused: the derivation encodes the route the *host* serves this solution on. Locally, derived from `/solutions//mcp`. Overridden by `codefly.For(ctx).WorkspaceConfiguration("mcp", "public-url")` (must end in `/mcp`). With neither, the resource identifier is reconstructed per request from `x-forwarded-proto` / `x-forwarded-host` / `x-forwarded-prefix` | — (declared configuration, no env override) | + +Every `workload-identity` value is a **path, never material**, the three +admission sets included — and that is what makes them live. They are admission +decisions, and an admission decision read once at boot cannot narrow: removing a +compromised identity from either file takes effect on the next handshake, the +next dial and the next connection check, with no restart. A set that resolves to +nothing, or that cannot be read, **refuses** rather than falling back to the set +the process booted with. + +> They were configuration *values* until round four of review, re-resolved on +> every handshake through the SDK's accessor — which does not work and this +> README claimed it did. A configuration value, inline or file-carried, is fixed +> at process start (the SDK reads the file once and keeps it, `file_carrier.go`), +> so the re-resolution returned the boot answer for the life of the process, the +> refusal branch was unreachable in any deployment, and revoking a caller +> silently needed a restart. They are paths now, read by this process like the +> trust anchor beside them, which is the only form that can be what was +> documented. + +The files behind these paths are read by this process — the token at every mint, +the key pair and the anchor at every handshake, the two admission sets at every +handshake and dial — so the SDK's value accessors are the wrong tool for any of +it: a file-carried configuration value is read once and kept for the life of the +process, which is right for a configuration value and wrong for anything the +platform changes underneath one. + +The three `module-authority` values are read through the SDK's authority reader +and **frozen**: the credential this process holds is sealed to the values it was +minted under, so a value that resolves differently later is an error and not a +reload. They are rechecked before every renewal, which is the one moment a +drifted value would otherwise be laundered into a credential nobody approved. + +Every unresolved value is refused at boot, naming the value and the provisioning +path that fixes it — including the difference between "the SDK resolved nothing" +and "loading the injected environment failed first", because one generic message +sent an operator to inspect endpoint resolution over a variable they had broken +themselves. + +`PUBLIC_URL` survived the cutover with one consumer. It fed the manifest +registration, where building an absolute URL from it was wrong — the route a +browser reaches this solution through is the host's to resolve — and the +manifest is a path now. The MCP resource identifier an agent client binds its +token to does need a public origin, and derives from it when no `mcp/public-url` +is declared. Whether a runtime should derive the host's `/solutions/` route +even for that is an open follow-up. + +**Removed in this cutover, with the registrations they fed**: `SELF_UPSTREAM`, `HOST_REGISTER_URL`, `GATEWAY_REGISTER_URL`, +`GATEWAY_MODULE_REGISTER_URL`, `GATEWAY_MODULE_REGISTRATION_TOKEN_URL`, +`GATEWAY_SOLUTION_REGISTRATION_TOKEN_URL`, `CODEFLY_INTERNAL_TOKEN`, +`CODEFLY__SOLUTION_REGISTRATION_SECRET`, `CODEFLY__MODULE_REGISTRATION_SECRETS`, +and `CODEFLY__SOLUTION_REGISTRATION_INTERVAL`. Setting any of them now does +nothing. + +`CODEFLY_HOST_FRONTEND` is **not** in that list, and this said it was. It went +with the registrations and then came back for one consumer, the same way +`PUBLIC_URL` did: it names the host's frontend service, whose resolved URL is +the fallback the MCP issuer identifier is derived from when no `mcp/issuer-url` +is declared (`resolveMCPIssuer`). It is read in `loadConfig` like every other +value here. Nothing reads the shared cluster-internal token or +a per-solution registration secret any more: the credential this runtime +presents attests which workload it is, and there is no fallback to one that +attests nothing. The host it plugs into is named by Codefly-convention **service roles**, not by -its workspace module name: the runtime discovers the single module that owns each -role — from the injected endpoint carriers when deployed, or the workspace on -disk when run locally — so a solution composing the host as `saas`, -`saas-starter`, or any other name resolves identically (codefly-dev/core#382). -The roles are overridable: `CODEFLY_HOST_FRONTEND` (default `frontend`), -`CODEFLY_HOST_GATEWAY` (default `auth-gateway`, falling back to the pre-v0.0.49 -`auth-sidecar` when unresolved). `CODEFLY_HOST_MODULE` is empty by default and -only needs setting to disambiguate a composition where **more than one** module -exposes the same role — otherwise an ambiguous match resolves to nothing and the -runtime fails loud at boot rather than picking a host arbitrarily. Concrete -addresses always come from the SDK. +its workspace module name: the runtime discovers the single module that owns the +gateway role — from the injected endpoint carriers when deployed, or the +workspace on disk when run locally — so a solution composing the host under any +module name resolves identically (codefly-dev/core#382). +The role is overridable: `CODEFLY_HOST_GATEWAY` (default `auth-gateway`). There +is no fallback to the pre-v0.0.49 `auth-sidecar` role: a composition exposing +only that one is a composition to re-render, and an unresolved current gateway +silently selecting the service that used to answer the deleted registration +endpoints is worse than a boot that fails naming the role. `CODEFLY_HOST_MODULE` is +empty by default and only needs setting to disambiguate a composition where +**more than one** module exposes the same role — otherwise an ambiguous match +resolves to nothing and the runtime fails loud at boot rather than picking a host +arbitrarily. Concrete addresses always come from the SDK. + +## The workload identity + +The listener presents the X.509-SVID issued to this workload, over TLS 1.3, and +there is no plain-HTTP listener. A configuration that cannot produce an identity +is refused at boot, naming the missing material: a solution that came up on +plain HTTP would be refused at the edge instead, for a reason only the edge can +see, which is the failure the delivered-presence model exists to end. + +The default source is the pair the platform projects at the two paths above, +read through the SDK's certificate reloader: it re-reads the files when they +change and keeps serving the last good pair through a half-written replacement. +That matters more here than anywhere else, because workload leaves are +deliberately short-lived and rotated well before expiry — a process that loaded +its leaf once at boot serves a stale leaf while a valid one sits on disk, and +then fails every handshake with the issuer reporting nothing wrong. + +Issuing an identity is the platform's job, not this runtime's, so the source is +a hook: + +```go +solution.New(manifest).Identity(mySPIFFEWorkloadAPI{}).Serve() +``` + +An `IdentitySource` returns the `*tls.Config` the listener serves with, so an +issuer with its own rotation, revocation or peer-verification rules keeps them. +A source that returns no certificate is refused rather than started: a listener +that completes no handshake reports a TLS error naming nothing to every caller. + +**The listener authorises its callers, not merely authenticates them.** Every +workload in the trust domain holds a certificate from the same anchor — +including the modules this solution consumes — so verifying one answers "did the +platform issue this identity", not "may this caller call me". The admitted set +is therefore declared and provisioned (`workload-identity`/`ALLOWED_CALLERS_FILE`) +and the boot is refused when it is absent, naming the value; the comparison is +the SPIFFE ID in the caller's URI SAN. Without it a consumed module could call a +handler or a passthrough route directly, bypassing the admission the host +decides on its own routes, and — because `x-org-id` and `x-session-id` are read +as headers the gateway stamped from a verified bearer — set them itself and have +this runtime mint capabilities under its attestation for an organization and +session nobody authenticated. A confused deputy, where the deputy is the one +process the issuer trusts to say which module is asking. The refusal names the +identity that arrived and never the admitted set. + +**The listener authenticates its callers, and so does the identity it presents +outward.** It requires and verifies a caller's certificate against the projected +anchor — not optionally, and not when one happens to be configured: a listener +that verifies no peer accepts anything that can route to the pod, which bypasses +the admission and exposure decisions the host makes on its own routes. An absent +anchor is a boot refusal naming the value; an unusable one fails the handshake +rather than falling back, because judging callers by a stale anchor lets in who +should be refused, which is not symmetric with serving a stale leaf. Peer trust +is re-read **per handshake**, so removing a compromised root from the bundle +stops it authenticating new callers without a restart. + +And every **established** connection is re-verified too, once a second, in both +directions — the caller's chain against the current anchor and the current +admitted set, the platform's against the same. Re-reading trust per handshake +bounds nothing about a connection that has already handshaken: a caller removed +from the admitted set, or whose issuing root was pulled, kept the keep-alive +connection it already held and kept being served on it, and so did an outbound +connection carrying a request every 100ms. An idle timeout cannot bound a +connection that is never idle. A connection whose peer stops verifying is closed +— busy or idle, request or stream — and nothing else closes it: a failure to +re-read *this* workload's own key pair is not a judgement about the peer and +leaves established connections alone. The listener also sets a handshake and +header deadline and an idle timeout, so a peer that connects and then stalls +mid-`ClientHello` no longer holds a goroutine and a descriptor indefinitely; +neither bounds a whole request, because a declared long-running stream is +conforming traffic. + +**A consumer's `IdentitySource` that sets `GetConfigForClient` must implement +`PeerAnchorSource`** — whatever it carries on the base configuration. Re-verifying an established caller, and +verifying the platform this runtime dials, both need the anchor *as it is now* — +and if the only way to get it is the source's `GetConfigForClient`, the runtime +would have to call it with a `ClientHelloInfo` nobody sent. That is the defect +this cutover was already blocked on one field along: a source keyed on the hello +answers a synthetic probe with one thing and the real handshake with another, +demonstrably. So the source is asked directly — +`PeerAnchor() (*x509.CertPool, error)` — and one that can only answer through a +handshake is refused at boot. Carrying a pool on the base as well does not +answer it: that pool is one object fixed when the source built it, so it would +freeze the anchor this process judges by while the callback went on resolving a +fresh one — which is exactly what happened when the refusal only fired on a nil +base pool. A source with no callback at all may carry its anchor on the +configuration and needs nothing extra. The projected source implements the +interface. + +Nor may a source set `VerifyPeerCertificate`: it is handed the parsed chain this +runtime reads the caller's identity from, and it runs first, so a source that +rewrote the leaf's URI SANs decided who was admitted. The identity is read from +a fresh parse of the raw DER regardless — the bytes the peer signed are the only +thing that cannot be edited between the handshake and the check — and the field +is refused as well. + +The same fabrication is still used for the outbound **certificate**, and that is +sound for a reason worth stating: whatever comes back is held to the frozen +principal at the moment it is presented, so a source answering differently from +the real handshake is caught by that hold. An anchor has no check behind it — it +*is* the check. + +**An outbound TLS handshake carries its own 10-second deadline.** A custom +`DialTLSContext` takes `net/http` out of the handshake, so +`Transport.TLSHandshakeTimeout` does nothing and the only bound would be the +caller's context — which is right for a request and wrong for the one place it +matters, since the client that carries streams sets no timeout on purpose. A +handshake is not a stream. + +**An admission set answered twice is refused, not ranked.** The env override and +the platform's provisioning are two sources for one authorization fact; if both +answer, the boot fails naming both. Logging which one won reports the conflict +without resolving it, and for admission the cost of picking wrong is admitting a +caller. The other `workload-identity` paths still take the override first: the +worst case there is this process reading its own material from elsewhere. + +**This listener does not resume TLS sessions.** A resumed connection presents +**no certificate** — the peer is accepted on a ticket — and this listener's +whole model is that a caller's certificate names them and the provisioned set +decides whether that name may call. So anything able to forge a ticket is able +to be admitted as whoever it was issued to. A review demonstrated it: with a +source-supplied ticket key, a client holding only that key and an admitted +caller's ticket resumed as that caller, presenting nothing, and the per-second +recheck then re-verified the stolen certificate and kept the connection open. +A denylist could not close that, because `SetSessionTicketKeys` installs keys +that cannot be read back off the configuration — so resumption is turned off on +the configuration served and on every per-connection answer. It costs one round +trip on a reconnect. + +**An admission entry must be a whole line.** The file is read per handshake and +per dial, so it is read while the platform may be rewriting it: a file caught +mid-write truncated an entry, and the truncation was admitted as an identity of +its own (`…/sa/gateway-internal` read as `…/sa/gateway`, admitting a caller the +set never named). A file that does not end in a newline is refused, every entry +must be a SPIFFE ID, control characters and a byte-order mark are refused, and +the file is capped at 64 KiB because it is re-read once a second per established +connection in each direction. + +**A credential-bearing URL carries a destination and nothing else.** Userinfo, a +query and a fragment are refused at boot on both the gateway and the mint: +credentials in a URL are a secret that gets logged and, for userinfo, that +net/http strips before the request is sent, so it is disclosed and never used. +Both URLs are logged through `url.Redacted()` with the query dropped regardless, +because a log line is the wrong place to depend on a check that runs elsewhere. + +**No credential-bearing request follows a redirect, and none leaves the +gateway's own origin.** Go copies a request's headers onto a redirected one and +strips only `Authorization`, `WWW-Authenticate` and `Cookie`; a 307 re-sends the +body; and this runtime sets the bearer per round trip, which puts back the one +header Go strips. So a single `307 Location: http://elsewhere/` would hand a +third party the viewer's bearer, the capability minted for them and this +workload's own credential, in cleartext — nothing about an `http://` hop involves +the TLS configuration that protects the first one. Every client is therefore +built with `ErrUseLastResponse` (a 3xx is reported, never taken), and the +transport refuses any destination that is not the gateway's origin, so a client +built later without the policy is still stopped. + +**Outbound trust is re-read per connection, not snapshotted at boot** — the same +rule as inbound, for the same reason, and it applies with more force: the two +destinations on the other side of that client receive the projected +service-account token, the viewer's bearer and this workload's own credential, so +a root removed because it was compromised must stop authenticating them without a +restart. Each dial builds its own configuration — and each +established connection re-verifies the peer it already has, once a second, +against the current anchor and the current the destination's own provisioned set, closing it when +that stops holding. + +The recheck is there because the idle cap it replaces was not a bound. An +inactivity timeout bounds a connection nobody is using; a connection carrying a +request every 100ms never becomes idle, and one was demonstrated still answering +31 seconds after its server's root was removed from the bundle, over a single +handshake. A stream is the same shape by construction, held open for as long as +its declared `MaxStreamDuration` allows. Re-verifying rather than imposing a +maximum connection age is deliberate: a maximum age would cut a conforming +30-minute stream to answer a question that can be answered without cutting it, +since the chain the peer presented is already in hand. A peer that no longer +verifies loses the connection, in flight or idle, stream or request. + +**And the destination is authorised, not just authenticated.** Verifying the +chain and the hostname says the cell issued a certificate for the address this +runtime dialled, which every workload in the cell holds one of — a neighbouring +workload answering at the gateway's address under a valid certificate was handed +the projected token, the viewer's bearer and this workload's credential. So the +peer's own SPIFFE ID must be in that destination's provisioned set. The outbound leaf is held to +the frozen principal too, at the moment it is presented, which matters because +on the default path it comes from a second reloader over the same files as the +listener's. + +The same identity goes out. Every platform request — the mint, and every call a +handler's gateway makes — presents this workload's X.509-SVID and verifies the +far end against the same anchor, because an `https` URL on its own says only +that the scheme is https: without an anchor the far end is checked against +whatever the image's system roots hold, and without a client certificate it +cannot tell this workload from anything else that reached it. Nothing is +proxied, and no redirect is followed. + +**The certificate served is the certificate checked.** The pair the platform +projects and the principal it provisioned are two facts nothing else compares, +so this runtime compares them: the SPIFFE ID in the leaf's URI SAN must equal +`module-authority/PRINCIPAL`. A pair projected for a neighbouring workload — the +wrong Secret mounted, a Certificate issued for another service — would otherwise +be served happily, and the mismatch would surface at whatever verifies this +destination, as a refusal naming neither file. + +The comparison is made on the pair a handshake actually selected, not on a +sample. A boot-time check that called `GetCertificate` once and approved the +configuration was demonstrated passing four configurations that then served +another workload's leaf: a callback keyed on the server name answering a probe +and a real hello differently, a `NameToCertificate` map, a static +`Certificates[0]` that Go reaches without consulting the callback at all when +the caller sends no SNI, and a `tls.Certificate.Leaf` that did not describe its +own DER. So the selection surface is narrowed to one certificate (a +`NameToCertificate` map or a list of several is refused — a workload has one +X.509-SVID), the DER is parsed rather than `Leaf` believed, and the callback is +wrapped so that what it returns for this handshake is what is compared. A +boot-time refusal remains, asked with the empty hello a peer addressed by IP +really sends, and it can only refuse a boot — never approve one. + +Every one of those applies to a consumer-supplied `IdentitySource` too, checked +on what it returns: a certificate, a TLS 1.3 floor, and a caller it +authenticates. Checking only that a certificate came back left the floor and +peer authentication silently optional for exactly the consumers who wrote their +own integration. + +It is checked **per connection** as well as at boot. A `*tls.Config` may carry a +`GetConfigForClient` callback, and the configuration that callback returns +replaces the base one for that handshake — floor, peer requirement and +certificate included — so a source that conforms at boot could answer each +individual connection with TLS 1.2, no peer authentication, or a neighbouring +workload's leaf. This is the documented shape here rather than an odd one: the +projected source uses that callback to re-read peer trust. So each returned +configuration is held to the same posture and the same frozen principal, and one +below it fails that handshake instead of serving it weakened. A callback that +returns `nil` is Go's "serve the base configuration", and passes through only +when that base really satisfies the posture on its own: a source answering for +one hello and not another was otherwise serving a listener that requires a +caller's certificate and verifies it against the host's system roots, which is +mutual TLS in every log line and a client certificate from any public CA +admitted. + +## One credential per execution + +`Serve` obtains this execution's credential once, before the listener exists, +through the SDK's mint client (`sdk-go/workcontext`): the projected +service-account token is presented to the host's mint endpoint, and the host — +which establishes the principal, the installation and the build from that token +and its own records, never from anything this process reports — answers with a +credential sealed to this build incarnation and this installation. + +The runtime holds no mint of its own, and it runs **no registration +heartbeat** — which is the claim this cutover is about. It does run other +timers, and saying otherwise here was wrong twice: the inbound and outbound +trust watchers re-verify an established peer once a second, the credential +acquisition holds off after a failure (capped, jittered), and the listener has +idle and header bounds. None of them tells the host this process exists, which +is the property; "no timers at all" was never it. Every use goes through the client, which hands back the +credential it holds while that one is current and renews it once it has entered +its renewal lead, re-reading the rotated projection and rechecking the frozen +authority values first. A process that runs for a week under credentials the +issuer chose to make long-lived mints once and renews a handful of times; it +never mints per beat or per probe, and does not mint per request: acquisition is +single-flight, a failed ask quiets the next ones for a second, and a credential +already in hand and unexpired is what serves meanwhile. That last part is not a +detail — renewal begins inside a lead *before* expiry, so refusing while the +issuer is slow would answer 503 to every viewer over a dependency this process +does not yet need. A route gate that asked per request turned a 503-ing issuer +into one audited mint per request, which is worse than the heartbeat this +runtime replaced, because the heartbeat at least minted on a timer. + +**A refused first mint fails the boot and is never retried. An unavailable one +is waited for, within a bound.** The two are different answers: + +- a **refusal** (`ErrMintRefused`, which the SDK latches for exactly two + statuses, `401` and `403`) is the issuer judging this workload: this build is not the one + its presence document approved, the pod's identity does not match, the token + attests to another subject. No number of attempts changes any of those + answers, so the boot reports what the issuer said and exits non-zero. A loop + around it is what this change exists to delete: the old runtime answered every + refusal by beating again, minting a fresh single-use token each time, so one + undeployable solution produced an audited mint every 15 seconds for as long as + it ran. +- an **unavailable** mint (`ErrMintUnavailable`: the host's `503`, `429`, any + other `5xx`, **and a `404`**) is not a judgement. A workload may legitimately start before its + presence generation has been applied, and then there is no incarnation to mint + against yet; the issuer may also be unable to reach the Kubernetes API or its + own policy log, and a host that issues nothing in those cases is behaving + correctly. So the boot asks again, backing off, for up to **two minutes**, and + then exits non-zero. + + A `404` is in this set, and that is a deliberate change rather than an + oversight: the rule here was "`429` and `5xx` retry, everything else is + terminal", and sdk-go v0.3.0 inverted it after measuring the cost — a `408` + from a proxy, with a valid credential in hand, latched for the life of the + process, and a `404` from an ingress mid-rollout would do the same. `401` and + `403` are the two statuses the host signs. So **an old runtime against a host + of this generation waits out its bounded window and exits non-zero for the + orchestrator rather than failing on the first answer.** It still does not + recover; what changed is the shape, not the outcome. + +The bound is what keeps the second case from being the heartbeat again: it is a +boot waiting for a dependency, with no steady state, nothing minted on success +beyond the one credential, and an exit — the orchestrator's cue to restart — +when it is spent. + +The credential is also what this runtime presents on the mint it runs **for a +viewer** (see [Reading a Work-Context-authenticated +module](#reading-a-work-context-authenticated-module)): the viewer's bearer says +on whose behalf, and this credential says which module is asking, so the issuer +can hold that mint to the installation and binding the credential is sealed to +instead of seeing only that somebody holding a viewer's bearer asked. + +**A mint this runtime cannot attest for is not sent.** If the credential cannot +be obtained — no source, a refused renewal, authority that drifted, a capability +that cannot be attached — the call is refused with `ErrNotAttested` and the page +sees `unavailable`: the viewer's own authority is not in question and one +renewal fixes it. + +It used to log and carry on, and that was wrong in the one case the attestation +exists for. The issuer stops renewing precisely when the installation has moved, +the build is no longer approved, or the principal's epoch has advanced — so a +workload in exactly that state kept minting viewer capabilities with no +execution binding on any of them. "The host does not require the attestation +yet" is a statement about the counterpart's current state, and boot-time +approval is not authorization at use. The refusal is logged once per distinct +failure rather than once per request, and once more when it recovers; what a +caller gets is not throttled, because every affected call is refused. + +**Nothing is inferred from a module's 409.** A capability is held until its own +expiry, whatever a callee answers. A 409 means "the state you were sealed to +has moved" and it means "that page already exists", and nothing on the wire +says which — so a runtime that read one as supersession evicted a perfectly +valid capability every time a module reported a duplicate or a lost update, and +the next call minted again. Matching installation metadata identifies the +*scope* of a conflict; it does not establish supersession. + +A genuinely superseded capability is therefore refused call by call by the far +end — correct if noisy — and the renewal that replaces it happens on the +credential's own schedule. Acting on a conflict needs a machine-readable +discriminator from the host or the verifier, which nothing publishes today; +what a viewer sees meanwhile is the 409 itself. + +A consumer whose issuer is reached another way supplies its own source: + +```go +solution.New(manifest).Credential(mySource).Serve() // Credential(ctx) (workcontext.Credential, error) +``` + +### This runtime neither signs, parses nor verifies a capability + +There is **one** implementation of a Work Context — Core's (`core/workcontext`: +deterministic proto, Ed25519) — and this runtime is not it. It obtains a +credential through the SDK's client, carries it as the string it travels as, +and lets whoever consumes it decide. It declares none of a capability's fields +and reads none of them: the capability a handler's gateway presents to a module +is a string this package never decoded, and the one check that must happen +before a call rather than at the far end — that a capability is sealed at all — +is the carrier's (`workcontext.Attach`). + +It also does not **verify**, and that is a position rather than an omission. +Core's verifier requires four live sources — the authorization revision, +replay, grants and seals — and refuses everything without them, deliberately, +so the strongest check in the model cannot become the easiest to skip. A +solution runtime holds none of those: it is the party presenting a capability, +not the party deciding on one. So there is no verifier here to configure, and a +change that adds one has to answer where those four sources come from. + +Core's conformance kit is still run, through the boundary this package does own: +all 39 fixtures are offered to the carrier, and each one's refusal is attributed +to a side. A capability whose bytes are visibly not a capability — no separator, +an unreadable payload, no seal, another encoding — is refused here, before a +request is sent, with Core's own sentinel. A tampered payload, an unknown key, +an audience in the payload and every sealed value the issuer has moved past are +carried, because judging them needs those four sources and a carrier that +refused them would be claiming a strength it does not have. The classification +is per fixture, so a fixture Core adds fails the test until somebody decides +which side it belongs to. + +Both halves are pinned by `work_context_boundary_test.go`, which fails on a +signing primitive, on a `WorkContext`-named type or function of this package's +own, and on a call to a verifier. The duplication it prevents already happened +once, in a module whose only job was to carry these things, and 3,532 lines had +to be deleted to get back to one implementation. Every step of that was locally +reasonable, which is why the first step fails a test here. + +A module's refusal of a presented capability reaches a handler by kind: +`ErrRevoked` — the capability was sound when minted and the state moved under +it (an installation revision, a principal's epoch, a build incarnation, a +binding) — is reported as `aborted`, because the call was not made and +retrying the same call will not make it. It does **not** retire the capability: +see "Nothing is inferred from a module's 409" above, since that answer is +indistinguishable from an ordinary business conflict. `ErrInvalid` and `ErrNotACoreToken` are reported as `internal`: a +credential this solution could not present is this solution's problem, not a +module that is briefly unreachable. There are two, not three: core v0.9.0 +**deleted** `ErrUnsealed`, because the seal is now required by the schema and +`protovalidate` refuses a capability carrying none before any seal check is +reached — so that case arrives as `ErrInvalid`, and a consumer who wrote a +branch for the dedicated sentinel had one that could never run. + +## The published contract + +A solution **declares** the most authority it may ever ask for: which audiences +it holds bindings for, and the ceiling on each. The runtime holds its own asks +to that declaration at boot, and answers it at `/.well-known/module-contract` so +an operator can read what a process holds itself to. + +> **This is not the document the renderer derives authority from.** That is +> `module.contract.codefly.yaml`, YAML, strict-decoded, read at the module +> directory the composition resolved, under the schema +> `codefly/module-contract/v1` — a shape whose audiences are +> `{from: /}` slots a composition resolves per environment, rather +> than ceilings keyed by profile (codefly-dev/cli#855). The runtime publishes +> JSON under its own schema string, `codefly/solution-runtime-contract/v1`, and +> nothing in this package writes the renderer's file. The two were briefly under +> one schema string, which is worse than a mismatch: strict decoding refuses the +> unknown fields, and a matching string makes that read as a *malformed* module +> contract rather than a document meant for someone else. + +```go +solution.New(manifest). + Consumes(solution.ConsumedModule{ + As: "documents", + Scopes: []solution.Scope{{ResourceKind: "documents", Actions: []string{"read"}}}, + Methods: []solution.ConsumedMethod{ /* … */ }, + }). + Contract(solution.ModuleContract{Ceilings: map[string]map[string][]solution.Scope{ + "local": {"documents": {{ResourceKind: "documents", Actions: []string{"read", "list"}}}}, + "staging": {"documents": {{ResourceKind: "documents", Actions: []string{"read"}}}}, + }}). + Serve() +``` + +The ceiling is **declared, never derived**. A ceiling computed from what the +code asks for would be satisfied by construction — a method that asked for one +more action would widen the ceiling meant to refuse it, and a reviewer approving +the contract would be approving whatever the next commit asks for. What *is* +derived is the binding set: the audiences are the `as` of the solution's +`api.consumes` entries, which Codefly projects and the declaration is already +checked against, so restating them would be a second source for one fact. The +principal is neither: it is the `module-authority` value the platform +provisioned, so the contract reports who this workload actually is rather than +who its author believed it would be. + +Keyed by **configuration profile**, because a deployment and a developer machine +do not grant the same authority. A deployed environment reads its own profile — +`staging`, say — and never the local one; that distinction is recent +(codefly-dev/core#687, closed in core v0.7.1), and the keying is what makes the +old mistake impossible. A contract that declares only `local` is refused in a +deployment, naming the profile it ran under, the profiles the contract does +declare, and the override that changes it. + +Every one of these is a boot failure naming the value: + +| Refused | Because | +|---|---| +| no ceiling for a consumed audience | this runtime would have no declared ceiling to check a mint against, for a module it does call | +| a ceiling for an audience nothing consumes | the ceiling governs a call that cannot happen, and whoever wrote it believes otherwise | +| an ask outside its ceiling (action, or resource id) | the ceiling is what a reviewer approved; widening it is an edit, not an inference | +| an ask across a whole resource kind under a ceiling naming resources | "every document" is not inside "these two documents" | +| a ceiling on a `ViewerBearer` module | it mints nothing, so the ceiling governs nothing — and whoever wrote it believes it does | +| no profile at all, for a solution that mints authority | a deployed process would have no profile to resolve its ceiling from | +| a profile name that is not a single path component | profile names select directories wherever one is read | + +A running solution publishes the contract of the profile it runs under at +`/.well-known/module-contract` — the resolved principal, the profile, and each +binding with its ceiling and the ask inside it. Nothing is pushed: answering +there makes this solution present to nobody. `ContractArtifact(id, contract, +modules...)` renders the same document at build time, with every profile and no +principal — that is a value only the deployment knows — checked by exactly the +rule the boot applies, so a contract that would refuse to boot cannot be +published. It is for review and for diffing a build, not for the renderer. ### Where the host reaches the solution -The two registrations carry two different addresses, for two different -callers, and neither is this process's listen address: - -- **`manifestUrl`** (host registration) is loaded by the **viewer's browser**. - With no `PUBLIC_URL` it is the root-relative path `/assets/mf-manifest.json` - on this backend, and the host resolves it against the route by which it - reaches the solution — the runtime does not know, and does not encode, the - host's route layout. `PUBLIC_URL` makes it absolute on that origin instead, - for an operator who exposes the solution's assets directly. (The one place the - runtime does encode that route is the MCP resource identifier, which has to be - byte-exact for a client and cannot be resolved by anyone else — see [The - resource identifier and the issuer](#the-resource-identifier-and-the-issuer).) -- **`upstream`** (gateway registration) is dialled by the **gateway**. It is - the address Codefly injects for reaching this service - (`CODEFLY__SELF_ENDPOINT__…`, beside the `CODEFLY__ENDPOINT__…` carrier that - stays the listen address), and `SELF_UPSTREAM` overrides it. `PUBLIC_URL` no - longer feeds it: the browser's origin is not the gateway's route. - -In a **deployed** runtime context (`CODEFLY__RUNTIME_CONTEXT` outside the local -kinds above) `validate()` refuses to boot when either address is loopback -(`localhost`, `*.localhost`, `127.0.0.0/8`, `::1`, the unspecified address): a -deployed solution that registered its listen address booted, looked healthy, and -was proxied by the gateway to the gateway itself, while the product reported it -as failed to load. The environment's *name* is never consulted — an environment -called `local-dogfood` may well be deployed. - -Until core v0.5.6 is released, the self endpoint is read from that carrier by -name in one helper (`selfEndpoint`), and a cell rendered by an older core has -neither the carrier nor the runtime-context signal: it falls back to the listen -address and is not refused. Both switch to the core/SDK accessors when released. +Neither address this runtime used to report is reported any more: the presence +document names the route and the destination, and the host resolves both. + +- The **Module Federation manifest** is served at `/assets/mf-manifest.json`, + a path on this backend, and that is the whole of what this runtime says about + it. It used to be an absolute URL, built from `PUBLIC_URL` or — with none set + — from this process's own listen address, so every deployed solution + registered a manifest no browser could load and the product showed it as + failed to load. The origin a browser reaches this solution through is the + host's route, and naming the host's route layout here would couple every + solution to it. +- The **upstream the gateway dials** is in the presence document, not in a + registration. `SELF_UPSTREAM` and the self-endpoint carrier it fell back to + are gone. A deployed solution used to register its own loopback listen + address, boot, look healthy, and be proxied by the gateway to the gateway + itself while the product reported it as failed to load; the refusal that + caught that is gone with the registration that needed it. + +This solution still publishes what it is, at fixed paths, for anything that +asks: `/.well-known/solution.json` (the manifest: nav, the exposed module, the +federation manifest path, the backend's service alias, declared surfaces and an +optional dashboard), `/.well-known/capabilities` (the contract id and the +majors), `/.well-known/module-contract` (the authority contract above) and +`/health`. Publishing is not announcing — nothing is pushed, and answering +makes this solution present to nobody. + +`/health` **asks the credential** rather than reporting only what some other +request happened to discover. Renewal is lazy by design, so a solution serving +only ViewerBearer routes, plain handlers and assets would otherwise never ask and +never learn that the issuer had stopped approving its build. Asking is not a +heartbeat: the client holds one credential and returns the same one until its own +renewal point, so a probe is a mutex and a comparison except at the renewal the +credential's expiry dictates. A credential that cannot currently be obtained is +503 and nothing ends; a refusal is 503 **and** ends the process. + +So `/health` answers 200 while this process can do its job, and **503 once the +issuer has refused this execution's credential for good** — `ErrMintRefused` at +renewal, which is a judgement about this execution that no retry changes. +(`ErrRevoked` is not one of these: it never comes back from the mint, only from +a callee rejecting a capability this runtime minted for a viewer, which is a 409 +on that request.) The process then stops serving and returns that reason, for the +orchestrator to restart it against the delivery as it stands. An unconditional +200 was wrong in exactly the case the probe exists for: the boot already treats a +refusal as terminal, while at renewal the same judgement reached a page as +"unavailable, one renewal fixes it", so a solution would answer 503 to every +request forever and report itself healthy throughout — a solution that serves +nothing and looks alive, which is what the delivered-presence model replaced the +heartbeat to avoid. An *unavailable* mint is not a judgement and deliberately +does not do this: exiting on a transient dependency failure is a crash loop. + +The manifest declares the contract majors it is built against +(`schemaVersion: 1`, `frontend.hostContract: 1`) rather than leaving a reader to +assume them, and the capability document declares the same majors from the same +constants — so a bump cannot leave one document announcing the old major while +the other announces the new one. + ### Serving the frontend @@ -133,149 +734,24 @@ and the remote entry above all, whose names never change — `no-cache`, as is any response that is not the file (a `404` for a hashed name must not be cached under the name the next build will serve). -### Self-registration - -On boot the runtime self-registers on a 15s heartbeat with **both** the host -frontend (host registration: the manifest) and the gateway (gateway -registration: the upstream). - -A host on module-saas-starter ≥ v0.0.61 binds each registration to the -solution's **publisher** (`module/SOLUTION_REGISTRATION.md` there): both -surfaces admit a registration only against a short-lived, solution-bound token -that accounts mints to a caller presenting the registration secret the -composition declared for that id, and neither accepts the shared -cluster-internal token any more — it attests to no publisher. The runtime -obtains that token by presenting its **solution registration secret** (plus the -internal token, the gateway's perimeter check) to -`/solutions/_registration-token`, and presents it as -`X-Codefly-Solution-Registration` on both registrations. Each surface burns a -token on use, so — unlike the module credential below — nothing is cached: a -fresh token is minted for every beat, on each surface. - -Provisioning both halves is the composition's job, and works the same for a -local `codefly run solution` and a deployed cell: - -- **host side** — declare `:sha256hex` in the saas `federation` - group's `SOLUTION_REGISTRATION_SECRETS` (separately from - `MODULE_REGISTRATION_SECRETS`: a solution credential additionally publishes - host-origin code, so the two are never shared); -- **solution side** — provision the plaintext as the `SECRET` key of the - `solution-registration` workspace secret group - (`codefly config generate solution-registration SECRET` locally; the cell's - secret store when deployed) and declare that group as a - `workspace-configuration-dependencies` entry of the solution's backend, so - the SDK injects it. `CODEFLY__SOLUTION_REGISTRATION_SECRET` is an explicit - override. - -There is no other credential, so there is no fallback. A boot without a -provisioned secret is refused by `validate()`, naming the two provisioning -paths, rather than coming up looking healthy while the host serves nothing. -A `404` on `/solutions/_registration-token` fails the beat and names the route: -either the host does not serve publisher-bound registration (module-saas-starter -< v0.0.61, which this runtime does not support) or the exchange URL is wrong -(`GATEWAY_SOLUTION_REGISTRATION_TOKEN_URL`, or the `GATEWAY_REGISTER_URL` it is -derived from). The shared cluster-internal token is never presented on a -registration: it proves no publisher, and a downgrade path would let anything -able to answer `404` at that URL turn the publisher-bound credential back into -it. - -Only a `401`/`403` from the exchange is reported as a credential fault (no -`SOLUTION_REGISTRATION_SECRETS` entry for this id, or a secret that does not -match its digest) — those are the statuses accounts answers after judging the -secret. A `5xx` is reported as `host unavailable (status N), retrying` with the -gateway's own error body, and any other status as a failure with its body; none -of them names the provisioning, and the heartbeat keeps retrying all of them. - -A refusal that carries reasons (the host's `409 incompatible_runtime`, say) is -logged with them, again whenever the reasons change and not only when the status -does — up to a handful of distinct reasons per status, after which the log says -it is suppressing them. The reasons are text the host chooses, so one that -varies per attempt (the `jti` of the token it just burned, say) must not be able -to turn the log into a stream of one line per beat. - -Two consequences of deriving the exchange from the registration endpoint are -worth stating outright, because both turn a working deployment into a failing -one: - -- `GATEWAY_REGISTER_URL`, if you override it, **must end in - `/solutions/_register`** — that suffix is what the exchange URL is derived - from by swapping it. An override with any other path cannot be paired, and - the boot is refused naming both this variable and - `GATEWAY_SOLUTION_REGISTRATION_TOKEN_URL`, which you can set explicitly - instead. This variable was free-form before the exchange existed. -- Both self-registrations now mint through the gateway, so a gateway outage - fails the **host frontend** registration too, even though the frontend is - healthy. A beat that never reached its registration surface minted nothing, - so it retries on a much shorter backoff cap (30s) than a beat the surface - actually refused (2m) — the long cap exists to bound audited mints, and an - unreachable dependency produces none, so paying it there would only keep this - solution out of a healthy host's nav for longer than necessary. - -Every registration request refuses to follow redirects, for the same reason none -of them may be proxied: `net/http` strips only `Authorization`, `WWW-Authenticate` -and `Cookie` when a redirect crosses hosts, so the `X-Codefly-*` headers these -requests carry — the plaintext registration secret and the cluster-internal -token — would be handed to whatever a `Location` named. - -A beat is not free any more: each self-registration beat runs an exchange whose -every success is an audited mint on the issuer, so at the 15s default across two -surfaces a solution mints roughly 11.5k tokens a day. The right period is -whatever the host's registration TTL allows, which this runtime cannot observe — -set `CODEFLY__SOLUTION_REGISTRATION_INTERVAL` (a Go duration, minimum `1s`) when -you know both numbers. An unparseable or too-small value fails the boot rather -than falling back to the default, so a typo cannot silently restore the fast -beat you were trying to slow down. +### Consumed-module federation -The manifest declares the contract majors it is built against -(`schemaVersion: 1`, `frontend.hostContract: 1`) rather than leaving the host -to assume them; a host checks both before activating a remote. +A solution that declares `api.consumes` reads each consumed module through the +gateway's `/v1//*` prefix. **Those routes are delivered**, like everything +else: the gateway holds them in its durable registry, from the presence +documents, and this runtime registers no upstream for them. The three +heartbeats that used to — one per consumed module, each brokering a signed +prefix-bound token through the gateway, plus the two self-registrations — are +deleted, along with `CODEFLY__MODULE_REGISTRATION_SECRETS` and the per-module +secrets it carried. -### Consumed-module federation +Codefly still projects the targets into `CODEFLY__API_CONSUMES`, and that +projection is still what the passthrough declaration is checked against — at +boot, resolved like every other value, so a malformed one is a refusal rather +than a log line. It used to be read at serve time: a malformed projection +disabled the whole federation with one log line while the solution served on, +so every consumed facade 404'd at the gateway and the solution looked healthy. -A solution that declares `api.consumes` also registers each consumed module's -upstream with the gateway, so `/v1//*` proxies to it. Codefly projects the -targets into `CODEFLY__API_CONSUMES`; each gets its own 15s registration -heartbeat alongside the two above. - -The gateway does **not** accept the shared internal token on `/modules/_register` -— it admits a registration only against a short-lived token signed by accounts -and bound to a single prefix, so holding the credential for one module never lets -you claim another's route. The runtime obtains that token per consumed module by -presenting the module's own **registration secret** to -`/modules/_registration-token`, which the gateway brokers to accounts (a composed -module cannot reach accounts' internal listener itself). The token is reused -until shortly before it expires — or until half its life is gone, if the issuer -chose a lifetime shorter than that lead, since how long a credential lives is -the issuer's call and not this runtime's to veto. - -Obtaining one is an audited security event on accounts, so the runtime does not -answer every failure by obtaining another. A refusal buys exactly one fresh -token: the gateway refusing a token minted moments earlier is not refusing it for -being stale, and re-minting cannot fix whatever it is refusing it for. Beats that -fail also back off, doubling up to two minutes, so a broken gateway costs a -bounded number of exchanges however long it stays broken — and the runtime keeps -retrying, at a rate that will not flood the issuer's audit log, until it -recovers. A response carrying no usable expiry is refused rather than cached, and -its two causes — an issuer that sent no `expiresAt` at all, and a clock skewed -past the credential's lifetime — are reported separately, because they have -different fixes. - -No registration request is unbounded: a gateway that accepts one and never -answers surfaces as a failed beat rather than silently parking that heartbeat -for the life of the process. - -Registration traffic — the exchange and all three heartbeats — never goes -through an HTTP proxy: every target is composition-local, and these requests -carry the registration secret, the internal token, and the signed token in -headers. - -Those secrets arrive in `CODEFLY__MODULE_REGISTRATION_SECRETS` as -comma-separated `prefix:secret` entries — the plaintext twin of the -`prefix:sha256hex` digests the same composition declares to accounts -(`MODULE_REGISTRATION_SECRETS`). A consumed module with no secret cannot be -registered, so the runtime skips it with a log naming this variable rather than -beating against a guaranteed 401. Provisioning both halves is the composition's -job (`codefly run solution`). ### Reading a Work-Context-authenticated module @@ -297,7 +773,9 @@ resp, err := solution.Unary[Req, Resp](ctx, docs, "/docs.v1.Documents/List", &Re `ForModule` mints a Task Work Context through accounts' `StartTask`, presenting the viewer's bearer so accounts resolves the same subject the module would have -seen. It names no actor principal, which makes the viewer both owner and actor +seen, and this execution's own credential beside it so the issuer knows which +module is asking (see [One credential per +execution](#one-credential-per-execution)). It names no actor principal, which makes the viewer both owner and actor of the Task; the audience is the module; the authority is the scopes asked for and nothing more. The returned gateway then carries **both** credentials — the bearer and the capability — on every request, so a module verifying either one @@ -331,6 +809,62 @@ into, so the Task it roots is attributable — the mint accounts journals names session that asked for it. A session id the runtime invented would be a well-formed UUID naming no session, attributable to nothing. +Every mint also **names the installation it acts under**. The host requires it +(core's `StartInput.InstallationID`) and refuses a mint naming none, and an +organization is not a substitute: one org may hold several installations, so a +capability minted without naming one is attributable to the org and to no +deployment inside it. + +For a viewer-driven mint it is the viewer's own, read from the +`x-codefly-installation-id` header the gateway stamps — on the same footing as +`x-org-id` and `x-session-id`, which the gateway also stamps from verified +claims and replaces whatever the caller sent. A request naming no installation +is refused, and the refusal is deliberately *not* a `ClientError`: no caller +can stamp one, so telling a viewer to fix it would point them at something that +is not theirs. + +This section described the opposite for two rounds — the installation read from +the **seal** of a carried capability, with the header as a fallback. That is the +SDK's rule for the direction where a capability *is* the authority being +carried: outbound, where `Attach` puts the seal's own installation beside it and +the far end refuses the two if they disagree. Inbound, that carrier is +caller-controlled — a browser can send `x-codefly-work-context`, and a +caller-supplied capability authenticates nothing here — so reading the +installation out of it would let a caller name the deployment their own mint is +attributed to. + +A background or delegated call would carry the installation its parent +capability was sealed to, read off this execution's held credential. **This +runtime has no such path today** — every mint it makes is driven by a viewer +request, and `ForModule` refuses one without an organization and a session. + +This execution's own credential is sealed to an installation by the host, from +the projected service-account token; the runtime names none on that mint and +cannot, since the SDK's `MintOptions` carries no such field. What it does is +refuse a credential sealed to no installation at all, and refuse a *renewal* +sealed to a different one than the credential this process has been acting +under: that is a different identity arriving through the renewal path, not a +renewal, and it would make the mints before and after it name different +deployments with nothing recording that they did. + +**An MCP tool call is held to the same credential as a handler.** A tool +receives a gateway carrying the viewer's bearer, so running one while the issuer +has withdrawn this execution's credential is acting for a viewer without +authority — and a tool that mints nothing still sends that bearer wherever it +dials. Every MCP method except the handshake, the keepalive, the listings and +notifications goes through the same check a handler route does, and the viewer +is told the same sanitized sentence. The list is of the methods that act for +nobody, so a method a future MCP SDK adds is checked rather than exempt. + +**An MCP error result never names an address this runtime dialled.** A tool's +own refusal reaches the agent client unchanged, but a result that would carry +the mint's or the gateway's address — in its text, or in its structured content +— is replaced with one sentence, and the replacement is logged with what was +caught. What is published is built from strings and bytes this runtime owns: +every marshaller runs first, then nothing is inspected that is not about to be +published. Content whose serialization this runtime cannot read is replaced +rather than passed on. + Rooting the Task there does **not** make the capability revocable, and nothing here should be read as saying it does. The edge verifies a presented context's signature and validity window, not the liveness of the session named in it, so @@ -373,9 +907,15 @@ is not refused: the lead is clamped to half its lifetime, with a one-time log, because how long a credential lives is the issuer's call and not this runtime's to veto. -This traffic is not proxied, for the same reason registration traffic is not: -every gateway target is composition-local, and these requests carry the viewer's -bearer and the capability minted for them in headers. +This traffic is never proxied: every gateway target is composition-local (its +address is resolved from the SDK's endpoint map), and these requests carry the +viewer's bearer and the capability minted for them in headers. The credential +mint this runtime runs for itself is not proxied either, and refuses to follow +a redirect: it presents the projected token that attests which workload this +process is, and `net/http` strips only `Authorization`, `WWW-Authenticate` and +`Cookie` when a redirect crosses hosts — so anything able to answer at the mint +URL with a `Location` would be handed that token, and the theft would look like +an ordinary successful mint. ### Calling a composed module over its REST binding @@ -531,11 +1071,11 @@ message of it). #### Testing the passthrough: `passthroughtest` -`Serve` resolves the gateway and api.consumes from the composition and -registers with the host, so a solution cannot run its own passthrough in a -test through it. Package +`Serve` resolves the gateway and api.consumes from the composition, mints this +execution's credential and listens with this workload's identity, so a solution +cannot run its own passthrough in a test through it. Package `github.com/codefly-dev/solution-runtime-go/passthroughtest` serves the real -passthrough — `Server.PassthroughHandler`, the handler `Serve` mounts, behind +passthrough — the handler `Serve` mounts, behind the same boot check — over an `httptest` server, against a fake host: ```go @@ -560,6 +1100,28 @@ cancellation. It never proves that a real host admits the solution, that accounts grants the scopes, or that the real module answers its binding the way the test's stand-in does. +**This seam is reachable only from here.** `Server.PassthroughHandler` was +exported until this change, and that made it a production bypass: a solution +could build a usable, credential-bearing handler that had skipped `validate()`, +the mTLS boot, the caller allow-list, the published ceiling and authenticated +outbound — a deployment calling it completed a viewer mint and a module call +over plaintext and answered 200, with the viewer's bearer and this workload's +own credential on the wire. Supplying a credential source, which a deployment +does, defeated the "no source, nothing to mint with" mitigation. It is reached +through `internal/seam` now, which Go's internal-package rule keeps inside this +module. The fake host is still plaintext and still has no contract, and those +are now properties of this module's own tests rather than of an API. A consumer +that called `PassthroughHandler` directly uses `passthroughtest.Handler` +instead. + +`passthroughtest` itself is importable by any module, so its own constructors +**panic outside a test binary** (`testing.Testing()`). Taking a `testing.TB` is +not the gate it resembles: `testing.TB`'s unexported method stops a type +*declaring* the interface, not a type that embeds it, so a few lines of +production code satisfy it. `testing.Testing()` is wrong for the root package — +where the legitimate caller is production code — and exactly right here, where +every legitimate caller is a test. + ### Exposing an MCP server A solution exposes its experience to an agent client — Claude Code, Claude @@ -575,11 +1137,18 @@ solution.New(solution.Manifest{ID: "wiki", Title: "Wiki"}). Serve() ``` -A composition that renders this solution supplies **one** declared value for it, -and only in a deployment: `issuer-url` in the `mcp` workspace-configuration -group, declared as a workspace-configuration dependency of the solution's -backend — the origin MCP clients authenticate against. Everything else is -derived or resolved; see [The resource identifier and the +A composition that renders this solution supplies **two** declared values for +it, and only in a deployment, both in the `mcp` workspace-configuration group, +declared as a workspace-configuration dependency of the solution's backend: + +- `issuer-url` — the origin MCP clients authenticate against; +- `public-url` — the URL clients dial, ending in `/mcp`. + +**This said *one* value, and a deployment following it does not boot.** The +public URL is derived from `PUBLIC_URL` **for a local run only**: a deployed +runtime context refuses a derived identifier, because the derivation encodes +the route the *host* serves this solution on. Both keys are required there, and +the boot names whichever is missing. See [The resource identifier and the issuer](#the-resource-identifier-and-the-issuer). `register` receives the official SDK's own `*mcp.Server` @@ -599,7 +1168,7 @@ func askWiki(ctx context.Context, _ *mcp.CallToolRequest, in askIn) (*mcp.CallTo if err != nil { return nil, askOut{}, err } - robin, err := gw.ForModule(ctx, "robin", solution.Scope{ResourceKind: "conversations", Actions: []string{"create"}}) + notes, err := gw.ForModule(ctx, "notes", solution.Scope{ResourceKind: "conversations", Actions: []string{"create"}}) if err != nil { return nil, askOut{}, err } @@ -661,10 +1230,16 @@ So it is **derived by construction** from the two things the runtime already has: the origin this product is reachable at and this solution's id — `/solutions//mcp`. This is the one place the runtime encodes the gateway's solution route (see [Where the host reaches the -solution](#where-the-host-reaches-the-solution)), and the reason it is worth -encoding is that it makes a composition declare nothing: a deployment that -already sets `PUBLIC_URL` to the origin browsers reach the product at has -addressed the MCP surface too. +solution](#where-the-host-reaches-the-solution)). + +**That derivation is for a local run only, and this said otherwise.** It read +as "a deployment that already sets `PUBLIC_URL` … has addressed the MCP surface +too", and a **deployed runtime context refuses a derived value at boot** +(`mcp.go`, `validateMCP`): the derivation encodes the route the *host* serves +this solution on, which this runtime does not know. So following that sentence +in a deployment failed the boot. In a deployed context the identifier is +**declared** — `mcp/public-url` in the workspace configuration, ending in +`/mcp` — and `PUBLIC_URL` addresses nothing about MCP there. A host whose gateway fronts solutions under some other route declares `public-url` in the `mcp` group instead. It must end in `/mcp` — the metadata @@ -724,33 +1299,63 @@ claude mcp add --transport http wiki https:///solutions//mcp \ Once the host supports it, the gateway's 401 challenge and the metadata document above are the whole of what a client needs to authenticate on its own. -#### Testing the tools: `MCPHandler` +#### Testing the tools: there is no supported path yet + +**This section documented `s.mcpHandler`, and a consumer cannot call it.** The +method is unexported, so the example could not compile from a consuming +package; it also said `Serve` "registers with the host", which is the thing +this cutover deleted. + +It was exported once, as `Server.MCPHandler`, and unexporting it was not a +tidy-up: an exported method handing back a servable handler built without the +boot is the `PassthroughHandler` incident verbatim — a handler that has skipped +`validate()`, the mTLS boot, the caller allow-list, the published ceiling and +authenticated outbound, carrying the viewer's bearer and this workload's own +credential over whatever it is mounted on. `TestNoExportedPathHandsOutAServableHandler` +fails if it comes back. So the example is removed rather than repaired, and +restoring the export to make it compile is the one fix that is not available. + +What a consumer can do today is drive their tools as ordinary functions and +test the authority each one asks for. What is missing is the MCP equivalent of +`passthroughtest` — a seam reached through `internal/seam`, whose constructors +panic outside a test binary, that runs the real MCP surface against a fake +host. That is a follow-up on #45, not a capability this README should describe +as existing. -`Serve` resolves the gateway and the issuer from the composition and registers -with the host, so a solution cannot drive its own tools in a test through it. -`Server.MCPHandler` serves the MCP surface exactly as `Serve` serves it — the -same endpoint, the same metadata document, the same refusals, mounted by the same -code — against an environment the test supplies. Nothing else of the solution is -served and nothing registers anywhere: +What a green test there proves is the solution's side: the declaration, the +authority each method mints, the fields that reach the page, stream bounds and +cancellation. It never proves that a real host admits the solution, that +accounts grants the scopes, or that the real module answers its binding the way +the test's stand-in does. -```go -handler, err := s.MCPHandler(solution.MCPEnvironment{ - GatewayURL: fakeHost.URL, // answers the accounts StartTask mint and /v1//* - IssuerURL: "https://host.test", -}) -server := httptest.NewServer(handler) -session, err := mcp.NewClient(&mcp.Implementation{Name: "agent", Version: "v0"}, nil). - Connect(ctx, &mcp.StreamableClientTransport{ - Endpoint: server.URL + solution.MCPPath, - HTTPClient: &http.Client{Transport: stampsTheViewersIdentity{}}, - }, nil) -``` +**This seam is reachable only from here.** `Server.PassthroughHandler` was +exported until this change, and that made it a production bypass: a solution +could build a usable, credential-bearing handler that had skipped `validate()`, +the mTLS boot, the caller allow-list, the published ceiling and authenticated +outbound — a deployment calling it completed a viewer mint and a module call +over plaintext and answered 200, with the viewer's bearer and this workload's +own credential on the wire. Supplying a credential source, which a deployment +does, defeated the "no source, nothing to mint with" mitigation. It is reached +through `internal/seam` now, which Go's internal-package rule keeps inside this +module. The fake host is still plaintext and still has no contract, and those +are now properties of this module's own tests rather than of an API. A consumer +that called `PassthroughHandler` directly uses `passthroughtest.Handler` +instead. + +`passthroughtest` itself is importable by any module, so its own constructors +**panic outside a test binary** (`testing.Testing()`). Taking a `testing.TB` is +not the gate it resembles: `testing.TB`'s unexported method stops a type +*declaring* the interface, not a type that embeds it, so a few lines of +production code satisfy it. `testing.Testing()` is wrong for the root package — +where the legitimate caller is production code — and exactly right here, where +every legitimate caller is a test. + +> **This whole section was duplicated.** Two near-identical copies of +> "Exposing an MCP server" sat 200 lines apart, which is why correcting one +> passage left the same error in place — the `mcpHandler` example was removed +> twice and the `PUBLIC_URL` claim corrected twice before the cause was +> obvious. The stale copy is deleted. -What a green test there proves is the solution's side: the tools it offers, the -authority each one mints, and the refusals its callers get. It never proves that -a real gateway stamps what the test's `RoundTripper` stamps, that accounts mints -a Work Context, or that a real MCP client's OAuth flow completes against the -host. ### Generated messages in a response @@ -772,7 +1377,18 @@ source of what is served and what is published, and an operation missing its documentation is refused. A `Message[T]` field is described from its descriptor, with protobuf JSON names. -> **Note:** SDK in-process endpoint resolution for a solution composed on an -> out-of-repo host depends on codefly-core accepting the composed module path in -> its workspace loader (codefly-dev/core#365, merged); the `core`/`sdk-go` pins -> in `go.mod` carry that fix, so no `replace` is needed. +> **Note on pins.** SDK in-process endpoint resolution for a solution composed +> on an out-of-repo host depends on codefly-core accepting the composed module +> path in its workspace loader (codefly-dev/core#365, merged); the +> configuration-profile rule and `core/workcontext` come from core; and the +> certificate reloader, the authority reader and the mint client come from +> sdk-go. The `core` and `sdk-go` pins in `go.mod` carry all of it, and there is +> **no `replace` directive** — both are ordinary pseudo-version `require`s, so +> this module builds from its own tag for anyone. +> +> Both pins are **released** now: `core v0.9.0` and `sdk-go v0.3.0`, with the +> `sdk-go/workcontext` leaf at a `main` pseudo-version because that leaf module +> has no tag by design. This said they were unreleased branches that "must move +> to the releases at merge"; they moved. Every capability's format changed in +> that cutover, so verifiers upgrade before minters or calls fail closed in the +> window — which is a statement about the rollout order, not about these pins. diff --git a/boot_test.go b/boot_test.go new file mode 100644 index 0000000..4e4a641 --- /dev/null +++ b/boot_test.go @@ -0,0 +1,1709 @@ +package solution + +import ( + "cmp" + "context" + "crypto/ed25519" + "crypto/rand" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/json" + "encoding/pem" + "errors" + "fmt" + "io" + "math/big" + "net" + "net/http" + "net/http/httptest" + "net/http/httptrace" + "net/url" + "os" + "path/filepath" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/codefly-dev/core/solution/manifest" + corework "github.com/codefly-dev/core/workcontext" + codefly "github.com/codefly-dev/sdk-go" + "github.com/codefly-dev/sdk-go/workcontext" +) + +// The authority-bearing values every test in this package boots under. They are +// one set on purpose: the SDK pins an authority-bearing value process-wide at +// the first read, so two tests reading the same name with different values +// would have the second refused as a value that drifted — which is the +// behaviour, not a test defect. +const ( + // testSolutionID and testSolutionTitle are a solution's identity in these + // fixtures. Neutral on purpose: this runtime knows nothing about any + // specific solution, and a product's name in its fixtures is a name the + // next reader copies into production. + testSolutionID = "widgets-go" + testSolutionTitle = "Widgets" + + testPrincipal = "spiffe://codefly.test/ns/solutions/sa/widgets" + testGatewayPrincipal = "spiffe://codefly.test/ns/platform/sa/gateway" + testAudience = testSolutionID + testProjectionAudience = "accounts" +) + +// The sealed values the stand-in host issues against are core's own conformance +// values: the credential is minted by core's authority from core's fixture +// identities, so the seal this runtime logs and a test asserts is the live one +// that authority holds rather than a number invented here. +const ( + testInstallation = corework.FixtureInstallation + testBuildIncarnation = uint64(corework.FixtureBuildIncarnation) +) + +// authorityValues provisions the module-authority group the way the platform +// does, through the carriers the SDK reads. +func authorityValues(t *testing.T) { + t.Helper() + for key, value := range map[string]string{ + AuthorityPrincipalKey: testPrincipal, + AuthorityAudienceKey: testAudience, + AuthorityProjectionAudienceKey: testProjectionAudience, + } { + t.Setenv("CODEFLY__WORKSPACE_CONFIGURATION__MODULE_AUTHORITY__"+key, value) + } + if err := codefly.LoadEnvironmentVariables(); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = codefly.LoadEnvironmentVariables() }) +} + +// hostMint stands in for the host's mint endpoint: it reviews the projected +// token the workload presents and answers with a credential sealed to this +// execution, exactly as the real one does — signed, because the SDK's client +// reads the seal out of the signature and not out of the response body. +type hostMint struct { + *httptest.Server + authority *corework.Authority + + // status, when non-zero, is answered instead of a credential: 403 for a + // workload the host refuses, 503 for an issuer that has nothing to mint + // against yet. + status int + // callers records the SPIFFE identity each request presented, so a test + // can assert the runtime named itself on the way out as well as in, and + // paths records what was asked for, so a surviving registration is visible + // as the path it would have used. + callers []string + paths []string + // recoverAfter, when positive, is how many times status is answered before + // the host starts issuing: an issuer catching up with a presence + // generation that has just been applied. + recoverAfter int64 + // ttl is how long the credential it issues is valid. + ttl time.Duration + // switchTo and switchAfter make this host answer with a DIFFERENT shape + // once switchAfter credentials have been minted: the renewal path, driven + // through the real client rather than by calling the wrapper directly. + switchTo *hostMint + switchAfter int64 + // cell is the anchor this mint's own identity is issued under, and the one + // a caller's certificate is verified against. Set by every constructor, + // because the mint endpoint now requires a client certificate and is + // itself admitted by the SPIFFE ID in its single URI SAN. + cell *cell + // caller is the leaf a test's own mint client presents, issued under the + // same anchor. The booted path presents the runtime's projected pair + // instead; this is for the unit-level clients. + caller *tls.Certificate + // cellRoots is set when serveTLS re-serves this mint under a cell's + // anchor, so roots() answers that rather than httptest's own certificate. + cellRoots *x509.CertPool + // digest and incarnation override the build this host attests, so a test + // can have a RENEWAL seal a different execution than the first credential. + digest string + incarnation uint64 + // executionFree makes this host mint a credential sealing NO execution, + // which core allows: the sealed execution became optional because a + // principal that bears no approved build — a person at a terminal — must + // not have one invented for it. It is the shape of core's own + // execution-missing fixture, and it is not a shape a WORKLOAD credential + // may have. + executionFree bool + // installation overrides the installation it seals credentials to, so a + // test can have a renewal answer about a different one. Empty means + // testInstallation. + installation string + + mints int64 + // mu guards what serve records. The viewer's mint and this workload's + // arrive on the same host, and a test that drives concurrent requests + // reaches it from several goroutines. + mu sync.Mutex + // signErr is the fake host failing to produce a credential, which is a + // defect in the test and not an outcome the runtime should see. + signErr error + // presented records the projected token each mint arrived with, so a test + // can prove a rotated projection is re-read rather than cached, and + // headers records what else each mint carried. + presented []string + headers []http.Header +} + +// prepareAuthority gives this fake mint the issuer its fields describe: core's +// stand-in, or one whose seal source has been moved so it can attest a +// different installation, a different build, or no execution at all. +// +// Separate from newHostMint so ONE mint can hold a second shape and switch to +// it after the first credential — which is what a booted renewal test needs, +// and the only way to exercise the renewal path through the real client. +func (mint *hostMint) prepareAuthority(t *testing.T) { + t.Helper() + mint.authority = standInAuthority() + if mint.executionFree { + // An issuer that mints a credential sealing NO execution, which core + // permits only for a principal bearing no approved build — a person at + // a terminal. Built by taking core's fixture seal source and recording + // the installation seal and epoch WITHOUT an approved build, which is + // the shape of core's own execution-missing fixture. + // + // Core refuses to mint execution-free for a principal that does bear + // one, which is why this cannot be done by simply omitting the field. + seals := corework.NewMemorySealSource() + if err := seals.Put(corework.FixturePrincipal, corework.Seal{ + InstallationID: testInstallation, + InstallationRevision: corework.FixtureInstallationRevision, + }); err != nil { + t.Fatalf("seal an execution-free issuer: %v", err) + } + if err := seals.PutEpoch(corework.FixturePrincipal, corework.FixturePrincipalEpoch); err != nil { + t.Fatalf("record the principal's epoch: %v", err) + } + // Explicitly bearing no execution, which is core's own distinction: + // a principal with an approved build MUST attest it, one recorded as + // bearing none legitimately carries no execution, and one core knows + // nothing about is refused. This is the middle case — a person at a + // terminal — and it is the shape a WORKLOAD credential must never have. + if err := seals.PutBearsNoExecution(corework.FixturePrincipal); err != nil { + t.Fatalf("record the principal as bearing no execution: %v", err) + } + _, key := corework.FixtureKeyPair() + mint.authority = &corework.Authority{ + Issuer: corework.FixtureIssuer, + KeyID: corework.FixtureKeyID, + Key: key, + Revisions: corework.FixtureRevisions(), + Seals: seals, + } + } + if mint.digest != "" || mint.incarnation != 0 { + // An issuer whose APPROVED BUILD has moved, which is the only way it + // can attest a different one: core refuses to seal an execution the + // issuer does not approve for that principal. Built from core's own + // fixture seal source with the approved build overridden. + seals := corework.FixtureSeals() + if err := seals.PutApprovedBuild(corework.FixturePrincipal, + cmp.Or(mint.digest, corework.FixtureImageDigest), + cmp.Or(mint.incarnation, uint64(corework.FixtureBuildIncarnation))); err != nil { + t.Fatalf("move the issuer's approved build: %v", err) + } + _, key := corework.FixtureKeyPair() + mint.authority = &corework.Authority{ + Issuer: corework.FixtureIssuer, + KeyID: corework.FixtureKeyID, + Key: key, + Revisions: corework.FixtureRevisions(), + Seals: seals, + } + } + if mint.installation != "" { + // An issuer that seals to a different installation, for the renewal + // refusal. Built from CORE's fixture seal source with one entry + // overridden rather than from a seal source of this repo's own: the + // rest of what that fixture configures — epochs, approved builds, + // bindings — is core's to define, and duplicating it here is how a + // fixture drifts from the thing it stands in for. + seals := corework.FixtureSeals() + // Two fields, because core v0.9.0's Seal is the installation and its + // revision and nothing else: the principal's epoch and the approved + // build moved off the installation seal, where they described the + // OWNER's workload however many delegation hops had been added. The + // fixture's own epoch and approved-build entries are already in the + // source this starts from. + if err := seals.Put(corework.FixturePrincipal, corework.Seal{ + InstallationID: mint.installation, + InstallationRevision: corework.FixtureInstallationRevision, + }); err != nil { + t.Fatalf("seal the stand-in issuer to %s: %v", mint.installation, err) + } + _, key := corework.FixtureKeyPair() + mint.authority = &corework.Authority{ + Issuer: corework.FixtureIssuer, + KeyID: corework.FixtureKeyID, + Key: key, + Revisions: corework.FixtureRevisions(), + Seals: seals, + } + } + if mint.ttl == 0 { + // Core imposes no maximum lifetime — how long a credential lives is the + // issuing host's decision, and this runtime's renewal arithmetic simply + // follows whatever it is handed. Ten minutes keeps a test's renewal + // behaviour ordinary rather than asserting a cap that does not exist. + mint.ttl = 10 * time.Minute + } +} + +func newHostMint(t *testing.T, mint *hostMint) *hostMint { + t.Helper() + mint.prepareAuthority(t) + if mint.switchTo != nil { + mint.switchTo.prepareAuthority(t) + } + // TLS, not plaintext, and under a cell of its own. The SDK's mint client + // refuses plain HTTP outright — the projected service-account token + // travels on that request — and httptest's shared certificate carries no + // SPIFFE ID, so a client that admits the endpoint by identity cannot + // accept it. A booted test calls serveTLS afterwards to move this host + // under the boot's anchor instead. + mint.serveUnder(t, newCell(t)) + return mint +} + +// serveUnder serves this fake mint under one cell: its own identity from that +// anchor, and the caller's certificate REQUIRED and verified against it. +// +// Required, not "verified if given". The fixture was weakened to +// VerifyClientCertIfGiven when the SDK owned the whole transport and presented +// nothing, and a reviewer was right that this established the changed +// behaviour rather than closing it: a fake host that tolerates an anonymous +// caller cannot tell a runtime that presents its X.509-SVID from one that does +// not. sdk-go#51 gives the client a ClientCertificate reader, so the fixture +// goes back to demanding what a real mint endpoint demands. +func (m *hostMint) serveUnder(t *testing.T, c *cell) { + t.Helper() + m.cell, m.cellRoots = c, c.roots + m.caller = c.identity(t, testPrincipal) + m.Server = httptest.NewUnstartedServer(http.HandlerFunc(m.serve)) + m.TLS = &tls.Config{ + MinVersion: tls.VersionTLS13, + Certificates: []tls.Certificate{*c.identity(t, testGatewayPrincipal)}, + ClientCAs: c.roots, + ClientAuth: tls.RequireAndVerifyClientCert, + } + m.StartTLS() + t.Cleanup(m.Close) +} + +// serveTLS restarts this host under the cell's anchor, presenting an identity +// of its own and requiring the caller's — which is what the runtime's outbound +// client expects, because an https URL alone says only that the scheme is +// https. +// roots is the pool that may sign this fake mint's certificate, which the SDK +// now requires and will not default to the system pool for. +func (m *hostMint) roots() (*x509.CertPool, error) { + if m.cellRoots != nil { + return m.cellRoots, nil + } + pool := x509.NewCertPool() + pool.AddCert(m.Certificate()) + return pool, nil +} + +func (m *hostMint) serveTLS(t *testing.T, c *cell) { + t.Helper() + m.Close() + m.serveUnder(t, c) +} + +// execution is the build this host attests for the credential it mints, or +// nothing at all when the test asks for an execution-free one. +// installationFor is the installation the shape in force seals to. +func (m *hostMint) installationFor() string { + if m.switchTo != nil && atomic.LoadInt64(&m.mints) > m.switchAfter { + return m.switchTo.installation + } + return m.installation +} + +func (m *hostMint) execution() corework.Execution { + if m.executionFree { + return corework.Execution{} + } + return corework.Execution{ + ImageDigest: cmp.Or(m.digest, corework.FixtureImageDigest), + BuildIncarnation: cmp.Or(m.incarnation, uint64(corework.FixtureBuildIncarnation)), + } +} + +func (m *hostMint) serve(w http.ResponseWriter, r *http.Request) { + m.mu.Lock() + m.presented = append(m.presented, r.Header.Get("authorization")) + m.headers = append(m.headers, r.Header.Clone()) + m.callers = append(m.callers, callerIdentity(r)) + m.paths = append(m.paths, r.URL.Path) + m.mu.Unlock() + // The gateway this harness stands in for routes the viewer's mint as well + // as this workload's, on the same base URL. They are counted apart: one per + // process start is the claim about the workload credential, and a test about + // what a handler mints needs to see only the viewer's. + if r.URL.Path == workContextStartTaskProcedure { + var mint mintRequest + body, _ := io.ReadAll(r.Body) + _ = json.Unmarshal(body, &mint) + writeJSON(w, http.StatusOK, map[string]any{ + "token": capability("context-" + mint.Audience + ".1"), "orgId": mint.OrgID, + "expiresAt": time.Now().Add(m.ttl).UTC().Format(time.RFC3339), + }) + return + } + atomic.AddInt64(&m.mints, 1) + // recoverAfter 0 means this host never issues; a positive one means it + // issues once that many asks have been answered with status. + if m.status != 0 && (m.recoverAfter == 0 || atomic.LoadInt64(&m.mints) <= m.recoverAfter) { + w.WriteHeader(m.status) + return + } + issuer, execution := m.authority, m.execution() + if m.switchTo != nil && atomic.LoadInt64(&m.mints) > m.switchAfter { + issuer, execution = m.switchTo.authority, m.switchTo.execution() + } + token, _, err := issuer.Start(r.Context(), corework.StartInput{ + TenantID: corework.FixtureTenant, + OwnerPrincipalID: corework.FixturePrincipal, + OwnerPrincipalKind: "human", + TaskID: fmt.Sprintf("workload-execution-%d", atomic.LoadInt64(&m.mints)), + Audience: testAudience, + OrganizationID: corework.FixtureOrganization, + InstallationID: cmp.Or(m.installationFor(), testInstallation), + TTL: m.ttl, + // Core v0.9.0 requires the caller to attest the build it is running: + // a principal that exercises an approved execution must name the image + // digest and incarnation, and the issuer checks them against the + // approved build it holds for that principal. The execution moved off + // the installation seal, where it described the OWNER's workload + // however many delegation hops had been added. + Execution: execution, + }) + if err != nil { + m.signErr = err + http.Error(w, "stand-in issuer: "+err.Error(), http.StatusInternalServerError) + return + } + writeJSON(w, http.StatusOK, map[string]string{"work_context": token}) +} + +func (m *hostMint) count() int64 { return atomic.LoadInt64(&m.mints) } + +// observedStartTasks is the viewer mints this host was asked for, which is not +// the same question as count(): that one is this workload's own credential. +func (m *hostMint) observedStartTasks() []string { + m.mu.Lock() + defer m.mu.Unlock() + var asked []string + for _, path := range m.paths { + if path == workContextStartTaskProcedure { + asked = append(asked, path) + } + } + return asked +} + +// callerIdentity is the SPIFFE ID the caller's own certificate named, or "" on +// a plain connection. +func callerIdentity(r *http.Request) string { + if r.TLS == nil || len(r.TLS.PeerCertificates) == 0 || len(r.TLS.PeerCertificates[0].URIs) == 0 { + return "" + } + return r.TLS.PeerCertificates[0].URIs[0].String() +} + +// cell is one deployment's trust anchor: the root every identity in a test is +// issued under — this workload's, its callers', and the platform hosts it +// dials. One anchor on purpose, because that is the shape the runtime assumes: +// the pair its listener presents and the anchor it verifies peers with are also +// what it presents and verifies on the way out. +type cell struct { + anchorPEM []byte + anchor *x509.Certificate + anchorKey ed25519.PrivateKey + roots *x509.CertPool +} + +func newCell(t *testing.T) *cell { + t.Helper() + pem, leaf, key := issueAnchor(t) + roots := x509.NewCertPool() + roots.AddCert(leaf) + return &cell{anchorPEM: pem, anchor: leaf, anchorKey: key, roots: roots} +} + +// workload writes the X.509-SVID the platform projects for a workload — a leaf +// whose URI SAN is its SPIFFE ID, which is what names a workload in this model +// and what the listener is held to — plus the anchor it was issued under, which +// is also the anchor this listener verifies its callers against. +// +// It returns a caller certificate too, because the listener requires one: a +// test that could reach this solution without presenting an identity would be +// exercising a listener this runtime refuses to start. +func (c *cell) workload(t *testing.T, spiffeID string) (certFile, keyFile, bundleFile string, caller *tls.Certificate, roots *x509.CertPool) { + t.Helper() + dir := t.TempDir() + bundleFile = filepath.Join(dir, "ca.crt") + writeFile(t, bundleFile, string(c.anchorPEM)) + + certFile, keyFile = filepath.Join(dir, "tls.crt"), filepath.Join(dir, "tls.key") + leafPEM, keyPEM, _ := issueLeaf(t, c.anchor, c.anchorKey, spiffeID) + writeFile(t, certFile, string(leafPEM)) + writeFile(t, keyFile, string(keyPEM)) + + return certFile, keyFile, bundleFile, c.identity(t, testGatewayPrincipal), c.roots +} + +// identity is a TLS certificate for one SPIFFE ID, for a party in a test that +// serves or dials rather than reading projected files. +func (c *cell) identity(t *testing.T, spiffeID string) *tls.Certificate { + t.Helper() + certPEM, keyPEM, _ := issueLeaf(t, c.anchor, c.anchorKey, spiffeID) + pair, err := tls.X509KeyPair(certPEM, keyPEM) + if err != nil { + t.Fatal(err) + } + return &pair +} + +// workloadIdentity is one workload in a cell of its own, for a test that does +// not need the platform's side of the anchor. +func workloadIdentity(t *testing.T, spiffeID string) (certFile, keyFile, bundleFile string, caller *tls.Certificate, roots *x509.CertPool) { + t.Helper() + return newCell(t).workload(t, spiffeID) +} + +// issueAnchor is the trust anchor a cell's identities are issued under. +func issueAnchor(t *testing.T) ([]byte, *x509.Certificate, ed25519.PrivateKey) { + t.Helper() + public, private, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + t.Fatal(err) + } + template := &x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{CommonName: "codefly.test workload CA"}, + NotBefore: time.Now().Add(-time.Minute), + NotAfter: time.Now().Add(time.Hour), + KeyUsage: x509.KeyUsageCertSign | x509.KeyUsageDigitalSignature, + BasicConstraintsValid: true, + IsCA: true, + } + der, err := x509.CreateCertificate(rand.Reader, template, template, public, private) + if err != nil { + t.Fatal(err) + } + leaf, err := x509.ParseCertificate(der) + if err != nil { + t.Fatal(err) + } + return pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}), leaf, private +} + +// issueLeaf issues one workload identity under the anchor: a leaf whose only +// identity is the SPIFFE ID in its URI SAN. +func issueLeaf(t *testing.T, anchor *x509.Certificate, anchorKey ed25519.PrivateKey, spiffeID string) (certPEM, keyPEM []byte, leaf *x509.Certificate) { + t.Helper() + public, private, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + t.Fatal(err) + } + id, err := url.Parse(spiffeID) + if err != nil { + t.Fatal(err) + } + template := &x509.Certificate{ + SerialNumber: big.NewInt(time.Now().UnixNano()), + Subject: pkix.Name{CommonName: spiffeID}, + NotBefore: time.Now().Add(-time.Minute), + NotAfter: time.Now().Add(time.Hour), + KeyUsage: x509.KeyUsageDigitalSignature, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth}, + DNSNames: []string{"localhost"}, + IPAddresses: []net.IP{net.ParseIP("127.0.0.1")}, + URIs: []*url.URL{id}, + } + der, err := x509.CreateCertificate(rand.Reader, template, anchor, public, anchorKey) + if err != nil { + t.Fatal(err) + } + keyDER, err := x509.MarshalPKCS8PrivateKey(private) + if err != nil { + t.Fatal(err) + } + parsed, err := x509.ParseCertificate(der) + if err != nil { + t.Fatal(err) + } + return pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}), + pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: keyDER}), + parsed +} + +// booted is a solution serving on an ephemeral port, with the client that +// trusts its workload identity. +type booted struct { + server *Server + base string + client *http.Client + mint *hostMint + // cell is the CA the listener verifies callers against, for a test that + // must issue another certificate it will accept. + cell *cell + // stop ends this runtime and waits for serve to return, so a test can + // observe what a SHUTDOWN does rather than only what a serving process + // does. It is idempotent, and the cleanup calls it too — a test that stops + // the runtime itself must not leave a second cancel racing the first. + stop func() + // ended waits for serve to return by itself, within a bound, and reports + // its error and whether it returned at all. Nothing here cancels, which is + // the difference between observing a process end and watching cleanup end + // it. + ended func(time.Duration) (error, bool) +} + +// bootEnvironment provisions everything a deployed solution's boot reads — the +// projected identity, the anchor, the projected token, the resolved gateway — +// and starts the fake host under that same anchor, because the runtime's +// outbound client presents its identity and verifies the host against it. +// +// Shared by every boot test, including the ones that must fail: a boot refused +// for the reason under test has to be refused with everything else in place, +// or the test is asserting the first missing value instead. +func bootEnvironment(t *testing.T, mint *hostMint) (caller *tls.Certificate, roots *x509.CertPool) { + t.Helper() + caller, roots, _ = bootCell(t, mint) + return caller, roots +} + +// bootCell is bootEnvironment with the cell exposed, for a test that must issue +// another certificate under the SAME anchor the listener verifies against — a +// forged leaf from any other cell is refused for its chain, which would answer +// the test before the property under test is reached. +func bootCell(t *testing.T, mint *hostMint) (caller *tls.Certificate, roots *x509.CertPool, cell *cell) { + t.Helper() + certFile, keyFile, bundleFile, caller, roots, cell := bootIdentityIn(t, mint, testPrincipal) + _, _, _ = certFile, keyFile, bundleFile + return caller, roots, cell +} + +// bootIdentity is bootEnvironment with the projected material exposed, for a +// test that supplies its own IdentitySource and must do so under the *same* +// anchor the fake host is served with — otherwise the boot fails verifying the +// host rather than on the property under test. +func bootIdentity(t *testing.T, mint *hostMint, principal string) (certFile, keyFile, bundleFile string, caller *tls.Certificate, roots *x509.CertPool) { + certFile, keyFile, bundleFile, caller, roots, _ = bootIdentityIn(t, mint, principal) + return +} + +func bootIdentityIn(t *testing.T, mint *hostMint, principal string) (certFile, keyFile, bundleFile string, caller *tls.Certificate, roots *x509.CertPool, issuing *cell) { + t.Helper() + cell := newCell(t) + issuing = cell + certFile, keyFile, bundleFile, caller, roots = cell.workload(t, principal) + mint.serveTLS(t, cell) + authorityValues(t) + tokenFile := filepath.Join(t.TempDir(), "token") + writeFile(t, tokenFile, "projected-token") + + // Bound here and handed to whichever Server this test boots, so the port is + // never released between being chosen and being served on. + ln, port := boundPort(t) + provideListener(t, ln) + t.Setenv("PORT", port) + t.Setenv("GATEWAY_URL", mint.URL) + // Set explicitly, as a deployment must: nothing derives a mint address + // from the resolved gateway, because the projected token goes there. + t.Setenv(CredentialMintURLEnvironmentVariable, mint.URL+credentialMintPath) + t.Setenv(ProjectedTokenFileEnvironmentVariable, tokenFile) + t.Setenv(IdentityCertFileEnvironmentVariable, certFile) + t.Setenv(IdentityKeyFileEnvironmentVariable, keyFile) + t.Setenv(IdentityTrustBundleFileEnvironmentVariable, bundleFile) + // Who may call: the gateway, which is the identity cell.workload hands back + // as the caller. Holding a certificate from the cell's anchor is not the + // same as being a caller this solution serves, so the admitted set is + // provisioned rather than derived from the anchor. + t.Setenv(IdentityAllowedCallersFileEnvironmentVariable, identitiesFile(t, testGatewayPrincipal)) + // And whom this runtime may present credentials to: the same identity, for + // the mirror-image reason. The fake host serves under it. + t.Setenv(IdentityMintPeersFileEnvironmentVariable, identitiesFile(t, testGatewayPrincipal)) + t.Setenv(IdentityGatewayPeersFileEnvironmentVariable, identitiesFile(t, testGatewayPrincipal)) + t.Setenv(ContractProfileEnvironmentVariable, localProfile) + // A test that serves assets provisions the directory before booting, since + // a boot's configuration is read by the serving goroutine and must not be + // written after it starts. + if os.Getenv("ASSETS_DIR") == "" { + t.Setenv("ASSETS_DIR", t.TempDir()) + } + return certFile, keyFile, bundleFile, caller, roots, issuing +} + +// boot runs the whole boot a deployed solution runs — configuration, the +// contract, the one mint, the TLS listener — on an ephemeral port. +func boot(t *testing.T, server *Server, mint *hostMint) *booted { + t.Helper() + caller, roots, cell := bootCell(t, mint) + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + ln, err := takeListener(server).start(ctx) + if err != nil { + t.Fatalf("boot: %v", err) + } + served := make(chan error, 1) + go func() { served <- server.serve(ctx, ln) }() + var stopped sync.Once + stop := func() { + stopped.Do(func() { + cancel() + <-served + }) + } + t.Cleanup(stop) + // ended waits for serve to return ON ITS OWN and reports what it returned. + // + // It must not cancel: a test asserting that this process ends has to + // observe it ending by itself, and cleanup cancelling afterwards is + // exactly how such a test passes against a runtime that never would. The + // error goes back on the buffered channel so stop() still completes. + ended := func(within time.Duration) (error, bool) { + select { + case err := <-served: + served <- err + return err, true + case <-time.After(within): + return nil, false + } + } + return &booted{ + stop: stop, + ended: ended, + server: server, + base: "https://127.0.0.1:" + os.Getenv("PORT"), + // The caller presents its own identity, because the listener requires + // one: a client that could reach this solution without presenting a + // certificate would be exercising a listener this runtime refuses to + // start. + client: &http.Client{Transport: &http.Transport{TLSClientConfig: &tls.Config{ + RootCAs: roots, + Certificates: []tls.Certificate{*caller}, + MinVersion: tls.VersionTLS13, + }}}, + mint: mint, + cell: cell, + } +} + +// bootFails runs the same boot and returns what it refused with. +func bootFails(t *testing.T, server *Server, mint *hostMint) error { + t.Helper() + return bootFailsWithin(t, server, mint, 30*time.Second) +} + +// bootFailsWithin is bootFails with a bound on how long the boot may take to +// fail. +// +// The bound is the point. Several of these tests assert that a window is a +// deadline, and they measured the elapsed time *after* start() returned — so +// against a runtime where the window is not a deadline, start() never returns, +// the assertion is never reached, and the only thing that ends the test is `go +// test -timeout`. That kills the mutant by hanging, which is indistinguishable +// from an infrastructure problem and takes the whole suite's timeout to report. +func bootFailsWithin(t *testing.T, server *Server, mint *hostMint, within time.Duration) error { + t.Helper() + bootEnvironment(t, mint) + type outcome struct { + ln net.Listener + err error + } + done := make(chan outcome, 1) + go func() { + ln, err := takeListener(server).start(context.Background()) + done <- outcome{ln, err} + }() + select { + case got := <-done: + if got.ln != nil { + _ = got.ln.Close() + } + if got.err == nil { + t.Fatal("the boot came up; this helper is for boots that must fail") + } + return got.err + case <-time.After(within): + // Not t.Fatal: this goroutine is the test's, but the boot's is still + // running and will outlive the test. Reporting and failing is all that + // is safe. + t.Fatalf("the boot neither came up nor failed within %s: a bounded window that does not reach the operation leaves the process with no listener, no log and nothing for an orchestrator to act on, and a test that waits for the harness timeout cannot tell you that", within) + return nil + } +} + +// TestBootMintsExactlyOnceAndServesOverTLS is the acceptance criterion of this +// runtime's whole boot: one mint per process start, zero registrations over any +// interval, and a listener that answers only over TLS. +func TestBootMintsExactlyOnceAndServesOverTLS(t *testing.T) { + mint := newHostMint(t, &hostMint{}) + solution := boot(t, New(Manifest{ID: testSolutionID, Title: testSolutionTitle}). + Handle("/thing", func(context.Context, *Gateway) (any, error) { return map[string]string{"ok": "yes"}, nil }), mint) + + for range 5 { + if status := getStatus(t, solution.client, solution.base+HealthPath); status != http.StatusOK { + t.Fatalf("health = %d, want 200 answered on the destination the presence document names", status) + } + if status := getStatus(t, solution.client, solution.base+"/.well-known/solution.json"); status != http.StatusOK { + t.Fatalf("served manifest = %d, want 200", status) + } + } + if got := mint.count(); got != 1 { + t.Errorf("the host minted %d credentials for one process start, want exactly 1: nothing may mint per request, per beat or per probe", got) + } + if want := "Bearer projected-token"; mint.presented[0] != want { + t.Errorf("the mint was presented %q, want %q — the projected service-account token is what attests which workload this is", mint.presented[0], want) + } + + // The same port answers no plain-HTTP request: the TLS handshake it expects + // never happens, so the request is refused before any route is reached. The + // control is the TLS health check above, which the same port answers 200. + plain, err := http.Get("http://" + solution.base[len("https://"):] + HealthPath) + if err == nil { + defer func() { _ = plain.Body.Close() }() + if plain.StatusCode == http.StatusOK { + t.Error("plain HTTP was served: the listener presents this workload's identity and there is no plain-HTTP listener") + } + } +} + +// TestABootedRuntimeMintsOnceUnderConcurrentRequests is the mint-once claim +// about *production wiring* rather than about one SDK instance. +// +// The earlier version drove the client directly, which would stay green if the +// runtime created a fresh client per request — the defect that would make this +// change worse than the heartbeat it replaced, since every token such a client +// issues is valid and every functional test passes. So this boots the real +// runtime, drives concurrent requests through its handler, and counts what the +// host was asked. +func TestABootedRuntimeMintsOnceUnderConcurrentRequests(t *testing.T) { + // The handler has to *use* the credential, or this test is green against a + // runtime that builds a client per request: a second reviewer pointed out + // that the previous handler returned a literal, so no request reached the + // credential source at all and the only mint being counted was the boot's. + t.Setenv(manifest.APIConsumesEnvironmentVariable, consumesThings) + mint := newHostMint(t, &hostMint{}) + solution := boot(t, New(Manifest{ID: testSolutionID}). + Consumes(passthroughModule()). + Contract(ModuleContract{Ceilings: map[string]map[string][]Scope{ + localProfile: {"things": {{ResourceKind: "things", Actions: []string{"read"}}}}, + }}). + Handle("/thing", func(ctx context.Context, gw *Gateway) (any, error) { + if _, err := gw.ForModule(ctx, "things", Scope{ResourceKind: "things", Actions: []string{"read"}}); err != nil { + return nil, err + } + return map[string]string{"ok": "yes"}, nil + }), mint) + + const callers = 16 + var wg sync.WaitGroup + start := make(chan struct{}) + for range callers { + wg.Add(1) + go func() { + defer wg.Done() + <-start + request, err := http.NewRequest(http.MethodGet, solution.base+"/thing", nil) + if err != nil { + t.Error(err) + return + } + request.Header.Set("authorization", viewerBearer()) + // The identity headers the gateway stamps from the verified bearer; + // ForModule needs both to mint. + request.Header.Set(orgHeader, "org-1") + request.Header.Set(sessionHeader, "session-1") + request.Header.Set(workcontext.InstallationIDHeaderName, testInstallation) + resp, err := solution.client.Do(request) + if err != nil { + t.Errorf("call: %v", err) + return + } + _ = resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Errorf("handler answered %d, want 200", resp.StatusCode) + } + }() + } + close(start) + wg.Wait() + + if got := mint.count(); got != 1 { + t.Errorf("the host minted %d credentials for one process serving %d concurrent requests, want exactly 1: a runtime that minted per request would be worse than the heartbeat it replaced", + got, callers) + } + // And the requests really did reach the credential: without this the + // assertion above holds for a runtime whose handler never touches it. + if got := len(mint.observedStartTasks()); got == 0 { + t.Error("no viewer mint was attempted, so nothing in this test exercised the credential at all") + } + // The mint is dialled WITH this workload's own certificate, and the + // identity it presented is asserted. + // + // This assertion has now been all three things. It asserted the identity; + // then the sdk-go v0.3.0 pin took the transport over and presented + // nothing, so it was changed to require the caller identity be EMPTY and + // the fake endpoint was weakened to VerifyClientCertIfGiven; a reviewer + // said that established the changed behaviour rather than closing the + // finding, and was right. sdk-go#51 gives the client a ClientCertificate + // reader, so both go back: the endpoint REQUIRES a certificate and this + // names which one it must be. + // + // The projected service-account token is still checked, because it is + // still what the host validates the projection by — it is simply no longer + // the whole of what attests who is asking. + if len(mint.presented) == 0 { + t.Fatal("the mint saw no authorization header at all, so nothing attested which workload was asking") + } + if got := mint.presented[0]; !strings.Contains(got, "projected-token") { + t.Errorf("the mint was presented %q, want the projected service-account token", got) + } + if len(mint.callers) == 0 { + t.Fatal("the mint endpoint recorded no caller, so the handshake carried no client certificate: the projected token would be the whole of what attests which workload is asking") + } + if got := mint.callers[0]; got != testPrincipal { + t.Errorf("the mint was dialled with client identity %q, want %q: the host holds this process to the identity its presence document approved, and the hop that carries the projected service-account token is the one that must present it", got, testPrincipal) + } +} + +// TestUnauthenticatedCallIsRefusedNotProxied keeps the boot's serving contract: +// a handler call with no bearer is a 401 from this runtime, never a 502 from +// something downstream. +func TestUnauthenticatedCallIsRefusedNotProxied(t *testing.T) { + mint := newHostMint(t, &hostMint{}) + solution := boot(t, New(Manifest{ID: testSolutionID}). + Handle("/thing", func(context.Context, *Gateway) (any, error) { return nil, nil }), mint) + if status := getStatus(t, solution.client, solution.base+"/thing"); status != http.StatusUnauthorized { + t.Errorf("unauthenticated handler call = %d, want 401", status) + } +} + +// TestServedContractReportsWhatThisProcessHoldsItselfTo: the published contract +// names the principal the platform provisioned, the profile this process runs +// under, and the ceiling each consumed audience is capped at. +func TestServedContractReportsWhatThisProcessHoldsItselfTo(t *testing.T) { + mint := newHostMint(t, &hostMint{}) + t.Setenv(manifest.APIConsumesEnvironmentVariable, consumesThings) + server := New(Manifest{ID: testSolutionID}). + Consumes(passthroughModule()). + Contract(ModuleContract{Ceilings: map[string]map[string][]Scope{ + localProfile: {"things": {{ResourceKind: "things", Actions: []string{"read", "list"}}}}, + }}) + solution := boot(t, server, mint) + + resp, err := solution.client.Get(solution.base + ContractPath) + if err != nil { + t.Fatalf("GET %s: %v", ContractPath, err) + } + defer func() { _ = resp.Body.Close() }() + var published effectiveContract + if err := json.NewDecoder(resp.Body).Decode(&published); err != nil { + t.Fatalf("decode contract: %v", err) + } + if published.Principal != testPrincipal { + t.Errorf("published principal = %q, want the provisioned %q", published.Principal, testPrincipal) + } + if published.Profile != localProfile { + t.Errorf("published profile = %q, want %q", published.Profile, localProfile) + } + if len(published.Bindings) != 1 || published.Bindings[0].Audience != "things" { + t.Fatalf("published bindings = %+v, want the one audience this solution consumes", published.Bindings) + } + if got := published.Bindings[0].Ceiling; len(got) != 1 || len(got[0].Actions) != 2 { + t.Errorf("published ceiling = %+v, want the declared read+list ceiling", got) + } + if got := published.Bindings[0].Asked; len(got) != 1 || got[0].Actions[0] != "read" { + t.Errorf("published ask = %+v, want the single read the declaration asks for", got) + } +} + +// boundPort binds a listener and returns it with its port, so a test hands the +// *listener* to the boot rather than a port number. +// +// This replaces a helper that bound :0, read the port, closed the socket and +// returned the number — a time-of-check/time-of-use race against everything +// else in this suite asking the OS for an ephemeral port, and the cause of the +// intermittent `bind: address already in use` failures that appeared in a +// different test each run. A retry narrowed the window and could not close it: +// the port is released by construction. Nothing is released here. +func boundPort(t *testing.T) (net.Listener, string) { + t.Helper() + ln, err := net.Listen("tcp", ":0") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = ln.Close() }) + _, port, err := net.SplitHostPort(ln.Addr().String()) + if err != nil { + t.Fatal(err) + } + return ln, port +} + +// getStatus GETs target through client and returns the status. +func getStatus(t *testing.T, client *http.Client, target string) int { + t.Helper() + resp, err := client.Get(target) + if err != nil { + t.Fatalf("GET %s: %v", target, err) + } + defer func() { _ = resp.Body.Close() }() + return resp.StatusCode +} + +// consumesThings is the api.consumes projection core surfaces for the "things" +// module the passthrough fixtures declare. The key is a wire contract +// (manifest.APIConsumesEnvironmentVariable). +const consumesThings = `[{"id":"thingstore.things","module":"thingstore","service":"things","endpoint":"rest","protocol":"rest","as":"things"}]` + +// TestTheBootedListenerDropsACallerItNoLongerAdmits drives the production +// serve() path, which is the only place the per-connection watch is installed. +// +// A unit test that builds its own http.Server proves the hook works and not +// that anything installs it: removing `ConnState: watch` from serve() left +// every such test passing. This boots the runtime the way Serve does. +func TestTheBootedListenerDropsACallerItNoLongerAdmits(t *testing.T) { + mint := newHostMint(t, &hostMint{}) + solution := boot(t, New(Manifest{ID: testSolutionID}). + Handle("/thing", func(context.Context, *Gateway) (any, error) { return map[string]string{"ok": "yes"}, nil }), mint) + + // AUTHORIZED, STATUS-CHECKED, DRAINED, AND ON ONE CONNECTION. + // + // This sent no bearer, ignored the status, closed the body without + // draining it and never looked at reuse — so "an admitted caller is + // served" was really "the request did not error", which a 401 satisfies, + // and an undrained body means the connection goes back to nobody rather + // than to the pool. The claim this control makes is that a handler ran + // over an existing BUSY connection, and none of those three omissions let + // it make that claim. + var reused atomic.Int64 + call := func() (int, error) { + request, err := http.NewRequest(http.MethodGet, solution.base+"/thing", nil) + if err != nil { + return 0, err + } + request.Header.Set("authorization", viewerBearer()) + request = request.WithContext(httptrace.WithClientTrace(request.Context(), &httptrace.ClientTrace{ + GotConn: func(info httptrace.GotConnInfo) { + if info.Reused { + reused.Add(1) + } + }, + })) + resp, err := solution.client.Do(request) + if err != nil { + return 0, err + } + status := resp.StatusCode + drainAndClose(resp) + return status, nil + } + + // The control: an admitted caller is SERVED — 200, body drained — and + // stays served on one busy connection across more than a recheck interval. + if status, err := call(); err != nil { + t.Fatalf("an admitted caller was refused: %v", err) + } else if status != http.StatusOK { + t.Fatalf("an admitted caller got %d, want 200: a refusal satisfies 'did not error', so without this the control proves nothing about the handler running", status) + } + quiet := time.Now().Add(2 * inboundTrustRecheckInterval) + for time.Now().Before(quiet) { + status, err := call() + if err != nil { + t.Fatalf("a request failed while this caller was still admitted, so nothing below is about admission: %v", err) + } + if status != http.StatusOK { + t.Fatalf("a request answered %d while this caller was still admitted", status) + } + time.Sleep(inboundTrustRecheckInterval / 10) + } + // ON ONE CONNECTION. The claim is that a handler ran over an existing BUSY + // connection — not that a fresh connection was made each time, which no + // per-connection recheck would need to notice. + if reused.Load() == 0 { + t.Fatal("not one request reused a connection, so this test never exercised an ESTABLISHED connection: a per-dial check alone would satisfy everything below") + } + + // The platform removes this caller from the provisioned file. Nothing + // restarts, and traffic keeps arriving on the connection it already holds. + callersFile := os.Getenv(IdentityAllowedCallersFileEnvironmentVariable) + if callersFile == "" { + t.Fatal("the boot resolved no allowed-callers path, so this test cannot withdraw admission") + } + writeFile(t, callersFile, "spiffe://codefly.test/ns/platform/sa/somebody-else\n") + + deadline := time.Now().Add(5 * inboundTrustRecheckInterval) + for { + if _, err := call(); err != nil { + return // the connection stopped being served, which is the point + } + if time.Now().After(deadline) { + t.Fatalf("this caller was still served %s after it was removed from the provisioned set, on the connection it already held: the booted listener installs no per-connection recheck, so revocation waits for a restart", + 5*inboundTrustRecheckInterval) + } + time.Sleep(inboundTrustRecheckInterval / 10) + } +} + +// TestABootWithAnUnusableAdmissionFileIsRefused: the path resolving is not the +// same question as the set being readable, and the second one has to be asked +// at boot rather than at the first handshake. +// +// A provisioned path naming a file that is empty, or absent, would otherwise +// start a listener that refuses every caller and a client that refuses every +// dial, and report it as an admission failure once traffic arrived instead of +// as the provisioning gap it is. +func TestABootWithAnUnusableAdmissionFileIsRefused(t *testing.T) { + for _, tc := range []struct { + name string + contents string + write bool + }{ + {"a file naming nobody", "\n", true}, + {"a file of comments only", "# the gateway goes here\n", true}, + {"a path with no file behind it", "", false}, + } { + t.Run(tc.name, func(t *testing.T) { + cell := newCell(t) + certFile, keyFile, bundleFile, _, _ := cell.workload(t, testPrincipal) + path := filepath.Join(t.TempDir(), "callers") + if tc.write { + writeFile(t, path, tc.contents) + } + server := New(Manifest{ID: testSolutionID}) + server.cfg = config{ + port: "8080", gatewayURL: "https://gateway:42152", + mintURL: "https://gateway:42152" + credentialMintPath, + identityCertFile: certFile, + identityKeyFile: keyFile, + projectedTokenPath: "t", + trustBundleFile: bundleFile, + allowedCallersFile: path, + mintPeersFile: identitiesFile(t, testGatewayPrincipal), + gatewayPeersFile: identitiesFile(t, testGatewayPrincipal), + profile: localProfile, + } + err := server.validateSources() + if err == nil { + t.Fatal("the boot accepted an admission set it cannot read: the listener would start and refuse every caller, reported as an admission failure rather than as provisioning") + } + if !strings.Contains(err.Error(), "not usable at boot") { + t.Errorf("the refusal %q does not say the provisioned set is unusable, so it reads as a different problem", err) + } + }) + } +} + +// TestABootRefusesAPlaintextMintBeforeSendingTheToken: validate() refuses a +// plaintext credential mint, and start() has to *call* validate() for that to +// mean anything. Removing the call left every boot test passing, because none +// of them boots with a configuration validate() would reject — they all +// provision a conforming one and then assert on behaviour. +// +// The consequence is the whole reason this runtime refuses a plaintext +// destination at boot: the request to the mint carries the projected +// service-account token that attests which workload this process is, and over +// http that is readable by anything on the path. +func TestABootRefusesAPlaintextMintBeforeSendingTheToken(t *testing.T) { + mint := newHostMint(t, &hostMint{}) + bootEnvironment(t, mint) + // Everything else resolved, and the mint moved to plaintext. + plaintext := strings.Replace(mint.URL, "https://", "http://", 1) + t.Setenv(CredentialMintURLEnvironmentVariable, plaintext+credentialMintPath) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + server := New(Manifest{ID: testSolutionID}) + ln, err := takeListener(server).start(ctx) + if ln != nil { + _ = ln.Close() + } + if err == nil { + t.Fatal("the boot accepted a plaintext credential mint: the projected service-account token attesting which workload this is would go out in the clear") + } + if !strings.Contains(err.Error(), "https") { + t.Errorf("the refusal %q does not say the destination has to be https", err) + } + // From THIS runtime's validate(), not from the SDK's own refusal. + // + // sdk-go v0.3.0 refuses a plaintext mint URL too, so this test passed with + // validate()'s check deleted — the mutation ledger caught it. Two + // refusals for one fact is fine; a test that cannot tell which one fired + // is not, because the boot's own check is what refuses BEFORE a mint + // client is built and names the variable an operator has to fix. + if !strings.Contains(err.Error(), CredentialMintURLEnvironmentVariable) { + t.Errorf("the refusal %q does not name %s, so it is not this runtime's boot check: the SDK refuses a plaintext mint as well, and a test that accepts either cannot tell whether validate() still does", + err, CredentialMintURLEnvironmentVariable) + } + if !strings.Contains(err.Error(), "credential mint URL") { + t.Errorf("the refusal %q is not validate()'s, which names the configuration it is refusing", err) + } + // And it did not reach the mint on the way to refusing. + if got := mint.count(); got != 0 { + t.Errorf("the boot minted %d times before refusing the configuration: the refusal has to come before the token is sent anywhere", got) + } +} + +// TestServeEndsOnATerminalCredentialRefusal: a refused build is not something +// to keep serving under. serve() watches credentialRefusedC and closes the +// listener; removing that watch left the process serving indefinitely while +// /health answered 503, which is the "answers 503 forever while reporting +// itself healthy" shape inverted — and no committed test observed serve +// returning. +func TestServeEndsOnATerminalCredentialRefusal(t *testing.T) { + mint := newHostMint(t, &hostMint{}) + bootEnvironment(t, mint) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + server := New(Manifest{ID: testSolutionID}) + ln, err := takeListener(server).start(ctx) + if err != nil { + t.Fatalf("boot: %v", err) + } + served := make(chan error, 1) + go func() { served <- server.serve(ctx, ln) }() + + // The control: it is still serving, so the assertion below is about the + // refusal and not about a listener that never came up. + select { + case err := <-served: + t.Fatalf("serve returned before anything refused this execution's credential: %v", err) + case <-time.After(250 * time.Millisecond): + } + + refusal := fmt.Errorf("%w: this build is not the one the presence document approved", workcontext.ErrMintRefused) + server.credentialRefused(refusal) + + select { + case err := <-served: + if err == nil { + t.Fatal("serve ended without reporting why: the orchestrator restarts against a judgement, so the reason is what it has to be given") + } + if !strings.Contains(err.Error(), "presence document approved") { + t.Errorf("serve ended with %q, which does not carry the issuer's judgement", err) + } + case <-time.After(5 * time.Second): + t.Fatal("serve did not end after this execution's credential was refused for good: the process keeps serving a binding it has no authority for, and the host keeps routing to it") + } +} + +// TestTheMintIsConfiguredWithTheFrozenAuthority: the SDK rechecks the authority +// before every renewal, which is the one moment a drifted authority value would +// be laundered into a credential nobody approved. Dropping it leaves the boot +// and the first mint perfect and only renewals wrong, which no test over a +// happy path can see — the mutation survived every one of them. +func TestTheMintIsConfiguredWithTheFrozenAuthority(t *testing.T) { + mint := newHostMint(t, &hostMint{}) + bootEnvironment(t, mint) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + server := New(Manifest{ID: testSolutionID}) + ln, err := takeListener(server).start(ctx) + if err != nil { + t.Fatalf("boot: %v", err) + } + _ = ln.Close() + + anchor := x509.NewCertPool() + leafAsked := false + options := server.mintOptions("projection", + func() (*x509.CertPool, error) { return anchor, nil }, + func(*tls.CertificateRequestInfo) (*tls.Certificate, error) { + leafAsked = true + return &tls.Certificate{}, nil + }) + if options.Authority == nil { + t.Error("the mint client is configured with no authority to recheck, so a renewal proceeds after the principal, the mint audience or the projection audience has drifted — the one moment that drift would otherwise be caught") + } + if options.Authority != server.authority { + t.Error("the mint client rechecks something other than the reader this boot froze: a second reader is a second answer, and the one that disagrees is the one nobody approved") + } + // The audience is NAMED, not resolved here and passed beside the pin. The + // SDK made that a typed reference because a string passed alongside meant + // the drift check guarded a value the mint did not use. + if want := (workcontext.AuthorityValue{Name: AuthorityGroup, Key: AuthorityAudienceKey}); options.Audience != want { + t.Errorf("the mint reads its audience from %v, want the pinned %v", options.Audience, want) + } + // The three readers sdk-go#51 takes in place of a pool by value, each + // asked DURING the handshake. Checking that they are wired AND that they + // answer from this boot's own material, because a reader that answers from + // somewhere else is the defect a pool-by-value could not have. + if options.TrustAnchor == nil { + t.Fatal("the mint client is configured with no trust-anchor reader, so the anchor is whatever it was when the client was built and a removed root never stops this process dialling the mint") + } + if pool, err := options.TrustAnchor(); err != nil || pool != anchor { + t.Errorf("the mint's trust-anchor reader answered (%v, %v), want this boot's own pool: the endpoint receiving the projected service-account token is verified against something else", pool, err) + } + if options.ClientCertificate == nil { + t.Fatal("the mint client presents no certificate, so the projected service-account token is the whole of what attests which workload is asking") + } + if _, err := options.ClientCertificate(); err != nil { + t.Errorf("the mint's certificate reader refused: %v", err) + } + if !leafAsked { + t.Error("the mint's certificate reader does not come from this runtime's one long-lived reloader, so a rotated leaf is not what gets presented") + } + if options.AdmittedPeers == nil { + t.Fatal("the mint client admits any peer the anchor signed, and every workload in the trust domain holds one of those: authentication is not authorisation") + } + peers, err := options.AdmittedPeers() + if err != nil { + t.Fatalf("the mint's admitted-peer reader refused: %v", err) + } + if len(peers) == 0 || peers[0] != testGatewayPrincipal { + t.Errorf("the mint admits %v, want the provisioned %q from MINT_PEERS_FILE", peers, testGatewayPrincipal) + } + if options.URL != server.cfg.mintURL { + t.Errorf("the mint is configured for %q, not the resolved %q", options.URL, server.cfg.mintURL) + } +} + +// TestACredentialThatCannotBePresentedRefusesTheMint: attestWorkload checks +// that the credential actually attached to the outgoing request. Ignoring that +// error sends the mint with no attestation on it, which the host answers by +// minting for the viewer with nothing saying which module asked — the confused +// deputy this runtime exists to prevent, reached by skipping an error check. +func TestACredentialThatCannotBePresentedRefusesTheMint(t *testing.T) { + gw := newModuleGateway(t, http.StatusOK, `{"entry_id":"e1"}`) + server := New(Manifest{ID: testSolutionID}). + Consumes(passthroughModule()). + Contract(ModuleContract{Ceilings: map[string]map[string][]Scope{ + localProfile: {"things": {{ResourceKind: "things", Actions: []string{"read"}}}}, + }}). + // A credential that holds no token. + // + // WHAT THIS COVERS, NARROWED. The comment said this exercises the + // Attach error branch, and it does not: usableCredential refuses a + // zero credential before anything is attached, so removing the Attach + // error check leaves this test green. A reviewer named that twice and + // was right both times. + // + // What it does cover is the property the test is named for — a + // credential this runtime cannot present produces no mint, reported as + // this solution failing to attest itself — reached through the + // usability check rather than through Attach. The Attach branch itself + // is unreachable behind that check and is documented as defence in + // depth in credential.go rather than as covered: reaching it would + // need a credential that passes usability and then fails to attach, + // which means inventing a capability shape the SDK does not produce. + Credential(stubCredentialSource{credential: workcontext.Credential{}}) + server.cfg.profile = localProfile + server.cfg.gatewayURL = gw.URL + contract, err := server.resolveContract() + if err != nil { + t.Fatalf("resolveContract: %v", err) + } + server.contract = contract + server.contractResolved = true + + header := http.Header{} + header.Set("authorization", viewerBearer()) + header.Set(orgHeader, "org-1") + header.Set(sessionHeader, "session-1") + header.Set(workcontext.InstallationIDHeaderName, testInstallation) + _, err = server.gatewayFor(header).ForModule(context.Background(), "things", + Scope{ResourceKind: "things", Actions: []string{"read"}}) + if err == nil { + t.Fatal("a mint went out with a credential that could not be presented: the host would mint for the viewer with nothing attesting which module asked") + } + if !errors.Is(err, ErrNotAttested) { + t.Errorf("the refusal %v is not reported as this solution failing to attest itself, which is what decides the status a handler answers", err) + } + if got := len(gw.observedMints()); got != 0 { + t.Errorf("observed %d mints for a credential that could not be attached, want 0", got) + } +} + +// provisionedWorkloadValue makes the SDK answer for one workload-identity key, +// the way authorityValues does for the authority group. +// +// It exists because a comment in this suite claimed a test could not make the +// SDK's workspace configuration answer. It can, and because it was believed it +// could not, the two-sources rule had no behavioural test and three mutants of +// it survived. +func provisionedWorkloadValue(t *testing.T, key, value string) { + t.Helper() + t.Setenv("CODEFLY__WORKSPACE_CONFIGURATION__WORKLOAD_IDENTITY__"+key, value) + if err := codefly.LoadEnvironmentVariables(); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = codefly.LoadEnvironmentVariables() }) +} + +// withoutWorkloadValues leaves the SDK with no workspace configuration at all, +// so a lookup errors rather than answering empty. +func withoutWorkloadValues(t *testing.T) { + t.Helper() + t.Setenv("CODEFLY__WORKSPACE_CONFIGURATION", "") + if err := codefly.LoadEnvironmentVariables(); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = codefly.LoadEnvironmentVariables() }) +} + +// provideListener parks a bound listener for the Server this test is about to +// boot, and takeListener collects it. +// +// A package-level handoff because the environment is how a boot learns its +// port and the Server does not exist yet when bootIdentity runs. Safe without +// locking: every test here calls t.Setenv, which forbids t.Parallel, so they +// run one at a time. +var parkedListener net.Listener + +func provideListener(t *testing.T, ln net.Listener) { + t.Helper() + parkedListener = ln + t.Cleanup(func() { parkedListener = nil }) +} + +// takeListener gives the server the listener bootIdentity bound, if there is +// one. A test that provisioned its environment some other way gets nil and the +// boot binds for itself. +func takeListener(server *Server) *Server { + if parkedListener != nil { + server.boundListener = parkedListener + parkedListener = nil + } + return server +} + +// listenOn binds a listener for this server and returns its port, for a test +// that drives listen() directly rather than booting. +func listenOn(t *testing.T, server *Server) string { + t.Helper() + ln, port := boundPort(t) + server.boundListener = ln + return port +} + +// TestAHandlerCannotRewriteTheIdentityThisConnectionWasAdmittedAs is R8-2, +// reproduced on a booted listener over mTLS. +// +// The pin used to be recorded on the recheck's first tick, a second after the +// connection was accepted. I wrote that the window was sound because "no +// in-process hook can run inside it". That was false: a RequestHandler is an +// in-process hook, it is handed r.TLS, and a tls.ConnectionState shares its +// certificates with the connection. A handler rewriting +// PeerCertificates[0].Raw inside that first second had the pin record the +// rewritten identity, and a caller removed from the admitted set went on being +// served on that connection. +// +// The identity is recorded at the first http.StateActive now, which net/http +// calls after reading a request and before ServeHTTP — ahead of any author +// code, on the first request and every one after. +func TestAHandlerCannotRewriteTheIdentityThisConnectionWasAdmittedAs(t *testing.T) { + const survivor = "spiffe://codefly.test/ns/platform/sa/somebody-else" + mint := newHostMint(t, &hostMint{}) + var ( + swapped atomic.Bool + forged *x509.Certificate + ) + + server := New(Manifest{ID: testSolutionID}). + HandleRequest("/thing", func(r *http.Request, _ *Gateway) (any, error) { + // What a hostile or merely careless handler can reach: the parsed + // chain behind r.TLS, which is the connection's own. + if r.TLS != nil && len(r.TLS.PeerCertificates) > 0 { + // Forged to the identity that will REMAIN admitted after the + // withdrawal below, and issued under the anchor this listener + // verifies against. + // + // Rewriting to an identity nobody admits proves nothing: the + // recheck closes such a connection whichever object it read, + // so a test doing that passes without the fix. The attack is + // to name the identity that survives the change. + r.TLS.PeerCertificates[0].Raw = forged.Raw + r.TLS.PeerCertificates[0].URIs = forged.URIs + swapped.Store(true) + } + return map[string]string{"ok": "yes"}, nil + }) + solution := boot(t, server, mint) + forged = parsedLeaf(t, solution.cell.identity(t, survivor)) + provisioned := os.Getenv(IdentityAllowedCallersFileEnvironmentVariable) + if provisioned == "" { + t.Fatal("the boot resolved no allowed-callers path, so this test cannot withdraw admission") + } + + call := func() error { + request, err := http.NewRequest(http.MethodGet, solution.base+"/thing", nil) + if err != nil { + return err + } + // The viewer's bearer, because the route gate answers 401 without one + // and the handler never runs — which is how the first version of this + // test "passed" while exercising nothing. + request.Header.Set("authorization", viewerBearer()) + resp, err := solution.client.Do(request) + if err != nil { + return err + } + drainAndClose(resp) + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("the solution answered %d", resp.StatusCode) + } + return nil + } + + // One request, which runs the handler and lets it rewrite the chain. The + // pin was taken before that handler ran. + if err := call(); err != nil { + t.Fatalf("an admitted caller was refused: %v", err) + } + if !swapped.Load() { + t.Fatal("the handler never saw a peer chain, so this test exercised nothing") + } + + // The caller is withdrawn. Whatever the handler wrote, the connection is + // held to the identity it was admitted as, so the recheck must close it. + writeFile(t, provisioned, survivor+"\n") + + deadline := time.Now().Add(6 * inboundTrustRecheckInterval) + for time.Now().Before(deadline) { + if err := call(); err != nil { + return // closed, which is the point + } + time.Sleep(inboundTrustRecheckInterval / 10) + } + t.Fatalf("a de-admitted caller was still served %s after its identity was withdrawn: a handler rewrote the chain the recheck reads, and the identity the connection was admitted as has to be recorded before any handler runs", + 6*inboundTrustRecheckInterval) +} + +// TestTheListenerDoesNotNegotiateHTTP2 pins the assumption the recheck's pin +// rests on. +// +// net/http calls the ConnState hook with StateActive before ServeHTTP for +// HTTP/1.x, and passes skipHooks for HTTP/2 — so under h2 the pin would never +// be taken at a request and would fall back to the recheck's first tick, which +// is the window R8-2 exploited. Nothing negotiates h2 here: the served +// configuration advertises no ALPN protocol and serve() uses Serve rather than +// ServeTLS, which is what would configure HTTP/2. +// +// If that changes, this fails rather than the pin quietly stopping happening. +func TestTheListenerDoesNotNegotiateHTTP2(t *testing.T) { + c := newCell(t) + certFile, keyFile, bundleFile, _, _ := c.workload(t, testPrincipal) + server := New(Manifest{ID: testSolutionID}) + server.cfg = config{identityCertFile: certFile, identityKeyFile: keyFile, trustBundleFile: bundleFile, + allowedCallersFile: identitiesFile(t, testGatewayPrincipal)} + server.principal = testPrincipal + config, err := server.serverIdentity() + if err != nil { + t.Fatalf("serverIdentity: %v", err) + } + for _, proto := range config.NextProtos { + if proto == "h2" { + t.Fatal("the listener advertises h2: net/http skips the StateActive ConnState hook for HTTP/2, so the identity a connection was admitted as would no longer be recorded before a handler runs — move the pin before enabling it") + } + } +} + +// TestABootedRenewalToAnotherExecutionEndsTheProcess is B3: the renewal +// evidence for R9-1, driven through a real boot and the real mint client +// rather than by calling the wrapper directly. +// +// The unit tests prove authorityHeldSource refuses a renewal sealed to no +// execution or to a different one. They do not prove the booted runtime +// classifies that refusal as terminal, stops serving, and ends — which is the +// behaviour an orchestrator depends on and the thing a wrapper test cannot +// see. The executed round validated these in scratch; they belong here. +// +// The issuer answers correctly for the first credential and switches shape +// afterwards. A credential whose whole remaining lifetime sits inside the +// renewal lead cannot be installed at all (the SDK floors that lead at five +// seconds), so the TTL is just above it: the first install succeeds and the +// next ask renews. +func TestABootedRenewalToAnotherExecutionEndsTheProcess(t *testing.T) { + for _, tc := range []struct { + name string + after *hostMint + }{ + {"a renewal sealing no execution", &hostMint{executionFree: true}}, + {"a renewal sealing a different build", &hostMint{incarnation: 99}}, + } { + t.Run(tc.name, func(t *testing.T) { + // The TTL sets the margin between the boot and the renewal, and + // six seconds was not a margin at all. + // + // The SDK floors the renewal lead at five seconds + // (minRenewalLeadTime), so a six-second credential is due for + // renewal ONE second after it is issued. The control call below + // has to land before that, and on a loaded CI runner the boot — + // cert generation, the TLS listener, the first mint — does not + // finish within a second, so the control call itself triggered the + // renewal, the switched issuer answered it, the process went + // terminal mid-request and the control saw a closed connection. + // It failed in CI with "the route answered 0 before any renewal" + // and passes six times out of six locally, which is what a margin + // that depends on machine speed looks like. + // + // Twelve seconds puts the renewal at ~7s and leaves the control + // call seven seconds of room. The wall-clock cost is the price of + // a test that does not depend on the boot being fast. + mint := newHostMint(t, &hostMint{ + ttl: 12 * time.Second, + switchAfter: 1, + switchTo: tc.after, + }) + var ran atomic.Bool + solution := boot(t, New(Manifest{ID: testSolutionID}). + HandleRequest("/thing", func(*http.Request, *Gateway) (any, error) { + ran.Store(true) + return map[string]string{"ok": "yes"}, nil + }), mint) + + // call reports the status, or 0 with the reason the request did + // not complete. The reason is carried rather than swallowed: this + // test read a bare 0 as "the listener is gone", and a 0 is also + // what a refused handshake, a closed connection mid-request and a + // not-yet-bound port look like — so the one failure this test hit + // in CI reported only "answered 0" and said nothing about why. + call := func() (int, error) { + request, err := http.NewRequest(http.MethodGet, solution.base+"/thing", nil) + if err != nil { + t.Fatal(err) + } + request.Header.Set("authorization", viewerBearer()) + resp, err := solution.client.Do(request) + if err != nil { + return 0, err + } + defer drainAndClose(resp) + return resp.StatusCode, nil + } + + // The control: the first credential is sound and the route serves. + if got, err := call(); got != http.StatusOK { + t.Fatalf("the route answered %d (%v) before any renewal, so nothing below is about the renewal", got, err) + } + if !ran.Load() { + t.Fatal("the handler never ran on the first call") + } + ran.Store(false) + + // Past the renewal lead, every ask renews — and the issuer has + // changed shape. + // + // THE ROUTE REFUSING IS NOT THE ASSERTION. This test returned on + // the first 503 or the first transport error, and a reviewer was + // right that a runtime refusing the request and then serving 503 + // forever satisfied it — as would an unrelated transient renewal + // failure, and a transport error was being read as proof the + // listener had disappeared when it proves only that one request + // did not complete. Worse, cleanup stops the server itself, so + // "the process ended" was never observed at all. + // + // So the refusal is a waypoint, and what is asserted is: the + // handler did not run on the refused call, serve returned BY + // ITSELF, and it returned the terminal reason. + refused := false + deadline := time.Now().Add(20 * time.Second) + for time.Now().Before(deadline) { + // Reset per call, not once: a call made while the FIRST + // credential is still valid runs the handler legitimately, so + // the question is whether the handler ran on the call that was + // refused — not whether it ran at all during the wait. + ran.Store(false) + status, _ := call() + if status == http.StatusServiceUnavailable || status == 0 { + if ran.Load() { + t.Error("the handler ran on the refused call: the refusal has to come before anything author-written") + } + refused = true + break + } + time.Sleep(200 * time.Millisecond) + } + if !refused { + t.Fatalf("the route was still answering 200 twenty seconds after the issuer began sealing a different execution: a renewal that replaces this execution is terminal, so the process stops serving and ends for the orchestrator") + } + + // And the process ends, by itself, with the reason. + // + // This is what an orchestrator depends on: a pod that refuses + // every request but stays up is reported healthy by its + // supervisor and keeps a routable endpoint, which is the exact + // failure the health/terminal coupling exists to prevent. + err, done := solution.ended(20 * time.Second) + if !done { + t.Fatal("serve was still running twenty seconds after the renewal was refused: the process refuses every request and never exits, so the orchestrator never restarts it and the host keeps routing to a binding this process has no authority for") + } + if err == nil { + t.Fatal("serve returned without reporting why: the orchestrator restarts against a judgement, so the reason is what it has to be given") + } + // The terminal reason, not any error: an unrelated transient + // failure ending the process would satisfy a nil check and is the + // crash loop ErrMintUnavailable must never cause. + if !errors.Is(err, workcontext.ErrMintRefused) { + t.Errorf("serve ended with %v, which does not wrap ErrMintRefused: a transient answer must not end this process, and only a terminal one may", err) + } + if !strings.Contains(err.Error(), "execution") { + t.Errorf("serve ended with %q, which does not say the execution was replaced", err) + } + }) + } +} + +// TestTheMintsAnchorIsPassedAsAReaderNotASnapshot is the narrow thing this +// runtime owns about the mint's anchor. +// +// Re-reading it per handshake is the SDK's behaviour and cannot be mutated from +// here. What IS this runtime's is passing the READER through rather than +// resolving it once and handing over the answer — and a mutant that captures +// the pool at construction is invisible to every test that only asks the +// reader what it says now, including the configuration test above. +// +// So the reader's answer is changed between calls, which is what a rotation +// looks like from the SDK's side. +func TestTheMintsAnchorIsPassedAsAReaderNotASnapshot(t *testing.T) { + mint := newHostMint(t, &hostMint{}) + bootEnvironment(t, mint) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + server := New(Manifest{ID: testSolutionID}) + ln, err := takeListener(server).start(ctx) + if err != nil { + t.Fatalf("boot: %v", err) + } + _ = ln.Close() + + first, second := x509.NewCertPool(), x509.NewCertPool() + second.AddCert(mint.Certificate()) + rotated := false + options := server.mintOptions("projection", + func() (*x509.CertPool, error) { + if rotated { + return second, nil + } + return first, nil + }, + func(*tls.CertificateRequestInfo) (*tls.Certificate, error) { return &tls.Certificate{}, nil }) + + if got, _ := options.TrustAnchor(); got != first { + t.Fatal("the mint's trust-anchor reader does not answer from the reader this runtime passed") + } + rotated = true + got, err := options.TrustAnchor() + if err != nil { + t.Fatalf("the mint's trust-anchor reader refused after the rotation: %v", err) + } + if got == first { + t.Error("the mint's trust-anchor reader still answers the pool it first returned: the anchor was resolved once and the ANSWER handed over, so the SDK re-reading per handshake re-reads a snapshot and a removed root never stops this process dialling the endpoint that receives the projected service-account token") + } + if got != second { + t.Errorf("the mint's trust-anchor reader answered %v, want the rotated pool", got) + } +} diff --git a/conformance_test.go b/conformance_test.go new file mode 100644 index 0000000..cbd3220 --- /dev/null +++ b/conformance_test.go @@ -0,0 +1,297 @@ +package solution + +import ( + "context" + "errors" + "net/http" + "net/http/httptest" + "net/url" + "testing" + "time" + + corework "github.com/codefly-dev/core/workcontext" + "github.com/codefly-dev/sdk-go/workcontext" +) + +// Core's published conformance fixtures, driven through this package's +// *carriage* boundary. +// +// The earlier claim was that there is no conformance kit for this package to +// run, on the grounds that the kit is for verifiers and this runtime verifies +// nothing. That was half right and it was the convenient half. Minting with +// Core's fixture identities — which is what the rest of the suite does — shows +// successful carriage and nothing else: it never presents the kit's negative +// fixtures, so the refusals this package *is* responsible for were never +// exercised against the bytes Core publishes for exactly that purpose. +// +// What this runtime is responsible for is the boundary it owns: a capability it +// is handed gets carried, and one it cannot carry is refused before a call is +// made on it. So each fixture is offered to the carrier, and the assertion is +// the division of labour: +// +// - a capability with no readable seal, and a token in another encoding, are +// refused HERE, with Core's own sentinels. Those are properties of the +// bytes in hand, and the moment to find them out is before a request is +// sent rather than at the far end, where the refusal would name an +// installation mismatch for a capability that never named an installation; +// - a tampered payload, an unknown key, a wrong audience and a sealed value +// the issuer has moved past are NOT refused here, and must not be. They +// are judgements against live state and a signature, which only a verifier +// holding the issuer's four sources can make. A carrier that refused them +// would be claiming a strength it does not have — the silent-downgrade +// failure this whole single-implementation effort exists to end. +// +// Running the kit this way is what makes that division checkable rather than +// asserted in a comment: if Core adds a fixture whose refusal belongs to the +// carrier, this test starts failing and the carrier has to grow. +func TestCoreConformanceFixturesThroughTheCarrier(t *testing.T) { + fixtures, err := corework.Fixtures(time.Now()) + if err != nil { + t.Fatalf("core conformance fixtures: %v", err) + } + if len(fixtures) < 39 { + t.Fatalf("core published %d fixtures; the kit is 39 at core v0.9.0, so this test is reading a trimmed set", len(fixtures)) + } + + var carriedCount, refusedCount int + for _, fixture := range fixtures { + t.Run(fixture.Name, func(t *testing.T) { + // Through *this package's* boundary, not the SDK's. + // + // This used to call workcontext.Attach directly, which a second + // reviewer rightly said is not "through its carriage boundary": it + // exercised sdk-go and touched no line of this runtime, so the + // division of labour it asserts could have been true of the SDK + // while this package did something else entirely. What carries a + // capability here is bearerTransport, on a gateway derived for a + // module, and that is what is driven now — the capability is put in + // the cache the way a mint would, and a real request is made. + request, err := carryThroughThisRuntime(t, fixture.Token) + + side, classified := boundary[fixture.Name] + if !classified { + t.Fatalf("core publishes a fixture this test does not classify (%s: %s). Decide which side of the carriage boundary its refusal belongs to and say so in `boundary`: a fixture nobody classified used to land in a bucket that accepted either answer, which is eight of twenty-one fixtures asserting nothing.", + fixture.Name, fixture.Reason) + } + switch side { + case carried: + if err != nil { + t.Fatalf("the carrier refused %s (%s) with %v: %s", fixture.Name, fixture.Reason, err, side.why()) + } + if got := request.Header.Get(workcontext.HeaderName); got != fixture.Token { + t.Error("the capability did not reach the request header") + } + if request.Header.Get(workcontext.InstallationIDHeaderName) == "" { + t.Error("the installation the capability is sealed to did not travel beside it") + } + carriedCount++ + + case refusedHere: + if err == nil { + t.Fatalf("the carrier accepted %s (%s): %s", fixture.Name, fixture.Reason, side.why()) + } + // With one of Core's sentinels, never an error of this + // package's own: the classification is Core's, and a refusal + // this runtime invented a taxonomy for is the beginning of the + // second implementation this module is gated against. + // + // ErrUnsealed was in this list and core v0.9.0 deleted it, for + // a reason worth repeating: protovalidate refuses a capability + // with no seal before anything in core's own reader is + // reached, so no branch could produce that sentinel — and "a + // sentinel no branch can produce is worse than no sentinel, + // because a consumer writes a handler for it and the handler + // never runs". An unsealed capability arrives as ErrInvalid. + if !errors.Is(err, corework.ErrInvalid) && !errors.Is(err, corework.ErrNotACoreToken) { + t.Errorf("the carrier refused %s with %v, which is none of Core's sentinels for a capability it cannot read", fixture.Name, err) + } + refusedCount++ + } + }) + } + + if carriedCount == 0 || refusedCount == 0 { + t.Errorf("the kit exercised carried=%d refused-here=%d: both have to be non-zero or this test is checking one side of the boundary", + carriedCount, refusedCount) + } + if carriedCount+refusedCount != len(fixtures) { + t.Errorf("classified %d of %d fixtures", carriedCount+refusedCount, len(fixtures)) + } +} + +// carriage is which side of this runtime's boundary a fixture's refusal belongs +// to. +type carriage int + +const ( + // carried: the capability goes on the request. Either it is sound, or + // judging it needs something this runtime does not hold. + carried carriage = iota + // refusedHere: the capability never goes on a request, because what is + // wrong with it is a property of the bytes in hand. + refusedHere +) + +func (c carriage) why() string { + if c == carried { + return "that judgement needs the issuer's live revision, replay, grant and seal sources, which this runtime does not hold — and a carrier that refused it would be claiming a strength it does not have, which is the silent downgrade this single-implementation effort exists to end" + } + return "this is a property of the bytes in hand, so the moment to find it out is before a request is sent: refused at the far end it would name an installation mismatch for a capability that never named an installation" +} + +// boundary classifies every fixture Core publishes, by name, from Core's own +// stated reason for each. +// +// It is a table and not a rule because the previous version was a rule with a +// default branch, and the default accepted *either* answer for the eight +// fixtures Core refuses as ErrInvalid — a bucket spanning "not two base64 +// segments", which this runtime must catch, and "one byte of the payload +// changed", which only a signature can catch. Eight of twenty-one fixtures +// therefore asserted nothing, and a reviewer showed the committed test passing +// with the runtime's carriage validation removed. +// +// The division itself is unchanged and is Core's: what is visible in the bytes +// is refused here, and what needs the issuer's four sources is carried. +var boundary = map[string]carriage{ + // Sound capabilities. They travel. + "session": carried, + "operation": carried, + "delegated": carried, + "delegated-operation": carried, + "grant": carried, + + // Not the token shape at all: no separator, nothing after it, a payload + // that is not base64, or nothing. Visible without a key, so this runtime + // refuses them rather than sending a request that cannot succeed. + "empty-token": refusedHere, + "no-separator": refusedHere, + "separator-only": refusedHere, + "payload-not-b64": refusedHere, + + // No readable seal. The carrier has to put the installation beside the + // capability, and a capability that names none cannot be carried — Core + // calls the one with a partial seal invalid and this runtime calls it + // unsealed, which is the same refusal reached one field earlier. + "missing-seal": refusedHere, + "seal-without-installation": refusedHere, + + // A genuinely signed token in another encoding. Refused as a foreign format + // before any signature is considered, which is the diagnosis Core's own + // fixture exists to protect. + "foreign-encoding": refusedHere, + + // Signature judgements. A tampered payload and a key the verifier does not + // hold are indistinguishable from a sound capability without the issuer's + // keys, and an audience is a claim inside the payload — this runtime holds + // none of what it would take to say so, and the far end says so for a + // living. + "tampered-payload": carried, + "unknown-key": carried, + "another-audience": carried, + + // Live state moved under a sound capability: an installation revision, a + // build incarnation, a principal epoch, a binding revision. Judging these + // is exactly what a carrier cannot do. + "stale-installation-revision": carried, + "future-installation-revision": carried, + "unknown-installation": carried, + "stale-build-incarnation": carried, + "stale-principal-epoch": carried, + "wrong-binding-revision": carried, + // --- core v0.9.0 added eighteen fixtures. Each is classified by the one + // question this boundary turns on: is the defect visible in the BYTES IN + // HAND, or is it a judgement against the issuer's live state and keys? + + // Schema violations, refused by protovalidate inside core's own decode — + // which this runtime's carrier reaches, because reading the sealed + // installation decodes the claims. A counter at zero names nothing and + // would compare equal to a source holding nothing; half a binding and half + // an execution are not one; an unknown field nested in the seal lets data + // ride inside a signed credential nothing here reads. All structural, all + // decidable without a key. + "zero-principal-epoch": refusedHere, + "zero-installation-revision": refusedHere, + "zero-build-incarnation": refusedHere, + "seal-half-execution": refusedHere, + "partial-operation-binding": refusedHere, + "actor-without-epoch": refusedHere, + "unknown-field": refusedHere, + + // Judgements this runtime cannot make. The issuer's identity is a claim + // and the key is not the trust decision; a validity window needs a clock + // this carrier does not apply; and every remaining one compares a sealed + // value against live state only the issuer holds — an approved build, a + // principal's epoch, an authorization revision, a binding's revision, + // incarnation, grantee or installation. A carrier that refused any of + // these would be claiming a strength it does not have. + "another-issuer": carried, + "expired": carried, + "not-yet-valid": carried, + "execution-missing": carried, + "unapproved-build": carried, + "stale-actor-epoch": carried, + "superseded-authorization-revision": carried, + "revoked-operation-binding": carried, + "wrong-binding-incarnation": carried, + "binding-of-another-principal": carried, + "binding-in-another-installation": carried, +} + +// carryThroughThisRuntime drives one capability through this package's carriage +// path and returns the request that reached the far end, or the refusal that +// stopped it before anything was sent. +// +// The path is the real one: a Gateway derived by ForModule holds a delegation, +// bearerTransport resolves the capability per round trip and attaches it, and +// the request only leaves if that succeeds. A capability this runtime cannot +// carry therefore has to fail here, before a module is called — which is the +// property the kit is being used to check, and the one that cannot be observed +// by calling the SDK directly. +func carryThroughThisRuntime(t *testing.T, token string) (*http.Request, error) { + t.Helper() + var reached *http.Request + far := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + reached = r.Clone(context.Background()) + writeJSON(w, http.StatusOK, map[string]string{"ok": "yes"}) + })) + defer far.Close() + + gateway := newGateway(far.URL, "Bearer viewer", "org-1", "session-1") + // The capability as a mint would have left it in the cache, so the + // transport resolves this exact token rather than minting another. + const key = "conformance" + gateway.delegation = &delegation{key: key} + if _, err := gateway.contexts.resolve(context.Background(), key, + func(context.Context) (string, time.Time, error) { + return token, time.Now().Add(time.Hour), nil + }); err != nil { + // The cache refused to hold it at all, which is this runtime refusing + // to carry it — the same answer as a refusal at attach time. + return nil, err + } + + request, err := http.NewRequest(http.MethodPost, far.URL+"/v1/things/search", nil) + if err != nil { + t.Fatal(err) + } + resp, err := gateway.HTTPClient().Do(request) + if err != nil { + return nil, unwrapURLError(err) + } + _ = resp.Body.Close() + if reached == nil { + t.Fatal("the far end was not reached and nothing refused the call") + } + return reached, nil +} + +// unwrapURLError returns what a transport refusal actually was: http.Client +// wraps a RoundTrip error in *url.Error, which errors.Is sees through but a +// reader of the message does not. +func unwrapURLError(err error) error { + var urlErr *url.Error + if errors.As(err, &urlErr) { + return urlErr.Err + } + return err +} diff --git a/contract.go b/contract.go new file mode 100644 index 0000000..ddb1777 --- /dev/null +++ b/contract.go @@ -0,0 +1,451 @@ +package solution + +import ( + "encoding/json" + "fmt" + "net/http" + "slices" + "sort" + "strings" + + "github.com/codefly-dev/core/resources" +) + +// ContractSchema is the version of the contract document this runtime +// publishes. A reader branches on it rather than on the presence of a field, so +// a later shape is a new version and never a stricter reading of these bytes. +// +// It is deliberately *not* `codefly/module-contract/v1`. That string belongs to +// the document the renderer reads — `module.contract.codefly.yaml`, YAML, +// strict-decoded, at the module directory the composition resolved, carrying +// slots rather than literals (codefly-dev/cli#855). This runtime's document is +// JSON, has a different shape, and answers a different question: what a process +// holds *itself* to, resolved for the one profile it runs under. +// +// Sharing the string was worse than a mismatch. Strict decoding refuses the +// unknown fields, and because the schema matched, the renderer would report this +// as a *malformed* `module-contract` rather than as a document meant for +// somebody else — a version skew that is not one, pointing at the wrong owner. +// Two shapes under one schema string is the one thing a reader branching on that +// string cannot survive. +const ContractSchema = "codefly/solution-runtime-contract/v1" + +// RendererContractSchema and RendererContractFile name what the *renderer* +// reads, so the distinction above is greppable from here rather than folklore. +// Nothing in this package writes that file; a generator for it would be a +// separate deliverable in this repository, and the renderer stays a reader of +// one file. +const ( + RendererContractSchema = "codefly/module-contract/v1" + RendererContractFile = "module.contract.codefly.yaml" +) + +// ContractPath is where a running solution publishes its effective contract. +// +// Published, not announced: nothing is pushed anywhere, and answering here +// makes this solution present to nobody. It is the same document the build-time +// artifact carries (ContractArtifact), narrowed to the one profile this process +// actually runs under, so an operator can read what a process holds itself to +// without inferring it from the composition's inputs. +const ContractPath = "/.well-known/module-contract" + +// ModuleContract is the authority contract a solution declares: the ceiling +// this runtime holds its own declaration to at boot, and the ceiling an +// operator reads off a running process. +// +// It is the author's declaration of the most authority this solution may ever +// ask for. It is not the renderer's input — that is RendererContractFile, whose +// shape resolves audiences from the composition per environment rather than +// keying ceilings by profile (codefly-dev/cli#855). +// +// It is declared, never derived from what the code happens to ask for. A +// ceiling computed from the declaration would be satisfied by construction: a +// method that asked for one more action would widen the ceiling that was +// supposed to refuse it, and a reviewer approving the contract would be +// approving whatever the next commit asks for. So the author declares the +// ceiling, the runtime refuses a declaration that exceeds it, and the two +// disagreeing is a boot failure naming the scope. +// +// What *is* derived is the binding set: the audiences are the `as` of the +// solution's api.consumes entries, which Codefly projects and the passthrough +// declaration is already checked against. Restating them here would be a second +// source for one fact. +type ModuleContract struct { + // Ceilings is the most authority this solution may ever ask a consumed + // module for, keyed by configuration profile and then by the audience it is + // asked of (the `as` of that module's api.consumes entry). Every audience + // the solution mints authority for needs a ceiling in the profile this + // process runs under, and every scope the declaration asks for must fall + // inside it. + // + // The profile key is why this is a map of maps rather than one table. A + // deployment and a developer machine do not grant the same authority, and a + // deployed environment reads its own profile — "staging", say — never the + // local one. That distinction is recent: a deployed environment used to + // render under the local profile (codefly-dev/core#687, closed in core + // v0.7.1), which is exactly the mistake this keying makes impossible. A + // contract that declares only "local" is refused in a deployment rather + // than read as if the deployment were somebody's laptop. + Ceilings map[string]map[string][]Scope +} + +// Contract declares the authority contract this solution publishes. Chainable. +// +// Not to be confused with Manifest.Contract, which is the capability contract +// id the host's handshake reads. This one is about authority: which audiences +// this solution holds bindings for, and how much it may ever ask of each. +func (s *Server) Contract(contract ModuleContract) *Server { + // FROZEN here, at the declaration boundary. + // + // This stored the caller's value, which means the caller's maps and the + // caller's backing arrays. Resolution and every gateway built afterwards + // retain the ceiling's slices, so keeping an `actions := []string{"read"}` + // used in the declaration and writing "delete" into it after boot changed + // the ceiling every later request is checked against — AND the contract + // this process publishes. The ask-copying fix hardened one side of that + // comparison; this is the other side, and a ceiling a caller can still + // edit is not a ceiling. + // + // At Contract() rather than at resolution, because that is the moment the + // value stops being the caller's business: a copy taken later leaves a + // window, and there is no second place to remember to take one. + s.declaredContract = frozenContract(contract) + return s +} + +// frozenContract is a deep copy of a declaration: the profile map, each +// audience map, each ceiling slice, and the Actions and ResourceIDs inside +// every Scope. Nothing in it aliases anything the caller still holds. +func frozenContract(contract ModuleContract) ModuleContract { + if contract.Ceilings == nil { + return contract + } + frozen := ModuleContract{Ceilings: make(map[string]map[string][]Scope, len(contract.Ceilings))} + for profile, ceilings := range contract.Ceilings { + audiences := make(map[string][]Scope, len(ceilings)) + for audience, ceiling := range ceilings { + audiences[audience] = frozenScopes(ceiling) + } + frozen.Ceilings[profile] = audiences + } + return frozen +} + +// frozenScopes is a deep copy of a scope slice, including the two string +// slices inside each Scope — which a plain append copies the headers of and +// nothing else. +func frozenScopes(scopes []Scope) []Scope { + if scopes == nil { + return nil + } + frozen := make([]Scope, len(scopes)) + for i, scope := range scopes { + frozen[i] = Scope{ + ResourceKind: scope.ResourceKind, + Actions: append([]string(nil), scope.Actions...), + ResourceIDs: append([]string(nil), scope.ResourceIDs...), + } + } + return frozen +} + +// effectiveContract is the contract of the one profile this process runs under: +// what it publishes, and what its declaration was held to at boot. +type effectiveContract struct { + Schema string `json:"schema"` + Solution string `json:"solution"` + Profile string `json:"profile"` + Principal string `json:"principal"` + Bindings []contractBinding `json:"bindings"` +} + +// contractBinding is one audience this solution holds authority for, and the +// ceiling it holds it within. +type contractBinding struct { + Audience string `json:"audience"` + Ceiling []Scope `json:"ceiling"` + // Asked is what the declaration actually asks for inside that ceiling — + // the authority a reviewer is approving rather than the authority the + // ceiling would allow a later commit to ask for. + Asked []Scope `json:"asked"` +} + +// resolveContract is the contract this process publishes, refused rather than +// guessed. Every refusal names the value that is missing or the scope that +// exceeds its ceiling, because the fix for each is a different edit in a +// different file: a missing profile is a contract change, a scope outside its +// ceiling is either a declaration change or a ceiling the reviewer has to widen +// deliberately. +func (s *Server) resolveContract() (effectiveContract, error) { + contract := effectiveContract{ + Schema: ContractSchema, + Solution: s.manifest.ID, + Profile: s.cfg.profile, + // The principal is not declared here: it is one of the + // authority-bearing values the platform provisioned and this process + // froze at boot, so the contract reports who this workload actually is + // rather than who its author believed it would be. A contract that + // restated it would be a second source for one fact, and the one that + // disagreed would be this one. + Principal: s.principal, + } + bindings, err := checkContract(s.consumed, s.declaredContract, s.cfg.profile) + if err != nil { + return contract, err + } + contract.Bindings = bindings + return contract, nil +} + +// checkContract is the whole of the contract's substantive rules, for one +// profile, and it returns the bindings that profile publishes. +// +// One function, called by the boot *and* by the artifact a renderer reads. They +// used to be separate, and the artifact checked only profile names and scope +// syntax: a declaration consuming `things` could publish an artifact whose only +// ceiling was `ghost`, and the artifact is the document authority is derived +// from. A rule enforced on the running process and not on the published +// document is a rule that governs the half nobody reads. +func checkContract(modules []ConsumedModule, contract ModuleContract, profile string) ([]contractBinding, error) { + ceilings, declared := contract.Ceilings[profile] + if !declared && mintsAuthority(modules) { + return nil, fmt.Errorf("the published contract declares no %q profile (it declares: %s): a deployed environment reads its own profile and never the local one, so declare this one — or set %s if this process runs under another", + profile, declaredProfiles(contract), ContractProfileEnvironmentVariable) + } + var bindings []contractBinding + for _, module := range modules { + ceiling, declared := ceilings[module.As] + if module.ViewerBearer { + // A ViewerBearer module is called with the viewer's bearer and no + // minted capability, so there is no authority to cap — and a + // ceiling declared for one would describe authority nothing asks + // for. Refused rather than ignored: a reviewer who wrote it down + // believes it governs something. + if declared { + return nil, fmt.Errorf("the published contract declares a scope ceiling for %q in the %q profile, but that module is declared ViewerBearer: it mints no authority, so the ceiling governs nothing", + module.As, profile) + } + bindings = append(bindings, contractBinding{Audience: module.As}) + continue + } + if !declared { + return nil, fmt.Errorf("the published contract declares no scope ceiling for %q in the %q profile: every consumed module this solution mints authority for needs one, and this runtime refuses an ask it cannot check against a declared ceiling", + module.As, profile) + } + if err := checkScopes(ceiling); err != nil { + return nil, fmt.Errorf("the scope ceiling for %q in the %q profile is unusable: %w", module.As, profile, err) + } + asked := askedScopes(module) + for _, scope := range asked { + if err := withinCeiling(scope, ceiling); err != nil { + return nil, fmt.Errorf("this solution asks %q for authority outside the ceiling its contract publishes for the %q profile: %w", + module.As, profile, err) + } + } + bindings = append(bindings, contractBinding{Audience: module.As, Ceiling: ceiling, Asked: asked}) + } + // A ceiling for an audience this solution does not consume is authority + // nobody can ask for, and the renderer would grant it: the solution's + // api.consumes is the binding set, so a name that is not in it is either a + // typo or a grant that outlives the consumption it was written for. + for audience := range ceilings { + if !slices.ContainsFunc(modules, func(m ConsumedModule) bool { return m.As == audience }) { + return nil, fmt.Errorf("the published contract declares a scope ceiling for %q in the %q profile, which this solution does not consume: it is a ceiling governing a call that cannot happen, and whoever wrote it believes otherwise", + audience, profile) + } + } + sort.Slice(bindings, func(i, j int) bool { return bindings[i].Audience < bindings[j].Audience }) + return bindings, nil +} + +// declaredProfiles lists the profiles a contract does declare, so a refusal +// naming the missing one also says which ones exist — the difference between +// "add staging" and "the profile name is not what you think it is". +func declaredProfiles(contract ModuleContract) string { + if len(contract.Ceilings) == 0 { + return "none" + } + names := make([]string, 0, len(contract.Ceilings)) + for name := range contract.Ceilings { + names = append(names, name) + } + sort.Strings(names) + return strings.Join(names, ", ") +} + +// mintsAuthority reports whether any consumed module has authority to cap. A +// solution whose every module is ViewerBearer mints nothing, and a solution +// that consumes none mints nothing either: requiring a profile from them would +// be requiring a declaration about authority that does not exist. +func mintsAuthority(modules []ConsumedModule) bool { + return slices.ContainsFunc(modules, func(m ConsumedModule) bool { return !m.ViewerBearer }) +} + +// askedScopes is every scope this solution's declaration asks of one module: +// the module's own, plus each method's override. The union is what the ceiling +// is checked against, because a per-method scope replaces the module's rather +// than narrowing it, so the widest ask is not the module-level one. +func askedScopes(module ConsumedModule) []Scope { + asked := append([]Scope{}, module.Scopes...) + for _, method := range module.Methods { + asked = append(asked, method.Scopes...) + } + return asked +} + +// withinCeiling reports whether one asked scope falls inside a ceiling: some +// one entry of the ceiling must cover its resource kind, every action it asks +// for, and — when that entry names resource ids — every id it names. +// +// Every entry of that kind is examined before anything is refused. It used to +// return the first matching kind's verdict, which made the answer depend on the +// order the ceiling was written in: with `things/read/{a}` and +// `things/read/{b}` declared in that order, an ask for `b` was refused against +// the first entry and the second was never read. A ceiling is a set, and a +// reviewer who writes one does not also choose a traversal order. +// +// Coverage has to come from a single entry rather than from the union of +// several. Two entries that each cover half an ask describe two grants, and +// treating their union as one would let a ceiling of "read on a" plus "list on +// b" authorize "read and list on a and b" — authority nobody declared. +// +// An entry with no resource ids covers the whole resource kind, which is how +// Scope itself reads an empty ResourceIDs. An ask with no ids against entries +// that all name some is therefore refused: "every document" is not inside +// "these two documents", and silently reading it as the narrower ask would mint +// authority the declaration did not write. +func withinCeiling(asked Scope, ceiling []Scope) error { + var reasons []string + kindNamed := false + for _, allowed := range ceiling { + if allowed.ResourceKind != asked.ResourceKind { + continue + } + kindNamed = true + if reason := coveredBy(asked, allowed); reason != "" { + reasons = append(reasons, reason) + continue + } + return nil + } + if !kindNamed { + return fmt.Errorf("the ceiling names no resource kind %q at all", asked.ResourceKind) + } + // Every entry of that kind was read and none covers the ask. The reasons + // are reported together, because "which of the three entries did you mean" + // is the first question a reader has. + return fmt.Errorf("no single ceiling entry for %q covers it: %s", asked.ResourceKind, strings.Join(reasons, "; ")) +} + +// coveredBy is why one ceiling entry does not cover an ask, or "" when it does. +func coveredBy(asked, allowed Scope) string { + for _, action := range asked.Actions { + if !slices.Contains(allowed.Actions, action) { + return fmt.Sprintf("the entry allowing [%s] does not allow %q", strings.Join(allowed.Actions, ", "), action) + } + } + if len(allowed.ResourceIDs) == 0 { + return "" + } + if len(asked.ResourceIDs) == 0 { + return fmt.Sprintf("the entry names only the resources [%s], and the ask covers the whole resource kind", strings.Join(allowed.ResourceIDs, ", ")) + } + for _, id := range asked.ResourceIDs { + if !slices.Contains(allowed.ResourceIDs, id) { + return fmt.Sprintf("the entry naming [%s] does not name %q", strings.Join(allowed.ResourceIDs, ", "), id) + } + } + return "" +} + +func (s *Server) handleContract(w http.ResponseWriter, _ *http.Request) { + writeJSON(w, http.StatusOK, s.contract) +} + +// ContractArtifact renders this runtime's contract document at build time: the +// binding set derived from this solution's api.consumes declaration, and the +// scope ceiling it may ever ask of each, per profile — the bytes to write beside +// the interface artifact (InterfaceArtifact). +// +// It is *not* the document the renderer derives authority from. The renderer +// reads RendererContractFile under RendererContractSchema, a YAML shape with +// `{from: /}` slots that a composition resolves per environment +// (codefly-dev/cli#855); these bytes are this runtime's own document, under this +// runtime's own schema string, and the renderer does not read them. What they +// are good for is review and diffing a build: the ceiling an author declared, +// checked by exactly the rule the boot applies. +// +// It takes the declaration rather than a running server, because a render +// happens where no solution is running, and it resolves no profile for the same +// reason: which profile applies is the renderer's to decide, and a build-time +// document that had already chosen one would be a document that is true in one +// environment. +// +// It carries no principal. The principal is an authority-bearing value the +// platform provisions and the running process freezes at boot +// (AuthorityGroup/PRINCIPAL); a build-time document that named one would be +// asserting at render time what only the deployment knows. +func ContractArtifact(id string, contract ModuleContract, modules ...ConsumedModule) ([]byte, error) { + if strings.TrimSpace(id) == "" { + return nil, fmt.Errorf("a contract artifact names no solution: pass the manifest id") + } + document := struct { + Schema string `json:"schema"` + Solution string `json:"solution"` + Bindings []artifactBinding `json:"bindings"` + Profiles map[string]map[string][]Scope `json:"profiles"` + }{Schema: ContractSchema, Solution: id, Profiles: map[string]map[string][]Scope{}} + + for _, module := range modules { + document.Bindings = append(document.Bindings, artifactBinding{Audience: module.As, ViewerBearer: module.ViewerBearer}) + } + sort.Slice(document.Bindings, func(i, j int) bool { return document.Bindings[i].Audience < document.Bindings[j].Audience }) + + for profile, ceilings := range contract.Ceilings { + if err := resources.ValidateConfigurationProfileName(profile); err != nil { + return nil, fmt.Errorf("the contract declares an unusable profile name: %w", err) + } + // Every rule the boot holds a running process to, held here too, per + // profile — so a document that would refuse to boot cannot be + // published at all. + if _, err := checkContract(modules, contract, profile); err != nil { + return nil, err + } + document.Profiles[profile] = ceilings + } + if mintsAuthority(modules) && len(document.Profiles) == 0 { + // Says what it checks. It used to demand "a deployed one beside local" + // while accepting a contract that declared only local, which is a + // refusal promising a stricter rule than the one it applies — and the + // reader who satisfies the message learns nothing, because they were + // already passing. + // + // Which profiles will be deployed is not knowable here: a build-time + // document is rendered once and read under whichever profile a + // composition runs. A deployed environment missing its profile is + // refused at boot, where the profile it ran under is a fact and the + // refusal can name it, rather than guessed at here — and a + // local-only contract is the correct contract for a solution that is + // only ever run locally. + return nil, fmt.Errorf("the contract declares no profile at all, and this solution mints authority for %d consumed module(s): declare the scope ceilings per profile — a deployed environment missing its own profile is then refused at boot, naming the profile it ran under", + len(modules)) + } + return json.MarshalIndent(document, "", " ") +} + +// artifactBinding is one audience the solution holds a binding for. A +// ViewerBearer module is reported as one, with the flag set: this solution mints +// no authority for it, and a binding simply missing from the document would read +// as a module the solution does not consume. (The renderer's own shape takes the +// opposite convention — a binding it derives no authority for is one you do not +// declare, so absence there means "asks for nothing" — which is one more reason +// these two documents do not share a schema string.) +type artifactBinding struct { + Audience string `json:"audience"` + ViewerBearer bool `json:"viewerBearer,omitempty"` +} + +// localProfile is the profile a local run resolves to: the local environment's +// own name, which is what Core uses for an environment that declares no profile +// of its own. +const localProfile = "local" diff --git a/contract_test.go b/contract_test.go new file mode 100644 index 0000000..7dea9b3 --- /dev/null +++ b/contract_test.go @@ -0,0 +1,754 @@ +package solution + +import ( + "context" + "encoding/json" + "errors" + "net/http" + "path/filepath" + "strings" + "testing" + + "github.com/codefly-dev/core/solution/manifest" + "github.com/codefly-dev/sdk-go/workcontext" +) + +// contractServer is a solution consuming one module, with the declaration and +// the contract a test varies. +func contractServer(t *testing.T, profile string, contract ModuleContract, modules ...ConsumedModule) *Server { + t.Helper() + if len(modules) == 0 { + modules = []ConsumedModule{passthroughModule()} + } + server := New(Manifest{ID: testSolutionID}).Consumes(modules...).Contract(contract) + server.cfg = config{profile: profile, apiConsumes: consumesThings} + server.principal = testPrincipal + return server +} + +// TestAContractWithoutThisProfileIsRefusedByName is the profile gap, closed. A +// deployed environment reads its own profile — core v0.7.1 stopped it rendering +// under the local one (codefly-dev/core#687) — so a contract that declares only +// "local" is refused in a deployment rather than read as if the deployment were +// somebody's laptop. +func TestAContractWithoutThisProfileIsRefusedByName(t *testing.T) { + server := contractServer(t, "staging", ModuleContract{Ceilings: map[string]map[string][]Scope{ + localProfile: {"things": {{ResourceKind: "things", Actions: []string{"read"}}}}, + }}) + _, err := server.resolveContract() + if err == nil { + t.Fatal("a contract declaring only the local profile was accepted in staging") + } + for _, want := range []string{"staging", localProfile, ContractProfileEnvironmentVariable} { + if !strings.Contains(err.Error(), want) { + t.Errorf("refusal %q does not name %q", err, want) + } + } +} + +// TestAnAskOutsideItsCeilingIsRefusedNamingTheScope: the ceiling is declared, +// so a declaration that asks for more is a boot failure rather than a ceiling +// that silently widened to fit it. +func TestAnAskOutsideItsCeilingIsRefusedNamingTheScope(t *testing.T) { + module := passthroughModule() + module.Methods[0].Scopes = []Scope{{ResourceKind: "things", Actions: []string{"delete"}}} + server := contractServer(t, localProfile, ModuleContract{Ceilings: map[string]map[string][]Scope{ + localProfile: {"things": {{ResourceKind: "things", Actions: []string{"read"}}}}, + }}, module) + _, err := server.resolveContract() + if err == nil { + t.Fatal("a method asking for an action outside the ceiling was accepted") + } + for _, want := range []string{"delete", "things", localProfile} { + if !strings.Contains(err.Error(), want) { + t.Errorf("refusal %q does not name %q", err, want) + } + } +} + +// TestACeilingNamingResourceIdsRefusesAnAskAcrossTheKind: "every thing" is not +// inside "these two things", and reading it as the narrower ask would mint +// authority the declaration never wrote. +func TestACeilingNamingResourceIdsRefusesAnAskAcrossTheKind(t *testing.T) { + server := contractServer(t, localProfile, ModuleContract{Ceilings: map[string]map[string][]Scope{ + localProfile: {"things": {{ResourceKind: "things", Actions: []string{"read"}, ResourceIDs: []string{"a", "b"}}}}, + }}) + _, err := server.resolveContract() + if err == nil { + t.Fatal("an ask across the whole resource kind was accepted under a ceiling naming two resources") + } + if !strings.Contains(err.Error(), "whole resource kind") { + t.Errorf("refusal %q does not say the ask covers the whole resource kind", err) + } +} + +// TestAMissingCeilingAndASuperfluousOneAreBothRefused: every audience that +// mints authority needs a ceiling, and a ceiling for an audience nothing +// consumes is authority the renderer would grant for a call that cannot happen. +func TestAMissingCeilingAndASuperfluousOneAreBothRefused(t *testing.T) { + t.Run("an audience with no ceiling", func(t *testing.T) { + server := contractServer(t, localProfile, ModuleContract{Ceilings: map[string]map[string][]Scope{ + localProfile: {"other": {{ResourceKind: "other", Actions: []string{"read"}}}}, + }}) + _, err := server.resolveContract() + if err == nil || !strings.Contains(err.Error(), "things") { + t.Fatalf("resolveContract = %v, want a refusal naming the audience with no ceiling", err) + } + }) + t.Run("a ceiling for an audience nothing consumes", func(t *testing.T) { + server := contractServer(t, localProfile, ModuleContract{Ceilings: map[string]map[string][]Scope{ + localProfile: { + "things": {{ResourceKind: "things", Actions: []string{"read"}}}, + "ghost": {{ResourceKind: "ghost", Actions: []string{"read"}}}, + }, + }}) + _, err := server.resolveContract() + if err == nil || !strings.Contains(err.Error(), "ghost") { + t.Fatalf("resolveContract = %v, want a refusal naming the audience nothing consumes", err) + } + }) +} + +// TestAViewerBearerModuleTakesNoCeiling: it mints nothing, so a ceiling +// declared for it governs nothing — and a reviewer who wrote one down believes +// it does. +func TestAViewerBearerModuleTakesNoCeiling(t *testing.T) { + module := ConsumedModule{As: "things", ViewerBearer: true, Methods: []ConsumedMethod{{ + Name: "/things.v1.Things/Search", Response: MustFieldMask(thingMessage("Thing"), "entry_id"), + }}} + t.Run("a ceiling on it is refused", func(t *testing.T) { + server := contractServer(t, localProfile, ModuleContract{Ceilings: map[string]map[string][]Scope{ + localProfile: {"things": {{ResourceKind: "things", Actions: []string{"read"}}}}, + }}, module) + _, err := server.resolveContract() + if err == nil || !strings.Contains(err.Error(), "ViewerBearer") { + t.Fatalf("resolveContract = %v, want a refusal naming the ViewerBearer declaration", err) + } + }) + t.Run("no profile is needed for it", func(t *testing.T) { + server := contractServer(t, "staging", ModuleContract{}, module) + contract, err := server.resolveContract() + if err != nil { + t.Fatalf("resolveContract = %v, want no refusal: a solution that mints nothing has no authority to cap", err) + } + if len(contract.Bindings) != 1 || len(contract.Bindings[0].Ceiling) != 0 { + t.Errorf("published bindings = %+v, want the audience with no ceiling", contract.Bindings) + } + }) +} + +// TestACeilingIsASetNotAnOrderedList: the verdict used to depend on which +// entry of the matching kind came first, so reordering a ceiling changed +// whether a declaration booted. A reviewer who writes a ceiling does not also +// choose a traversal order. +func TestACeilingIsASetNotAnOrderedList(t *testing.T) { + module := passthroughModule() + module.Scopes = []Scope{{ResourceKind: "things", Actions: []string{"read"}, ResourceIDs: []string{"b"}}} + entries := [][]Scope{ + {{ResourceKind: "things", Actions: []string{"read"}, ResourceIDs: []string{"a"}}, {ResourceKind: "things", Actions: []string{"read"}, ResourceIDs: []string{"b"}}}, + {{ResourceKind: "things", Actions: []string{"read"}, ResourceIDs: []string{"b"}}, {ResourceKind: "things", Actions: []string{"read"}, ResourceIDs: []string{"a"}}}, + } + for i, ceiling := range entries { + server := contractServer(t, localProfile, ModuleContract{Ceilings: map[string]map[string][]Scope{ + localProfile: {"things": ceiling}, + }}, module) + if _, err := server.resolveContract(); err != nil { + t.Errorf("ceiling order %d refused an ask a later entry covers: %v", i, err) + } + } + + // And the refusal, when no single entry covers the ask, says so rather + // than reporting whichever entry happened to be first. + module.Scopes = []Scope{{ResourceKind: "things", Actions: []string{"read", "list"}}} + server := contractServer(t, localProfile, ModuleContract{Ceilings: map[string]map[string][]Scope{ + localProfile: {"things": { + {ResourceKind: "things", Actions: []string{"read"}}, + {ResourceKind: "things", Actions: []string{"list"}}, + }}, + }}, module) + _, err := server.resolveContract() + if err == nil { + t.Fatal("two entries each covering half an ask were treated as covering it: that is authority nobody declared") + } + if !strings.Contains(err.Error(), "no single ceiling entry") { + t.Errorf("refusal %q does not say that no single entry covers the ask", err) + } +} + +// TestTheArtifactRefusesWhatTheBootWouldRefuse: the artifact is the document +// authority is derived from, so a rule enforced on the running process and not +// on the published document governs the half nobody reads. +func TestTheArtifactRefusesWhatTheBootWouldRefuse(t *testing.T) { + module := passthroughModule() + for _, tc := range []struct { + name string + ceilings map[string]map[string][]Scope + says string + }{ + { + name: "a surplus audience", + ceilings: map[string]map[string][]Scope{localProfile: {"things": {{ResourceKind: "things", Actions: []string{"read"}}}, "ghost": {{ResourceKind: "ghost", Actions: []string{"read"}}}}}, + says: "ghost", + }, + { + name: "an audience with no ceiling", + ceilings: map[string]map[string][]Scope{localProfile: {"ghost": {{ResourceKind: "ghost", Actions: []string{"read"}}}}}, + says: "things", + }, + { + name: "an ask outside its ceiling", + ceilings: map[string]map[string][]Scope{localProfile: {"things": {{ResourceKind: "things", Actions: []string{"list"}}}}}, + says: "outside the ceiling", + }, + { + name: "a profile that is fine beside one that is not", + ceilings: map[string]map[string][]Scope{ + localProfile: {"things": {{ResourceKind: "things", Actions: []string{"read"}}}}, + "staging": {"things": {{ResourceKind: "things", Actions: []string{"list"}}}}, + }, + says: "staging", + }, + } { + t.Run(tc.name, func(t *testing.T) { + _, err := ContractArtifact(testSolutionID, ModuleContract{Ceilings: tc.ceilings}, module) + if err == nil { + t.Fatal("the artifact published a contract the boot would refuse") + } + if !strings.Contains(err.Error(), tc.says) { + t.Errorf("refusal %q does not name %q", err, tc.says) + } + }) + } +} + +// TestTheContractArtifactCarriesEveryProfileAndNoPrincipal: the build-time +// document carries the binding set and the per-profile ceilings, and no +// principal — that is a value only the deployment knows. +// +// It was called "…IsWhatTheRendererReads" until cli#855 answered that the +// renderer reads RendererContractFile instead, in a different shape. The name +// was the only thing asserting that, and a test name is where a refuted claim +// survives longest: nothing here ever exercised a renderer, so nothing failed +// when it stopped being true. +func TestTheContractArtifactCarriesEveryProfileAndNoPrincipal(t *testing.T) { + raw, err := ContractArtifact(testSolutionID, ModuleContract{Ceilings: map[string]map[string][]Scope{ + localProfile: {"things": {{ResourceKind: "things", Actions: []string{"read"}}}}, + "staging": {"things": {{ResourceKind: "things", Actions: []string{"read", "list"}}}}, + }}, passthroughModule()) + if err != nil { + t.Fatal(err) + } + var document struct { + Schema string `json:"schema"` + Solution string `json:"solution"` + Bindings []artifactBinding `json:"bindings"` + Profiles map[string]map[string][]Scope `json:"profiles"` + } + if err := json.Unmarshal(raw, &document); err != nil { + t.Fatal(err) + } + if document.Schema != ContractSchema || document.Solution != testSolutionID { + t.Errorf("artifact names %q/%q, want %q/%q", document.Schema, document.Solution, ContractSchema, testSolutionID) + } + if len(document.Bindings) != 1 || document.Bindings[0].Audience != "things" { + t.Errorf("artifact bindings = %+v, want the one audience the declaration consumes", document.Bindings) + } + if _, ok := document.Profiles["staging"]; !ok { + t.Error("artifact has no staging profile: a deployed process would have none to resolve its ceiling from") + } + if strings.Contains(string(raw), testPrincipal) { + t.Error("the artifact carries a principal: that is a value only the deployment knows") + } + + if _, err := ContractArtifact(testSolutionID, ModuleContract{}, passthroughModule()); err == nil { + t.Error("a contract declaring no profile at all was rendered for a solution that mints authority") + } + if _, err := ContractArtifact("", ModuleContract{}); err == nil { + t.Error("an artifact naming no solution was rendered") + } +} + +// TestAnUnusableProfileNameIsRefused: a profile name selects a directory on +// disk wherever one is read, so Core's own rule applies here too. +func TestAnUnusableProfileNameIsRefused(t *testing.T) { + if _, err := ContractArtifact(testSolutionID, ModuleContract{Ceilings: map[string]map[string][]Scope{ + "../local": {"things": {{ResourceKind: "things", Actions: []string{"read"}}}}, + }}, passthroughModule()); err == nil { + t.Error("a traversing profile name was rendered") + } + cfg := config{ + port: "8080", gatewayURL: "https://gateway:42152", mintURL: "https://gateway:42152" + credentialMintPath, + identityCertFile: "c", identityKeyFile: "k", projectedTokenPath: "t", trustBundleFile: "b", profile: "../local", + } + if err := cfg.validate(); err == nil || !strings.Contains(err.Error(), ContractProfileEnvironmentVariable) { + t.Errorf("validate = %v, want a refusal naming %s", err, ContractProfileEnvironmentVariable) + } +} + +var _ = manifest.APIConsumesEnvironmentVariable + +// TestThisRuntimesContractDoesNotClaimTheRenderersSchema pins a cross-repo fact +// that is invisible from inside this package. +// +// The renderer reads one file — RendererContractFile, YAML, strict-decoded, at +// the module directory the composition resolved — under +// RendererContractSchema, in a shape carrying `{from: /}` slots a +// composition resolves per environment (codefly-dev/cli#855). This runtime +// publishes a different document, in JSON, keyed by profile, answering what a +// process holds itself to. +// +// Both surfaces of this package once claimed the renderer's schema string for +// that different shape, which is strictly worse than a mismatch: strict +// decoding refuses the unknown fields, and because the string matched, the +// renderer reports a *malformed* module-contract rather than a document meant +// for someone else — a version skew that is not one, pointing at the wrong +// owner. Two shapes under one schema string is the one case a reader branching +// on that string cannot survive. +// +// So this test fails if the strings ever converge again, whichever side moves. +func TestThisRuntimesContractDoesNotClaimTheRenderersSchema(t *testing.T) { + if ContractSchema == RendererContractSchema { + t.Fatalf("this runtime publishes %q, the schema string the renderer's own document uses: the renderer strict-decodes %s and would report these bytes as a malformed module contract rather than another document", + ContractSchema, RendererContractFile) + } + + // And it is claimed on both surfaces, so neither can drift back on its own: + // the build-time artifact... + raw, err := ContractArtifact(testSolutionID, ModuleContract{Ceilings: map[string]map[string][]Scope{ + localProfile: {"things": {{ResourceKind: "things", Actions: []string{"read"}}}}, + }}, ConsumedModule{As: "things", Scopes: []Scope{{ResourceKind: "things", Actions: []string{"read"}}}}) + if err != nil { + t.Fatal(err) + } + var artifact struct{ Schema string } + if err := json.Unmarshal(raw, &artifact); err != nil { + t.Fatal(err) + } + if artifact.Schema != ContractSchema { + t.Errorf("the build-time artifact names schema %q, want %q", artifact.Schema, ContractSchema) + } + + // ...and the document a running process answers with. + server := New(Manifest{ID: testSolutionID}).Consumes(ConsumedModule{ + As: "things", Scopes: []Scope{{ResourceKind: "things", Actions: []string{"read"}}}, + }).Contract(ModuleContract{Ceilings: map[string]map[string][]Scope{ + localProfile: {"things": {{ResourceKind: "things", Actions: []string{"read"}}}}, + }}) + server.cfg.profile = localProfile + effective, err := server.resolveContract() + if err != nil { + t.Fatal(err) + } + if effective.Schema != ContractSchema { + t.Errorf("the served contract names schema %q, want %q", effective.Schema, ContractSchema) + } +} + +// TestABootedRuntimeHoldsForModuleToThePublishedCeiling is the authority bypass +// a second reviewer found, and it defeats the point of publishing a ceiling at +// all. +// +// checkContract governs the Consumes *declaration*. ForModule is public, takes +// any audience and any scopes, and went straight to the mint — so the document +// saying "the most authority this solution may ever ask for" was a statement +// about a declaration the handlers did not have to use. Worse at the other end: +// a solution whose handlers only call ForModule consumes nothing, so +// mintsAuthority was false, so it needed no profile and published an empty +// contract while minting whatever it liked. +// +// This boots a real runtime with a declared ceiling and drives all three cases +// through its handler, counting what the host was asked — a refusal that still +// minted would be no refusal. +func TestABootedRuntimeHoldsForModuleToThePublishedCeiling(t *testing.T) { + type ask struct { + audience string + scope Scope + } + for _, tc := range []struct { + name string + ask ask + wantMint bool + says string + }{ + { + "inside the ceiling", ask{"things", Scope{ResourceKind: "things", Actions: []string{"read"}}}, + true, "", + }, + { + "an action the ceiling does not allow", ask{"things", Scope{ResourceKind: "things", Actions: []string{"delete"}}}, + false, "outside the ceiling its contract publishes", + }, + { + "an audience the contract names no binding for", ask{"ghost", Scope{ResourceKind: "things", Actions: []string{"read"}}}, + false, "names no binding for that audience", + }, + } { + t.Run(tc.name, func(t *testing.T) { + t.Setenv(manifest.APIConsumesEnvironmentVariable, consumesThings) + mint := newHostMint(t, &hostMint{}) + var mintErr error + solution := boot(t, New(Manifest{ID: testSolutionID}). + Consumes(passthroughModule()). + Contract(ModuleContract{Ceilings: map[string]map[string][]Scope{ + localProfile: {"things": {{ResourceKind: "things", Actions: []string{"read"}}}}, + }}). + Handle("/thing", func(ctx context.Context, gw *Gateway) (any, error) { + _, mintErr = gw.ForModule(ctx, tc.ask.audience, tc.ask.scope) + return map[string]string{"ok": "yes"}, nil + }), mint) + + request, err := http.NewRequest(http.MethodGet, solution.base+"/thing", nil) + if err != nil { + t.Fatal(err) + } + request.Header.Set("authorization", viewerBearer()) + request.Header.Set(orgHeader, "org-1") + request.Header.Set(sessionHeader, "session-1") + request.Header.Set(workcontext.InstallationIDHeaderName, testInstallation) + resp, err := solution.client.Do(request) + if err != nil { + t.Fatal(err) + } + // DRAINED, not just closed. + // + // mintErr is written by the HANDLER goroutine and read here. An + // undrained Close can return before the handler has finished, so + // nothing ordered that write before this read — a flake that + // surfaced once in a full run and reproduced in none of twelve + // isolated ones, which is what an unsynchronised read looks like. + // Draining waits for the response to complete, which is the + // happens-before this needs. + drainAndClose(resp) + + // The mint the boot itself ran is request-independent; what matters + // is whether the viewer's mint was attempted on top of it. + viewerMints := len(mint.observedStartTasks()) + switch { + case tc.wantMint && mintErr != nil: + t.Fatalf("an ask inside the published ceiling was refused: %v", mintErr) + case tc.wantMint && viewerMints != 1: + t.Errorf("an ask inside the ceiling produced %d viewer mints, want 1", viewerMints) + case !tc.wantMint: + if mintErr == nil { + t.Fatalf("ForModule minted %s for %q, which the published contract does not allow", + scopeText([]Scope{tc.ask.scope}), tc.ask.audience) + } + if !strings.Contains(mintErr.Error(), tc.says) { + t.Errorf("refusal %q does not say %q", mintErr, tc.says) + } + if viewerMints != 0 { + t.Errorf("the host was asked for %d viewer mint(s) on an ask outside the contract, want 0: a refusal that still mints is not a refusal", viewerMints) + } + // The viewer sees their own 403-shaped answer, not this + // solution's 502. + var clientErr *ClientError + if !errors.As(mintErr, &clientErr) || clientErr.StatusCode != http.StatusForbidden { + t.Errorf("refusal carries %T, want a ClientError with 403 so the page is told the solution asked for too much", mintErr) + } + } + }) + } +} + +// TestTheCeilingRefusesAsksThatStateNoAuthority closes three ways past the +// ceiling that a third reviewer found: the check governed what was asked for +// and not whether anything was asked for. +// +// - ForModule with no scopes, and a Scope naming a kind with no actions, both +// satisfied every ceiling by construction, because the loop had nothing to +// check. What accounts does with an empty authorityScopes is its decision, +// and leaving it there is this runtime declining to govern the one thing +// its contract claims to govern. +// - a ViewerBearer audience carries a nil ceiling and still sat in the map, +// so a declared lookup succeeded and ForModule minted real authority for +// the one kind of module the contract publishes as minting none. +func TestTheCeilingRefusesAsksThatStateNoAuthority(t *testing.T) { + t.Setenv(manifest.APIConsumesEnvironmentVariable, consumesThings) + mint := newHostMint(t, &hostMint{}) + type ask struct { + audience string + scopes []Scope + } + var attempted ask + var mintErr error + solution := boot(t, New(Manifest{ID: testSolutionID}). + Consumes(passthroughModule()). + Contract(ModuleContract{Ceilings: map[string]map[string][]Scope{ + localProfile: {"things": {{ResourceKind: "things", Actions: []string{"read"}}}}, + }}). + Handle("/thing", func(ctx context.Context, gw *Gateway) (any, error) { + _, mintErr = gw.ForModule(ctx, attempted.audience, attempted.scopes...) + return map[string]string{"ok": "yes"}, nil + }), mint) + + for _, tc := range []struct { + name string + ask ask + says string + }{ + {"no scopes at all", ask{"things", nil}, "no scopes at all"}, + {"a scope with no actions", ask{"things", []Scope{{ResourceKind: "things"}}}, "no actions"}, + } { + t.Run(tc.name, func(t *testing.T) { + attempted, mintErr = tc.ask, nil + before := len(mint.observedStartTasks()) + + request, err := http.NewRequest(http.MethodGet, solution.base+"/thing", nil) + if err != nil { + t.Fatal(err) + } + request.Header.Set("authorization", viewerBearer()) + request.Header.Set(orgHeader, "org-1") + request.Header.Set(sessionHeader, "session-1") + request.Header.Set(workcontext.InstallationIDHeaderName, testInstallation) + resp, err := solution.client.Do(request) + if err != nil { + t.Fatal(err) + } + // Drained, for the reason the sibling test above now gives: + // mintErr is the HANDLER's write and this is the reader. + drainAndClose(resp) + + if mintErr == nil { + t.Fatalf("ForModule(%q, %+v) minted: an ask the published ceiling cannot govern must be refused locally", tc.ask.audience, tc.ask.scopes) + } + if !strings.Contains(mintErr.Error(), tc.says) { + t.Errorf("refusal %q does not say %q", mintErr, tc.says) + } + if got := len(mint.observedStartTasks()) - before; got != 0 { + t.Errorf("the host was asked for %d viewer mint(s), want 0", got) + } + var clientErr *ClientError + if !errors.As(mintErr, &clientErr) || clientErr.StatusCode != http.StatusForbidden { + t.Errorf("refusal carries %T, want a ClientError with 403", mintErr) + } + }) + } +} + +// TestAViewerBearerAudienceCannotBeMintedFor is the third way past the ceiling, +// tested where the shape is reachable: a ViewerBearer binding carries a nil +// ceiling and still sat in the map a gateway is given, so the `declared` lookup +// succeeded and ForModule minted real authority for the one kind of module the +// published contract says mints none. +// +// Driven through resolveContract and gatewayFor rather than a boot, because the +// passthrough validator has its own requirements for a declared module's +// methods and they are not what is under test here. +func TestAViewerBearerAudienceCannotBeMintedFor(t *testing.T) { + gw := newModuleGateway(t, http.StatusOK, `{"entry_id":"e1"}`) + server := New(Manifest{ID: testSolutionID}). + Consumes(passthroughModule(), ConsumedModule{As: "pages", ViewerBearer: true}). + Contract(ModuleContract{Ceilings: map[string]map[string][]Scope{ + localProfile: {"things": {{ResourceKind: "things", Actions: []string{"read"}}}}, + }}) + server.cfg.profile = localProfile + server.cfg.gatewayURL = gw.URL + contract, err := server.resolveContract() + if err != nil { + t.Fatalf("resolveContract: %v", err) + } + server.contract = contract + // Said explicitly, because the ceiling is governing only for a contract a + // boot resolved. It used to be inferred from the contract carrying a + // solution id, which is why an empty Manifest.ID switched the ceiling off. + server.contractResolved = true + // The binding is published, with the flag and no ceiling. + var published bool + for _, binding := range contract.Bindings { + if binding.Audience == "pages" { + published = true + if len(binding.Ceiling) != 0 { + t.Errorf("the ViewerBearer binding publishes a ceiling of %+v, want none", binding.Ceiling) + } + } + } + if !published { + t.Fatal("the ViewerBearer module is not in the published contract at all") + } + + header := http.Header{} + header.Set("authorization", viewerBearer()) + header.Set(orgHeader, "org-1") + header.Set(sessionHeader, "session-1") + header.Set(workcontext.InstallationIDHeaderName, testInstallation) + _, err = server.gatewayFor(header).ForModule(context.Background(), "pages", + Scope{ResourceKind: "pages", Actions: []string{"read"}}) + if err == nil { + t.Fatal("ForModule minted authority for a module the contract publishes as minting none") + } + if !strings.Contains(err.Error(), "no ceiling at all") { + t.Errorf("refusal %q does not say the module has no ceiling", err) + } + if got := len(gw.observedMints()); got != 0 { + t.Errorf("observed %d mints for a ViewerBearer audience, want 0", got) + } +} + +// TestAnEmptyManifestIdCannotDisableTheCeiling: the ceiling applied only when +// the resolved contract carried a solution id, and the solution id is +// Manifest.ID — so the field that identifies the solution doubled as the +// sentinel for "a boot resolved this contract". An empty id left the ceiling +// switched off while every other part of the boot reported success, and +// ForModule asking for no scopes against an undeclared audience reached the +// host as a viewer mint. +// +// Two independent reasons it cannot recur: the ceiling is keyed on its own +// flag, and an empty id is refused at boot. A value that is also a sentinel +// has one, and which one it is depends on a caller. +func TestAnEmptyManifestIdCannotDisableTheCeiling(t *testing.T) { + t.Run("an empty id is refused at boot", func(t *testing.T) { + err := Manifest{ID: ""}.validateSurfaces() + if err == nil { + t.Fatal("a manifest with no id was accepted: it is what the presence document names this solution by, what the published contract reports, and what every refusal here identifies this process as") + } + if !strings.Contains(err.Error(), "Manifest.ID") { + t.Errorf("the refusal %q does not name the field that fixes it", err) + } + // And whitespace is not an id either. + if err := (Manifest{ID: " "}).validateSurfaces(); err == nil { + t.Error("a manifest whose id is whitespace was accepted") + } + }) + + // The one that matters: even reaching the state by hand, the ceiling is + // still governing, because it no longer reads an id to decide. + t.Run("the ceiling governs a resolved contract with no id", func(t *testing.T) { + gw := newModuleGateway(t, http.StatusOK, `{"entry_id":"e1"}`) + tokenFile := filepath.Join(t.TempDir(), "token") + writeFile(t, tokenFile, "projected") + mint := newHostMint(t, &hostMint{}) + server := New(Manifest{ID: ""}). + Consumes(passthroughModule()). + Contract(ModuleContract{Ceilings: map[string]map[string][]Scope{ + localProfile: {"things": {{ResourceKind: "things", Actions: []string{"read"}}}}, + }}). + // A working credential, so the only thing that can refuse this ask + // is the ceiling. Without it the ask fails for want of an + // attestation whether the ceiling governs or not, and this test + // passes either way — which is exactly how the mutation survived + // the first version of it. + Credential(mintClientFor(t, mint, tokenFile)) + server.cfg.profile = localProfile + server.cfg.gatewayURL = gw.URL + contract, err := server.resolveContract() + if err != nil { + t.Fatalf("resolveContract: %v", err) + } + if contract.Solution != "" { + t.Fatalf("this test needs a resolved contract carrying no solution id, got %q", contract.Solution) + } + server.contract = contract + server.contractResolved = true + + header := http.Header{} + header.Set("authorization", viewerBearer()) + header.Set(orgHeader, "org-1") + header.Set(sessionHeader, "session-1") + header.Set(workcontext.InstallationIDHeaderName, testInstallation) + // With a stated scope, so the ask reaches the ceiling rather than the + // earlier refusal for an ask that names no authority at all. + _, err = server.gatewayFor(header).ForModule(context.Background(), "undeclared", + Scope{ResourceKind: "secrets", Actions: []string{"read"}}) + if err == nil { + t.Fatal("an ask for an audience the contract does not declare was minted for, on a contract carrying no solution id: the ceiling has to be keyed on whether a boot resolved it, not on a field that happens to be non-empty") + } + // And refused BY THE CEILING. Any-error would be satisfied by an ask + // that failed for want of a credential, a gateway, or anything else. + if !strings.Contains(err.Error(), "names no binding for that audience") { + t.Fatalf("the ask was refused for %q, not by the published ceiling: the ceiling is not governing this contract at all", err) + } + + // The control: a declared audience inside the ceiling is still minted + // for, so the ceiling is applying rather than refusing everything. + if _, err := server.gatewayFor(header).ForModule(context.Background(), "things", + Scope{ResourceKind: "things", Actions: []string{"read"}}); err != nil { + t.Fatalf("an ask inside the published ceiling was refused: %v", err) + } + mints := gw.observedMints() + if len(mints) != 1 { + t.Errorf("observed %d mints, want exactly the one inside the ceiling: an ask outside it must never reach the host", len(mints)) + } + for _, mint := range mints { + if mint.Audience == "undeclared" { + t.Error("the ask for an undeclared audience reached the host") + } + } + }) +} + +// TestTheDeclaredCeilingIsFrozenAtDeclaration is round fifteen's first major. +// +// Contract() stored the caller's value — the caller's maps, and the caller's +// backing arrays inside every Scope. Resolution and every gateway built +// afterwards retain the ceiling's slices, so keeping the `actions := +// []string{"read"}` used in the declaration and writing "delete" into it after +// boot changed the ceiling every later request is checked against, and the +// contract this process publishes with it. +// +// The previous round froze the ASK. This is the other operand, and a ceiling a +// caller can still edit is not a ceiling. +func TestTheDeclaredCeilingIsFrozenAtDeclaration(t *testing.T) { + gw := newWorkContextGateway(t, &workContextGateway{}) + mint := newHostMint(t, &hostMint{}) + tokenFile := filepath.Join(t.TempDir(), "token") + writeFile(t, tokenFile, "projected") + + // The slices the caller keeps, on BOTH sides of the comparison. + ceilingActions := []string{"read"} + askedActions := []string{"read"} + module := passthroughModule() + module.Methods = nil + module.Scopes = []Scope{{ResourceKind: "things", Actions: askedActions}} + + server := New(Manifest{ID: testSolutionID}).Consumes(module). + Credential(mintClientFor(t, mint, tokenFile)). + Contract(ModuleContract{Ceilings: map[string]map[string][]Scope{ + localProfile: {"things": {{ResourceKind: "things", Actions: ceilingActions}}}, + }}) + server.cfg = config{gatewayURL: gw.URL, profile: localProfile, apiConsumes: consumesThings} + server.principal = testPrincipal + contract, err := server.resolveContract() + if err != nil { + t.Fatalf("resolve the published contract: %v", err) + } + server.contract, server.contractResolved = contract, true + + // After boot, after resolution: the caller writes through its own slices. + ceilingActions[0] = "delete" + askedActions[0] = "delete" + + // What this process PUBLISHES must still be the declaration. + published, err := json.Marshal(server.contract) + if err != nil { + t.Fatalf("render the published contract: %v", err) + } + if strings.Contains(string(published), "delete") { + t.Errorf("the published contract names an action the declaration never contained: %s", published) + } + + // And what it ENFORCES must still be the declaration. + header := http.Header{} + header.Set("authorization", viewerBearer()) + header.Set(orgHeader, viewerOrg) + header.Set(sessionHeader, viewerSession) + header.Set(workcontext.InstallationIDHeaderName, testInstallation) + gateway := server.gatewayFor(header) + + if _, err := gateway.ForModule(context.Background(), "things", + Scope{ResourceKind: "things", Actions: []string{"delete"}}); err == nil { + t.Error("an ask for delete was accepted: the ceiling was edited after boot through the caller's own slice, so the published contract and the authority this process mints no longer agree with each other or with what a reviewer approved") + } + select { + case got := <-gw.mints: + t.Errorf("a mint reached the issuer for %+v after the ceiling was edited: a refusal that happens after the ask is on the wire is not a ceiling", got.AuthorityScopes) + default: + } + + // The control: the declaration's own ask still works, so this froze the + // values rather than breaking the comparison. + if _, err := gateway.ForModule(context.Background(), "things", + Scope{ResourceKind: "things", Actions: []string{"read"}}); err != nil { + t.Errorf("the declared ask was refused after the freeze, which would make every contract unusable: %v", err) + } +} diff --git a/credential.go b/credential.go new file mode 100644 index 0000000..3e793f2 --- /dev/null +++ b/credential.go @@ -0,0 +1,701 @@ +package solution + +import ( + "context" + "crypto/tls" + "crypto/x509" + "errors" + "fmt" + "log" + "net/http" + "sync" + "time" + + codefly "github.com/codefly-dev/sdk-go" + "github.com/codefly-dev/sdk-go/workcontext" +) + +// CredentialSource is where this execution's credential comes from. +// +// It is the SDK mint client's own method, so *workcontext.MintClient satisfies +// it directly and this runtime holds no mint of its own: obtaining a credential +// from a projected service-account token, sealing it to a build incarnation and +// an installation, re-reading the rotated projection and rechecking the +// boot-read authority before each renewal are all the SDK's, and this runtime +// decides only when a credential is needed and what happens when there is none. +// +// Every use calls it. The client mints the first credential, hands back the one +// it holds while that one is current, and renews it once it has entered its +// renewal lead — so there is no path through this runtime on which a lapsed +// credential is presented, and no timer here that is not the credential's own +// expiry. +type CredentialSource interface { + Credential(ctx context.Context) (workcontext.Credential, error) +} + +// Credential sets where this execution's credential comes from, replacing the +// platform's mint endpoint. A test, or a consumer whose issuer is reached +// another way, supplies its own. Chainable. +func (s *Server) Credential(source CredentialSource) *Server { + s.credential = source + return s +} + +// AuthorityGroup is the workspace configuration group through which the +// platform tells this workload the authority-bearing values it runs under: the +// principal it is, the audience it mints against, and the audience its own +// projected token was issued for. +// +// They are read once, at boot, through the SDK's authority reader, and frozen: +// the credential this process holds is sealed to the values it was minted +// under, so a value that resolves differently later is an error and never a +// reload. The reader rechecks them before every renewal, which is the one +// moment a drifted value would otherwise be laundered into a new credential +// sealed to something nobody approved. +const AuthorityGroup = "module-authority" + +// The keys of AuthorityGroup. A missing one fails the boot naming +// "module-authority/", because the fix is to provision that value and a +// message that did not name it sends whoever reads it looking through all three. +const ( + // AuthorityPrincipalKey is the principal this workload runs as — what the + // published contract names, and what the delivered authority document + // grants bindings to. + AuthorityPrincipalKey = "PRINCIPAL" + // AuthorityAudienceKey is the Work Context audience this workload mints + // against, as the host names it. + AuthorityAudienceKey = "AUDIENCE" + // AuthorityProjectionAudienceKey is the audience the projected + // service-account token was itself minted for. It is sent with the mint so + // the host can refuse a projection aimed at something else, rather than + // reviewing whatever it is handed. + AuthorityProjectionAudienceKey = "PROJECTION_AUDIENCE" +) + +// readAuthority freezes the authority-bearing values this process runs under. +// The SDK names the missing one in its error, which is the whole reason this +// goes through it rather than three workspace reads: "authority-bearing value +// is not configured: module-authority/AUDIENCE" is actionable where "could not +// mint" is not. +func readAuthority(ctx context.Context) (*codefly.Authority, error) { + authority, err := codefly.ReadAuthority(ctx, + codefly.AuthorityValueName{Name: AuthorityGroup, Key: AuthorityPrincipalKey}, + codefly.AuthorityValueName{Name: AuthorityGroup, Key: AuthorityAudienceKey}, + codefly.AuthorityValueName{Name: AuthorityGroup, Key: AuthorityProjectionAudienceKey}, + ) + if err != nil { + return nil, fmt.Errorf("%w — provision the workspace configuration group %q and declare it as a workspace-configuration dependency of this backend", err, AuthorityGroup) + } + return authority, nil +} + +// platformCredentialSource is the default source: the SDK's mint client, +// pointed at the host's mint endpoint, presenting the service-account token the +// platform projects for this workload. +// +// It carries this runtime's outbound client, so the mint presents this +// workload's own X.509-SVID, verifies the host against the anchor the platform +// projected, is not proxied, and does not follow redirects: the request carries +// the projected token that attests which workload this process is, and a +// Location would hand that to whatever answered at the mint URL. +func (s *Server) platformCredentialSource() (CredentialSource, error) { + projectionAudience, err := s.authority.Value(AuthorityGroup, AuthorityProjectionAudienceKey) + if err != nil { + return nil, err + } + identity, err := s.serverIdentity() + if err != nil { + return nil, err + } + // ONE long-lived reloader for the leaf, built here and closed over. + // + // The thing that keeps a rotated leaf presented is the reloader's own + // re-read, not a fresh reloader: the SDK's swallows a half-written pair + // and keeps the last good one, and a constructor per handshake would + // report that error instead of surviving it. outboundLeaf says the same + // about the dial path for the same reason. + leaf, err := s.outboundLeaf(identity) + if err != nil { + return nil, err + } + // HELD TO THE FROZEN PRINCIPAL, at the moment it is presented. + // + // outboundLeaf is the raw reader — on the default path, the SDK's reloader + // over the projected files — and handing it to the SDK unwrapped was the + // identity binding this hop lost. The gateway's configuration gets the + // same hold through holdPresentedCertificate; this is the other outbound + // reader, and it is the one that carries the projected service-account + // token. A same-CA rotation to a neighbouring workload's certificate was + // presented to the mint while the frozen principal was still this one, and + // the SDK cannot catch it: it checks that a certificate is non-empty and + // has a key, which is the right division of labour and not a statement + // about whose certificate it is. + held := heldToTheFrozenPrincipal(leaf, s.principal, "the credential mint") + // Refused at boot as well, for the reason holdPresentedCertificate + // records: a mismatch found at the first dial is a mismatch found after + // the projected token has gone out under it. Only a MISMATCH refuses here + // — a read error is the reloader's business and it keeps the last good + // pair, so failing the boot on one would refuse a rotation that is merely + // half-written. + if pair, readErr := leaf(&tls.CertificateRequestInfo{}); readErr == nil && pair != nil { + if err := pairPresentsThisWorkload(pair, s.principal); err != nil { + return nil, fmt.Errorf("this workload would present a certificate the host has not approved it under to the credential mint: %w", err) + } + } + client, err := workcontext.NewMintClient(s.mintOptions(projectionAudience, s.peerTrustAnchor(identity), held)) + if err != nil { + return nil, fmt.Errorf("configure this workload's credential mint at %s: %w", s.cfg.mintURL, err) + } + return client, nil +} + +// heldToTheFrozenAuthority rechecks the authority whenever a source hands back +// a credential that is not the one it handed back last. +// +// A changed token is a renewal, which is the moment that matters: the host is +// the authority for the principal, the mint audience and the projection +// audience, and a value that drifted after boot must not be re-sealed into a +// new credential. Checking on the *change* rather than on every call keeps this +// a comparison on the hot path. +// +// A drifted authority is terminal, and reported as a refusal rather than as an +// unavailable mint: nothing about it gets better by asking again. +func heldToTheFrozenAuthority(source CredentialSource, authority *codefly.Authority) CredentialSource { + if authority == nil { + // No frozen authority to recheck against. Every booted path has one — + // openAuthority runs before this — so this is the unit-test shape, and + // wrapping it would only hide the absence. + return source + } + return &authorityHeldSource{inner: source, authority: authority} +} + +type authorityHeldSource struct { + inner CredentialSource + authority *codefly.Authority + // mu guards sealed, what this execution's credential was FIRST sealed to. + // It is frozen for the same reason the authority-bearing values are: every + // viewer mint this process runs is attested by that credential, and a + // renewal that quietly moved to another deployment or another build would + // attribute those mints to an execution that never asked for them. + mu sync.Mutex + sealed sealedIdentity +} + +func (a *authorityHeldSource) Credential(ctx context.Context) (workcontext.Credential, error) { + // Before the ask, because the ask is what mints. + // + // Rechecking afterwards discovers a withdrawn authority one renewal too + // late: the mint has already happened, sealed to values nobody approved, + // and the first caller is served with it. The SDK does this ahead of its + // own renewals for the platform source; a supplied source has no such + // hook, which is the whole reason this wrapper exists. + if err := a.authority.Recheck(ctx); err != nil { + return workcontext.Credential{}, fmt.Errorf("%w: an authority-bearing value has drifted from the one this process froze at boot, so nothing further is minted under it: %w", + workcontext.ErrMintRefused, err) + } + credential, err := a.inner.Credential(ctx) + if err != nil { + return credential, err + } + // Before the renewal comparison, because that comparison keys on the + // token *changing* and a token that is always "" never changes — so the + // zero credential walked past this wrapper every time. + if err := usableCredential(credential); err != nil { + return workcontext.Credential{}, err + } + // And again after, unconditionally — for drift that landed during the ask + // itself, including the FIRST credential and an ask that returned the one + // already held. This was once gated on the token having changed, which + // meant a drift arriving mid-ask went unnoticed until some later renewal + // happened to produce a different token. Removing the gate left the + // comparison, the token it remembered and the mutex guarding it all + // computed and discarded. + if err := a.authority.Recheck(ctx); err != nil { + return workcontext.Credential{}, fmt.Errorf("%w: this execution's credential was renewed while an authority-bearing value had drifted from the one this process froze at boot: %w", + workcontext.ErrMintRefused, err) + } + // And the installation the credential is sealed to, frozen at the first one + // and held there. + // + // usableCredential already refuses a credential sealed to no installation. + // This refuses one sealed to a *different* installation than the credential + // this process has been acting under — which is not a renewal but a + // different identity arriving through the renewal path. Every viewer mint + // names an installation; moving silently would make the mints before and + // after a renewal name different deployments with nothing recording that + // they did. + // + // Terminal, not transient: the host has answered about a different + // installation than the one this build was approved for, and no number of + // retries changes that. + if err := a.holdSealedIdentity(credential.Seal()); err != nil { + return workcontext.Credential{}, err + } + return credential, nil +} + +// holdSealedIdentity freezes what this execution's credential is sealed to — +// the installation AND the execution pair — and refuses a renewal sealed to +// anything else. +// +// usableCredential refuses a credential sealed to no installation and one +// sealed to no execution. This is the other axis: a renewal that is +// well-formed, carries both, and carries DIFFERENT ones. That is not a renewal +// of this execution's credential, it is a different execution's, and the +// executed round found both halves accepted silently — an installation move +// was caught and a build move from incarnation 11 to 99 was not. +// +// It matters for the same reason the installation does. Every work context this +// process mints for a viewer is attested by this credential, so the mints +// before and after a silent change would be attributed to different builds, +// and the host's ceiling for the build it approved would have been applied to +// one it did not. A process whose approved build has moved under it is a +// process that should end and be restarted against the delivery as it now +// stands, which is where that gets resolved. +// +// Read through the SDK's own accessor on the verified seal. No decoder here. +func (a *authorityHeldSource) holdSealedIdentity(seal *workcontext.SealedValues) error { + held := sealedIdentity{ + installation: seal.GetInstallationId(), + digest: seal.GetImageDigest(), + incarnation: seal.GetBuildIncarnation(), + } + a.mu.Lock() + defer a.mu.Unlock() + if a.sealed == (sealedIdentity{}) { + a.sealed = held + return nil + } + if a.sealed == held { + return nil + } + return fmt.Errorf("%w: this execution has been acting under installation %s, image %s, build incarnation %d, and its credential renewed sealed to installation %s, image %s, build incarnation %d. That is a different execution, not a renewal: every work context this process mints for a viewer is attested by this credential, so the mints before and after this point would name a different deployment or a different build — including one the host's approved-build ceiling was never applied to", + workcontext.ErrMintRefused, + a.sealed.installation, a.sealed.digest, a.sealed.incarnation, + held.installation, held.digest, held.incarnation) +} + +// sealedIdentity is what a credential is sealed to: which deployment, and +// which build inside it. Compared as one value, because a renewal that changed +// any part of it changed which execution is asking. +type sealedIdentity struct { + installation string + digest string + incarnation uint64 +} + +// mintOptions is how this runtime asks the SDK's mint client for a credential. +// +// A function of its own so it can be read in a test. Every field here is a +// decision, and the one most easily lost is Authority: the SDK rechecks it +// before every renewal, which is the single moment a drifted authority value +// would otherwise be laundered into a credential nobody approved. Dropping it +// leaves the boot and the first mint working perfectly and only renewals wrong, +// which is not a shape a test over the happy path can see. +func (s *Server) mintOptions( + projectionAudience string, + anchor func() (*x509.CertPool, error), + leaf func(*tls.CertificateRequestInfo) (*tls.Certificate, error), +) workcontext.MintOptions { + return workcontext.MintOptions{ + URL: s.cfg.mintURL, + // The pinned value the audience is READ from, not an audience this + // runtime resolved and passed beside the pin. The SDK made that a + // typed reference for a reason worth keeping in mind here: a string + // passed alongside the pin meant the drift check guarded a value the + // mint did not use. + Audience: workcontext.AuthorityValue{Name: AuthorityGroup, Key: AuthorityAudienceKey}, + ProjectedToken: workcontext.ProjectedTokenFile(s.cfg.projectedTokenPath), + ProjectionAudience: projectionAudience, + // The frozen reader, not a value read again here: it is what rechecks + // the authority-bearing values before each mint, including the first. + Authority: s.authority, + // The transport's three judgements, as READERS rather than values. + // + // This is sdk-go#51, and it is the shape this runtime argued for from + // both ends. The SDK still builds and owns the transport — a supplied + // *http.Client is a hole it cannot inspect, since a nil Transport + // means the global mutable default, a DialTLSContext bypasses + // TLSClientConfig entirely, and a caller holding the same transport + // can turn verification off after construction — but it no longer + // takes a pool by value and fixes it for the client's lifetime. + // + // Every one of these is called DURING the handshake, so withdrawal + // takes effect on the next handshake rather than on the next process: + // + // - TrustAnchor is the same reader the listener and the gateway dial + // use, read before and after each handshake and never cached, so a + // removed root stops this process dialling the mint. + // - ClientCertificate makes the mint request present this workload's + // X.509-SVID again. The projected service-account token is no + // longer the whole of what attests who is asking. + // - AdmittedPeers is MINT_PEERS_FILE, read per handshake, so the + // mint endpoint is held to WHICH party it is and not merely to a + // certificate from the right CA. Every workload in the trust + // domain holds one of those, which is why authentication was never + // authorisation here. + TrustAnchor: anchor, + // Asked with an empty request rather than nil: a consumer's own + // GetClientCertificate may read the acceptable CAs off it, and handing + // it nil would be this runtime choosing a dereference on somebody + // else's code path. + ClientCertificate: func() (*tls.Certificate, error) { return leaf(&tls.CertificateRequestInfo{}) }, + AdmittedPeers: s.admittedMint(), + } +} + +// openCredential obtains this execution's credential, once, before the listener +// exists. +// +// The two outcomes the SDK distinguishes are not the same outcome here, and +// conflating them was the first thing the host asked me to change: +// +// - A **refusal** is the host judging this workload: this build is not the one +// its presence document approved, the pod's identity does not match, the +// token attests to another subject. No number of attempts changes any of +// those answers, so the boot reports what the issuer said and returns — +// which is a non-zero exit for whoever called Serve, and a redeploy or a +// corrected delivery for whoever reads it. It is never retried: the old +// runtime answered every refusal by beating again, minting a fresh +// single-use token each time, so one undeployable solution produced an +// audited mint every 15 seconds for as long as it ran. +// +// - An **unavailable** mint is not a judgement at all. A workload may +// legitimately start before its presence generation has been applied, and +// then there is no incarnation to mint against yet; the issuer may also be +// unable to reach the Kubernetes API or its own policy log, and a host that +// issues nothing in those cases is a host behaving correctly. Each is +// transient by construction, so the boot waits and asks again, within a +// bound. +// +// The bound is what keeps this from being the heartbeat again. It is a boot +// waiting for a dependency, not a process announcing itself: there is no steady +// state in which it runs, nothing is minted on success beyond the one +// credential, and when the bound is spent the boot fails rather than settling +// into a loop. A process that cannot obtain a credential inside it has an +// orchestrator whose job is to restart it, and a restart is cheaper to read +// than a solution that serves nothing while logging forever. +func (s *Server) openCredential(ctx context.Context) error { + if s.credential == nil { + source, err := s.platformCredentialSource() + if err != nil { + return err + } + s.credential = source + } + // Whatever the source is, the authority recheck is this runtime's promise + // and not the source's. + // + // A supplied source replaces the mint client, and with it every option + // mintOptions carries — including Authority, which the SDK rechecks before + // each renewal. So the one guarantee that exists to stop a drifted + // authority value being laundered into a differently-sealed credential + // applied to the platform path only, while this package's own + // documentation promises it of the process. Wrapping here makes the promise + // true of every source, and it costs the platform source nothing: the SDK + // does it too, and a recheck of three frozen values is a comparison. + s.credential = heldToTheFrozenAuthority(s.credential, s.authority) + window := s.firstMintWindow + if window == 0 { + window = firstMintWait + } + // The window is a deadline on the whole operation, not a stopwatch read + // between attempts. It used to be the latter: elapsed time was checked only + // after an unavailable answer, so an attempt could start at the deadline + // and a success after it was accepted — and a source that blocks until its + // context is cancelled was never cancelled at all, so a boot could wait + // forever inside one attempt. + deadline, cancel := context.WithDeadline(ctx, time.Now().Add(window)) + defer cancel() + // The first wait is a quarter of the window at most, so a window shorter + // than the backoff still gets more than one attempt. At the real window + // (two minutes) this is just firstMintRetry; it matters for a caller that + // sets a short one, where a fixed first backoff would spend the whole + // window waiting after a single ask. + wait := firstMintRetry + if quarter := window / 4; quarter < wait { + wait = quarter + } + for attempt := 1; ; attempt++ { + credential, err := s.credential.Credential(deadline) + switch { + case err == nil && usableCredential(credential) != nil: + // A source may hand back the zero credential or an expired one, + // and this boot used to accept either: it logged a credential + // sealed to installation "" and served. The classification is + // usableCredential's — an empty token is a judgement, an expired + // one is not — so an unavailable answer here keeps waiting inside + // the window and a refused one fails the boot. + err = usableCredential(credential) + if !errors.Is(err, workcontext.ErrMintUnavailable) { + return fmt.Errorf("obtain this execution's credential from %s: %w", s.cfg.mintURL, err) + } + case err == nil && deadline.Err() == nil: + // Into the controller, which is the half that was missing: the + // boot obtained the credential and then nothing held it, so the + // first route after boot saw an empty controller, asked again, and + // got none of the backoff or held-credential behaviour the + // controller exists to provide. "One credential per execution" + // starts at the boot's own credential or it starts one request + // late. + s.holdCredential(credential) + // Through the generated GETTERS, not the fields. + // + // SealedValues is core's protobuf type now (an alias for + // basev0.WorkSealV1), so its scalars are POINTERS: formatting + // seal.BuildIncarnation with %d printed a pointer address into the + // boot log instead of the incarnation — the one line an operator + // reads to see which approved build this credential is sealed to. + // The getters return values and answer zero for a nil seal. + // + // golangci-lint's govet caught this and plain `go vet ./...` did + // not, which is worth knowing: the four gate commands would have + // shipped it. + seal := credential.Seal() + log.Printf("solution %q: holding one execution credential, sealed to installation %s revision %d and build incarnation %d, valid until %s", + s.manifest.ID, seal.GetInstallationId(), seal.GetInstallationRevision(), seal.GetBuildIncarnation(), + credential.ExpiresAt().UTC().Format(time.RFC3339)) + return nil + case err == nil: + // A credential that arrived after the window closed is not a + // credential this boot may act on: the deadline is what bounds the + // wait, and accepting a late success would make it advisory. + return fmt.Errorf("obtain this execution's credential from %s: a credential arrived after the %s window closed, so this boot does not hold one", + s.cfg.mintURL, window) + case !errors.Is(err, workcontext.ErrMintUnavailable): + // A judgement, or a configuration this client will not even send. + return fmt.Errorf("obtain this execution's credential from %s: %w", s.cfg.mintURL, err) + case deadline.Err() != nil: + return fmt.Errorf("obtain this execution's credential from %s: still unavailable after %s and %d attempt(s): %w — the issuer answers this way while a presence generation has not been applied yet, or while it cannot reach the Kubernetes API or its policy log; all three are transient, so this is an exit for the orchestrator to retry rather than a judgement on this build", + s.cfg.mintURL, window, attempt, err) + } + if remaining := time.Until(mustDeadline(deadline)); wait > remaining { + wait = remaining + } + log.Printf("solution %q: this execution's credential is not available yet (attempt %d, retrying in %s): %v", + s.manifest.ID, attempt, wait, err) + select { + case <-deadline.Done(): + return fmt.Errorf("obtain this execution's credential from %s: still unavailable after %s and %d attempt(s): %w — the issuer answers this way while a presence generation has not been applied yet, or while it cannot reach the Kubernetes API or its policy log; all three are transient, so this is an exit for the orchestrator to retry rather than a judgement on this build", + s.cfg.mintURL, window, attempt, err) + case <-time.After(wait): + } + if wait *= 2; wait > firstMintRetryCap { + wait = firstMintRetryCap + } + } +} + +// mustDeadline is the deadline of a context that has one. openCredential sets +// it, so the fallback is only there to keep the arithmetic total. +func mustDeadline(ctx context.Context) time.Time { + if at, ok := ctx.Deadline(); ok { + return at + } + return time.Now() +} + +// firstMintWait bounds how long a boot waits for a credential the issuer says +// is not available yet, and firstMintRetry/firstMintRetryCap pace the asks +// inside it. +// +// The bound is deliberately short. What it covers is a workload that started +// before its own presence generation was applied, which is a reconcile away, +// not an outage to ride out — and anything longer is a process holding a pod +// slot while serving nothing, which an orchestrator reads less clearly than an +// exit. +const ( + firstMintWait = 2 * time.Minute + firstMintRetry = 2 * time.Second + firstMintRetryCap = 15 * time.Second +) + +// openAuthority freezes the authority-bearing values this process runs under +// and reports the principal it runs as. +// +// It happens before the contract is resolved and before anything is minted, +// because the principal is part of what the contract publishes and because a +// value the platform never provisioned is a refusal that should not cost a +// round trip to discover. +// +// It happens on **every** path, including a boot that supplied its own +// credential source. A hook changes where a credential comes from; it does not +// change who this workload is. Skipping this for a custom source left the +// published contract naming no principal and the listener held to nothing — +// so the two strongest properties in the boot were off for exactly the +// consumers who had written their own integration. +func (s *Server) openAuthority(ctx context.Context) error { + authority, err := readAuthority(ctx) + if err != nil { + return err + } + s.authority = authority + s.principal, err = authority.Value(AuthorityGroup, AuthorityPrincipalKey) + return err +} + +// attestWorkload presents this execution's credential on a request this runtime +// makes on its own behalf, and refuses the request when it cannot. +// +// It is what makes the credential more than a boot formality: the mint this +// runtime runs for a viewer is an operation performed by *this module*, and +// without the workload's own credential on it the issuer knows only that +// somebody holding a viewer's bearer asked. With it, the issuer can hold the +// mint to the installation and the binding the credential is sealed to — which +// is where that check belongs, since nothing a process reports about itself can +// be trusted by the thing deciding what it may do. +// +// It used to log and carry on. That was fail-open, and it was wrong in the one +// case the attestation exists for: a workload whose renewal the issuer has +// started refusing — because the installation moved, the build is no longer +// approved, the principal's epoch advanced — kept minting viewer capabilities +// with no execution binding on them at all. "The host does not require the +// attestation yet" is a statement about the counterpart's current state, which +// is exactly the kind of concession that leaves a hole open for as long as the +// counterpart takes; and boot-time approval is not authorization at use, which +// is the whole premise of sealing a credential to an execution rather than +// trusting a process that started successfully. +// +// So a credential this runtime cannot present stops the request here. The +// viewer sees a refusal naming this solution's own authority rather than their +// own — the handler maps it to a 503 through relayedError, since the condition +// is this process's and one renewal fixes it — and no capability is minted +// under an attribution nobody can check. +// credentialAcquirer obtains this execution's credential through the one +// controller that governs every acquisition. +// +// A function rather than the CredentialSource itself, because this path used to +// hold the source and call it directly — so the single-flight, the backoff and +// the held-credential fallback that every other caller goes through did not +// apply to the viewer's mint, which is the hottest path there is. One +// controller means one controller. +type credentialAcquirer func(context.Context) (workcontext.Credential, error) + +// attestWorkloadReporting is attestWorkload with somewhere to report a +// *terminal* failure, which is the half that is easy to get backwards. +// +// The boot already separates the issuer's two answers and must: a refusal is a +// judgement on this build — the installation moved, the build is not the one +// the presence document approved, the principal's epoch advanced — and no +// number of attempts changes it, while an unavailable mint is a transient +// condition worth waiting out. At *renewal* both were collapsed into +// ErrNotAttested, which relayedError maps to unavailable with the note that one +// renewal fixes it. For a refusal that is false: nothing fixes it, so the +// process answered 503 to every request, forever, while reporting itself +// healthy — the one shape this cutover was supposed to end, because a solution +// that serves nothing while looking alive is exactly what the delivered-presence +// model replaced the heartbeat to avoid. +// +// A terminal refusal therefore ends the process. Health starts failing so the +// host's own probe takes the binding out, and serve returns non-zero so the +// orchestrator restarts it against the delivery as it now stands — which is +// where a build the host no longer approves gets resolved, and it is not here. +func attestWorkloadReporting(ctx context.Context, acquire credentialAcquirer, report *attestationReport, request *http.Request, id string, terminal func(error)) error { + if acquire == nil { + // No source at all is a programming error on a path that mints: every + // boot opens one, and a consumer that supplied its own supplied a + // source. Refused rather than treated as "nothing to attest with". + // (There were two of these branches, identical, one guarded by a + // condition that made the second unreachable.) + return fmt.Errorf("%w: this solution holds no credential source, so it cannot attest which module is asking", ErrNotAttested) + } + credential, err := acquire(ctx) + if err != nil { + report.say(id, "refusing to mint for a viewer: this execution's credential could not be obtained: "+err.Error()) + if terminal != nil && terminalCredentialFailure(err) { + terminal(err) + } + return fmt.Errorf("%w: %w", ErrNotAttested, err) + } + // Defence in depth, and unreachable by design since round seven: a + // credential reaching here has passed usableCredential, which requires a + // non-empty token and a non-empty seal — and a credential with those came + // from a mint, so its token parses and Attach succeeds. No test can + // distinguish this branch any more, which is why no test claims to; it is + // kept because it does the right thing if the invariant above ever stops + // holding, unlike the ErrRevoked branch this package deleted for doing the + // wrong thing while dead. + if err := credential.Attach(request); err != nil { + report.say(id, "refusing to mint for a viewer: this execution's credential could not be presented: "+err.Error()) + return fmt.Errorf("%w: %w", ErrNotAttested, err) + } + report.recovered(id) + return nil +} + +// ErrNotAttested is what a mint refused for want of this workload's own +// credential reports. It is this solution's condition and not the viewer's, and +// one renewal fixes it, which is what decides the status a handler answers. +var ErrNotAttested = errors.New("this solution could not attest which module is asking") + +// attestationReport throttles what attestWorkload says. One per server, not a +// package value, so one solution's failing renewal cannot silence another's in +// a process that runs two. +// +// It still throttles even though a failure now refuses the request: a renewal +// that is failing fails on every request, so a line per call would bury the +// first occurrence under the rest. What a caller gets is not throttled — every +// refused request is refused. +type attestationReport struct { + mu sync.Mutex + last string +} + +func (r *attestationReport) say(id, what string) { + if r == nil { + log.Printf("solution %q: %s", id, what) + return + } + r.mu.Lock() + repeat := r.last == what + r.last = what + r.mu.Unlock() + if !repeat { + log.Printf("solution %q: %s", id, what) + } +} + +// recovered closes an episode, so the next failure is reported even if it is +// the same one. A renewal that recovers and fails again is two occurrences. +func (r *attestationReport) recovered(id string) { + if r == nil { + return + } + r.mu.Lock() + had := r.last + r.last = "" + r.mu.Unlock() + if had != "" { + log.Printf("solution %q: presenting this workload's credential again", id) + } +} + +// terminalCredentialFailure reports whether the issuer's answer is a judgement +// no retry can change. +// +// ErrMintRefused only, and the correction is worth recording because the +// it is easy to get from the wrong direction. It also treated ErrRevoked +// as terminal here, on the reasoning that live state moving under a sound +// credential is as final as a refused build. Two things are wrong with that: +// the mint client does not return ErrRevoked at all (it comes back from a +// *callee* that was shown a capability sealed to state that has moved), and the +// SDK's own contract for it is a refresh and one retry — so treating it as +// terminal would have ended the process on a condition the client is built to +// recover from, if the path had ever been reachable. It was not, which is why +// no test noticed; a dead branch that would do the wrong thing if it ever woke +// up is worse than no branch. +// +// A callee's ErrRevoked arrives on a viewer's own request and is answered +// there, as a 409 to that viewer. Nothing is inferred from it about the +// capability or about this execution's credential: a 409 means "the state you +// were sealed to has moved" and it means "that page already exists", and +// nothing on the wire says which (see the RoundTrip comment in solution.go). +// +// Everything else, ErrMintUnavailable above all, is transient by construction +// and must not end the process: an issuer that cannot reach its own policy log +// is an issuer behaving correctly, and exiting on it would turn a dependency +// blip into a crash loop. +func terminalCredentialFailure(err error) bool { + return errors.Is(err, workcontext.ErrMintRefused) +} diff --git a/credential_test.go b/credential_test.go new file mode 100644 index 0000000..50e8d07 --- /dev/null +++ b/credential_test.go @@ -0,0 +1,2743 @@ +package solution + +import ( + "connectrpc.com/connect" + "context" + "crypto/tls" + "crypto/x509" + "errors" + "fmt" + "log" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/codefly-dev/core/solution/manifest" + corework "github.com/codefly-dev/core/workcontext" + codefly "github.com/codefly-dev/sdk-go" + "github.com/codefly-dev/sdk-go/workcontext" +) + +// TestARefusedMintFailsTheBootAndIsNeverRetried is the model's sharpest edge: a +// host that has judged this workload is not a condition to retry. The old +// runtime answered every refusal by beating again, minting a fresh single-use +// token each time, so one undeployable solution produced an audited mint every +// 15 seconds for as long as it ran. This boot reports the refusal and returns, +// which is a non-zero exit for the process that called Serve. +func TestARefusedMintFailsTheBootAndIsNeverRetried(t *testing.T) { + for _, tc := range []struct { + name string + status int + }{ + {"the host refuses this workload's build or identity", http.StatusForbidden}, + {"the projected token is not acceptable", http.StatusUnauthorized}, + // 404 is NOT here any more, and that is sdk-go v0.3.0 inverting the + // default rather than this test being relaxed. + // + // The rule was "429 and 5xx retry, everything else is terminal", which + // made every status anything between the host and this process might + // invent into a permanent stop — measured: a 408 from a proxy, with a + // valid credential in hand, latched for the life of the process, and a + // 404 from an ingress mid-rollout would do the same. Only 401 and 403 + // latch now, because those are the two the host signs: the projected + // token is not acceptable, or the build is not approved. + // + // An old runtime against a new host still does not recover — it waits + // out its bounded window and exits non-zero for the orchestrator + // instead of failing on the first answer. Covered by the unavailable + // cases, and AGENTS.md says so now. + } { + t.Run(tc.name, func(t *testing.T) { + mint := newHostMint(t, &hostMint{status: tc.status}) + err := bootFails(t, New(Manifest{ID: testSolutionID}), mint) + if !errors.Is(err, workcontext.ErrMintRefused) { + t.Fatalf("boot error = %v, want one wrapping %v so a caller can tell a judgement from an outage", err, workcontext.ErrMintRefused) + } + if !strings.Contains(err.Error(), mint.URL) { + t.Errorf("boot error %q does not name the mint endpoint it asked", err) + } + if got := mint.count(); got != 1 { + t.Errorf("the boot asked the host %d times, want exactly 1: a refusal is terminal", got) + } + }) + } +} + +// TestAnUnavailableMintIsWaitedForWithinABound is the other half, and the half +// the host asked for: a workload may legitimately start before its presence +// generation has been applied, and the issuer answers that — and its own +// unreachable dependencies — with a retryable status. Treating it as terminal +// would make a pod that is one reconcile early fail for a reason that is not +// about it. +func TestAnUnavailableMintIsWaitedForWithinABound(t *testing.T) { + t.Run("it recovers when the issuer catches up", func(t *testing.T) { + mint := newHostMint(t, &hostMint{status: http.StatusServiceUnavailable}) + mint.recoverAfter = 2 + server := New(Manifest{ID: testSolutionID}) + server.firstMintWindow = 20 * time.Second + solution := boot(t, server, mint) + if status := getStatus(t, solution.client, solution.base+HealthPath); status != http.StatusOK { + t.Fatalf("health = %d, want 200: the boot waited out an unapplied presence generation", status) + } + if got := mint.count(); got < 3 { + t.Errorf("the host was asked %d times, want the two refusals plus the one that answered", got) + } + }) + + t.Run("it gives up when the window is spent, and the window is a deadline", func(t *testing.T) { + mint := newHostMint(t, &hostMint{status: http.StatusServiceUnavailable}) + server := New(Manifest{ID: testSolutionID}) + // Long enough for more than one attempt over TLS, short enough that + // the test spends the window rather than waiting out the real two + // minutes. + server.firstMintWindow = 1500 * time.Millisecond + started := time.Now() + // Bounded here, not measured afterwards: against a runtime whose + // window is not a deadline this call never returns, and every + // assertion below is unreachable. + err := bootFailsWithin(t, server, mint, 4*server.firstMintWindow) + elapsed := time.Since(started) + + if !errors.Is(err, workcontext.ErrMintUnavailable) { + t.Fatalf("boot error = %v, want one wrapping %v", err, workcontext.ErrMintUnavailable) + } + if !strings.Contains(err.Error(), "transient") { + t.Errorf("boot error %q does not say the cause is transient, which is what makes the exit the orchestrator's to retry", err) + } + if got := mint.count(); got < 2 { + t.Errorf("the host was asked %d times, want more than one: an unavailable mint is waited for, not judged — boot said: %v", got, err) + } + // The window bounds the whole operation. It used to be read only + // between attempts, so an attempt could start at the deadline and run + // as long as it liked past it. + if elapsed > 4*server.firstMintWindow { + t.Errorf("the boot took %s for a %s window: the window is a deadline on the operation, not a stopwatch read between attempts", elapsed.Round(time.Millisecond), server.firstMintWindow) + } + }) + + t.Run("a source that never answers is cut off by the window", func(t *testing.T) { + // The sharpest case for the deadline: a source that waits for its + // context rather than returning. Without the window on the context it + // is handed, the boot waits in one attempt forever — no listener, no + // log, and nothing for an orchestrator to act on. + mint := newHostMint(t, &hostMint{}) + blocking := &blockingCredentialSource{entered: make(chan struct{}, 1)} + server := New(Manifest{ID: testSolutionID}).Credential(blocking) + server.firstMintWindow = 400 * time.Millisecond + started := time.Now() + err := bootFailsWithin(t, server, mint, 10*server.firstMintWindow) + elapsed := time.Since(started) + if err == nil { + t.Fatal("the boot came up on a source that never answered") + } + if elapsed > 10*server.firstMintWindow { + t.Fatalf("the boot waited %s on a source that never answers, for a %s window", elapsed.Round(time.Millisecond), server.firstMintWindow) + } + select { + case <-blocking.entered: + default: + t.Error("the source was never called, so this test did not exercise the deadline") + } + if !blocking.sawCancellation() { + t.Error("the source's context was never cancelled: the window has to reach the attempt, not just the loop around it") + } + }) +} + +// blockingCredentialSource waits for its context instead of answering: the +// shape of a source talking to something that has stopped responding. +type blockingCredentialSource struct { + entered chan struct{} + mu sync.Mutex + cancelled bool +} + +func (b *blockingCredentialSource) Credential(ctx context.Context) (workcontext.Credential, error) { + select { + case b.entered <- struct{}{}: + default: + } + <-ctx.Done() + b.mu.Lock() + b.cancelled = true + b.mu.Unlock() + return workcontext.Credential{}, fmt.Errorf("%w: %w", workcontext.ErrMintUnavailable, ctx.Err()) +} + +func (b *blockingCredentialSource) sawCancellation() bool { + b.mu.Lock() + defer b.mu.Unlock() + return b.cancelled +} + +// TestTheProjectedTokenIsReadAtEveryMintNotCachedAtBoot pins the rotation +// property, by forcing a renewal and reading what the second mint presented. +// +// It used to stop after the first mint, which proved nothing: a client that +// read the projection once at boot and kept it forever passed. The platform +// rotates that file under a running process, so a runtime holding the boot-time +// copy renews with a token the issuer stopped honouring long before the process +// stopped running — and the refusal reads as "this build is not approved". +func TestTheProjectedTokenIsReadAtEveryMintNotCachedAtBoot(t *testing.T) { + // A short credential so the client's own renewal lead is reached inside a + // test rather than in ten minutes. + mint := newHostMint(t, &hostMint{ttl: 30 * time.Second}) + tokenFile := filepath.Join(t.TempDir(), "token") + writeFile(t, tokenFile, "first-projection") + source := mintClientFor(t, mint, tokenFile) + + first, err := source.Credential(context.Background()) + if err != nil { + t.Fatalf("first mint: %v", err) + } + // Still current: the same credential is handed back and nothing is minted. + if _, err := source.Credential(context.Background()); err != nil { + t.Fatalf("second call: %v", err) + } + if got := mint.count(); got != 1 { + t.Fatalf("the host minted %d times for two calls, want 1: a current credential is reused, never re-minted per call", got) + } + + // The projection rotates, and the credential enters its renewal lead: the + // renewal has to present what is on disk now. + writeFile(t, tokenFile, "rotated-projection") + mint.ttl = time.Hour + waitUntilRenewed(t, source, first.Token()) + + if got := mint.count(); got != 2 { + t.Fatalf("the host minted %d times, want 2: the renewal never happened, so this test proves nothing about rotation", got) + } + if got := mint.presented[0]; got != "Bearer first-projection" { + t.Errorf("the first mint presented %q, want the projection as it was then", got) + } + if got := mint.presented[1]; got != "Bearer rotated-projection" { + t.Errorf("the renewal presented %q, want the rotated projection: a token read once at boot is expired long before the process is", got) + } +} + +// waitUntilRenewed calls the source until it hands back a carrier other than +// the one given, or the renewal plainly is not coming. +// +// The client renews at a fraction of the credential's own lifetime, so the wait +// is on the issuer's clock rather than this test's: polling is what a caller +// does anyway (every use goes through Credential), and the assertion is that a +// renewal happens at all, not when. +func waitUntilRenewed(t *testing.T, source CredentialSource, held string) { + t.Helper() + deadline := time.Now().Add(30 * time.Second) + for time.Now().Before(deadline) { + credential, err := source.Credential(context.Background()) + if err != nil { + t.Fatalf("renewal: %v", err) + } + if credential.Token() != held { + return + } + time.Sleep(50 * time.Millisecond) + } + t.Fatal("the credential was never renewed: its lifetime entered the renewal lead and the client kept handing back the same carrier") +} + +// TestEveryCallerOfOneExecutionSharesOneCarrier is the mint-once assertion that +// a client minting per request would fail. +// +// Comparing what a credential *describes* does not catch that defect: a client +// that mints per request answers the same audience and the same seal every +// time, because the host issues both again. So this compares the credential +// **material** — the carrier itself — across every caller of a concurrent +// burst, and counts what the host was asked. The runtime resolves a credential +// per use (the passthrough calls it on every request, from whatever goroutine +// the server hands it), so the burst is the shape production actually takes. +// +// It matters more than it looks: the saving this whole change buys is one mint +// plus a handful of renewals an hour against the heartbeat's 240. A client that +// silently minted per request would be *worse* than the heartbeat it replaced, +// and every functional test would still pass, because every token it issued +// would be valid. +func TestEveryCallerOfOneExecutionSharesOneCarrier(t *testing.T) { + mint := newHostMint(t, &hostMint{}) + tokenFile := filepath.Join(t.TempDir(), "token") + writeFile(t, tokenFile, "projected") + source := mintClientFor(t, mint, tokenFile) + + const callers = 32 + carriers := make([]string, callers) + seals := make([]*workcontext.SealedValues, callers) + start := make(chan struct{}) + var wg sync.WaitGroup + for i := range callers { + wg.Add(1) + go func() { + defer wg.Done() + <-start + credential, err := source.Credential(context.Background()) + if err != nil { + t.Errorf("caller %d: %v", i, err) + return + } + carriers[i], seals[i] = credential.Token(), credential.Seal() + }() + } + close(start) + wg.Wait() + + for i, carrier := range carriers { + switch { + case carrier == "": + t.Fatalf("caller %d obtained no credential", i) + case carrier != carriers[0]: + // Naming which part moved rather than printing both carriers: two + // different capabilities render alike at a glance, so "X vs X" is + // what a reader would otherwise be left with. + t.Fatalf("caller %d holds a different carrier from caller 0 (same length: %t, same seal: %t) — one execution holds one credential", + i, len(carrier) == len(carriers[0]), seals[i] == seals[0]) + } + } + if got := mint.count(); got != 1 { + t.Errorf("the host was asked %d times by %d concurrent callers, want exactly 1", got, callers) + } + // The client's own accounting, which is the only place a source that + // re-dialled the host for the same bytes would be visible at all. + if client, ok := source.(*workcontext.MintClient); ok { + if counts := client.Counts(); counts.Mints != 1 || counts.Renewals != 0 { + t.Errorf("the client reports %d mints and %d renewals, want 1 and 0", counts.Mints, counts.Renewals) + } + } +} + +// TestABootedRuntimeRegistersNothingWithEveryLegacyKeySet is the cutover +// evidence the earlier version of this test did not provide. +// +// That version called the SDK's mint client directly and asserted no legacy +// header was on the request — which would have stayed green with the whole +// registration path still in the runtime. This one boots the real runtime with +// every deleted configuration key set to a value a host would once have acted +// on, and asserts what the host received: one request, to the mint, and +// nothing resembling a registration. +func TestABootedRuntimeRegistersNothingWithEveryLegacyKeySet(t *testing.T) { + mint := newHostMint(t, &hostMint{}) + + // A listener of its own for every deleted URL, which boot cannot move. + // + // These pointed at mint.URL, captured before boot. The boot then calls + // hostMint.serveTLS, which CLOSES that server and starts another on a new + // port — so every deleted URL named an address nothing was listening on, + // and a surviving registration would have failed to connect and been + // counted by nobody. The test passed because nothing could be observed. + // + // This is the second time this test watched the wrong thing. The first + // version aimed the URLs at example.com, which a reviewer caught for the + // same reason: aimed somewhere nothing answers, it proved only that the + // old code never minted. The comment then said "every deleted URL points + // at the host this test counts requests on", which was true when it was + // written and stopped being true when serveTLS started replacing the + // server underneath it. + var legacyHits atomic.Int64 + var legacyPaths sync.Map + legacy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + legacyHits.Add(1) + legacyPaths.Store(r.URL.Path, r.Header.Clone()) + w.WriteHeader(http.StatusOK) + })) + t.Cleanup(legacy.Close) + + // Every key the cutover deleted. If any of them still reaches a code path, + // the observer above sees a request — or a header it should never be shown + // again. + for key, value := range map[string]string{ + "PUBLIC_URL": "https://public.example.com", + "SELF_UPSTREAM": "https://self.example.com", + "HOST_REGISTER_URL": legacy.URL + "/api/solutions/register", + "GATEWAY_REGISTER_URL": legacy.URL + "/solutions/_register", + "GATEWAY_MODULE_REGISTER_URL": legacy.URL + "/modules/_register", + "GATEWAY_MODULE_REGISTRATION_TOKEN_URL": legacy.URL + "/modules/_registration-token", + "GATEWAY_SOLUTION_REGISTRATION_TOKEN_URL": legacy.URL + "/solutions/_registration-token", + "CODEFLY_INTERNAL_TOKEN": "internal-token-that-must-not-travel", + "CODEFLY__SOLUTION_REGISTRATION_SECRET": "solution-secret-that-must-not-travel", + "CODEFLY__MODULE_REGISTRATION_SECRETS": "things:module-secret-that-must-not-travel", + "CODEFLY__SOLUTION_REGISTRATION_INTERVAL": "1s", + "CODEFLY_HOST_FRONTEND": "frontend", + } { + t.Setenv(key, value) + } + + // reached is what the observer has seen since the last reset, with the + // paths, so every assertion below can name what called a deleted endpoint + // rather than only that something did. + reached := func() (int64, []string) { + paths := []string{} + legacyPaths.Range(func(k, _ any) bool { paths = append(paths, k.(string)); return true }) + return legacyHits.Load(), paths + } + reset := func() { + legacyHits.Store(0) + legacyPaths.Range(func(k, _ any) bool { legacyPaths.Delete(k); return true }) + } + // listening proves the observer can still see a request AT THE MOMENT it + // is asked, and leaves the counters clean for the next window. + // + // The defect this replaced was a test concluding "nothing called it" while + // watching a closed port, so every zero below has to rest on something + // checked rather than assumed — and checked at both ends of the window it + // certifies, because the first version of this check was true when it was + // written and stopped being true when the boot moved the server. + listening := func(when string) { + t.Helper() + reset() + probe, err := http.Get(legacy.URL + "/api/solutions/register") + if err != nil { + t.Fatalf("the observer for the deleted endpoints is not reachable %s, so a zero count around it would prove nothing: %v", when, err) + } + drainAndClose(probe) + if got, _ := reached(); got != 1 { + t.Fatalf("the observer counted %d of its own probe %s, want 1: it is not counting what reaches it", got, when) + } + reset() + } + + // THE BOOT WINDOW, and it has to come first. + // + // The observer is proved reachable BEFORE the boot and the count is + // asserted BEFORE anything resets it. Neither was true: the helpers were + // defined after boot() returned and the first listening() call reset the + // counters, so a one-time registration or token exchange during boot + // disappeared before any assertion looked. The mint.count() check at the + // end is about the separate mint server and says nothing about these + // endpoints. + // + // Boot is where the deleted code would have run: a registration was what a + // runtime did on the way up, and the beat only followed it. + listening("before the boot") + t.Setenv(manifest.APIConsumesEnvironmentVariable, consumesThings) + solution := boot(t, New(Manifest{ID: testSolutionID}).Consumes(passthroughModule()). + Contract(ModuleContract{Ceilings: map[string]map[string][]Scope{ + localProfile: {"things": {{ResourceKind: "things", Actions: []string{"read"}}}}, + }}), mint) + if got, paths := reached(); got != 0 { + t.Errorf("a deleted endpoint was called %d times at %v DURING THE BOOT: the self-registration, the consumed-module registrations and the single-use token exchange are deleted, so the way up may not touch one", got, paths) + } + listening("after the boot") + + // THE PERIODIC WINDOW, in two parts, because one of them was the wrong + // length. + // + // The shortest interval the deleted configuration accepted was 1s, set + // above, so three seconds is three periods of the fastest beat that was + // ever configurable — and a reviewer pointed out that it says nothing + // about a survivor that IGNORES the removed setting and keeps the old + // 15-second default. That one fires in neither three seconds nor any + // window shorter than its period, so the window has to be longer than the + // default was. + // + // This is the one place in the suite that waits on real time, and it is + // deliberate: the two heartbeat tests that waited 10s and 5s went with the + // heartbeat, and this is the test that answers whether the heartbeat is + // actually gone. A shorter window here is the assertion that was already + // wrong once. + const deletedDefaultInterval = 15 * time.Second + time.Sleep(3 * time.Second) + if got, paths := reached(); got != 0 { + t.Errorf("a deleted endpoint was called %d times at %v during three periods of the fastest heartbeat interval this configuration ever accepted: the beat, both registrations and the token exchange are deleted, so nothing may reach one", got, paths) + } + time.Sleep(deletedDefaultInterval + time.Second - 3*time.Second) + if got, paths := reached(); got != 0 { + t.Errorf("a deleted endpoint was called %d times at %v within %s of the boot: that is longer than the registration heartbeat's own default period, so this catches a surviving beat that ignores the interval configuration as well as one that honours it", got, paths, deletedDefaultInterval+time.Second) + } + listening("after the periodic window") + + if status := getStatus(t, solution.client, solution.base+HealthPath); status != http.StatusOK { + t.Fatalf("health = %d, want 200", status) + } + + // The SHUTDOWN window. Every assertion used to run while the solution was + // still serving, and the shutdown happened in t.Cleanup afterwards with + // nothing left to read the count — so a deregistration on the way out was + // unobserved. The cutover's claim is that nothing reaches a deleted + // endpoint, and the last thing a process does is part of that. + listening("before the shutdown") + solution.stop() + if got, paths := reached(); got != 0 { + t.Errorf("a deleted endpoint was called %d times at %v while this runtime shut down: there is no deregistration, so nothing may reach one on the way out either", got, paths) + } + // The observer outlives the runtime, so the zero above is a statement + // about the runtime rather than about a port that closed with it. + listening("after the shutdown") + + if got := mint.count(); got != 1 { + t.Errorf("the host received %d requests, want exactly 1 (the mint): anything more is a registration that survived", got) + } + for _, header := range []string{ + "x-codefly-internal-token", + "x-codefly-solution-registration", + "x-codefly-module-registration", + "x-codefly-module-secret", + } { + if value := mint.headers[0].Get(header); value != "" { + t.Errorf("the one request this runtime made presented %s=%q: this runtime has no registration credential to present", header, value) + } + } + for _, path := range mint.paths { + if path != credentialMintPath { + t.Errorf("the host was asked for %q: the only endpoint this runtime calls on its own behalf is the mint", path) + } + } +} + +// TestTheMintCarriesThisWorkloadsCredentialForAViewersMint: the mint this +// runtime runs on a viewer's behalf says which module is asking. Without it the +// issuer sees only that somebody holding a viewer's bearer asked, and the +// installation and binding checks at the issuer have nothing to bind to. +func TestTheMintCarriesThisWorkloadsCredentialForAViewersMint(t *testing.T) { + host := newHostMint(t, &hostMint{}) + tokenFile := filepath.Join(t.TempDir(), "token") + writeFile(t, tokenFile, "projected") + source := mintClientFor(t, host, tokenFile) + credential, err := source.Credential(context.Background()) + if err != nil { + t.Fatal(err) + } + + gw := newModuleGateway(t, http.StatusOK, `{"entry_id":"e1"}`) + server := New(Manifest{ID: testSolutionID}).Credential(source) + server.cfg.gatewayURL = gw.URL + header := http.Header{} + header.Set("authorization", viewerBearer()) + header.Set(orgHeader, "org-1") + header.Set(sessionHeader, "session-1") + header.Set(workcontext.InstallationIDHeaderName, testInstallation) + gateway := server.gatewayFor(header) + if _, err := gateway.ForModule(context.Background(), "things", Scope{ResourceKind: "things", Actions: []string{"read"}}); err != nil { + t.Fatalf("ForModule: %v", err) + } + mints := gw.observedMints() + if len(mints) != 1 { + t.Fatalf("observed %d mints, want 1", len(mints)) + } + if mints[0].Bearer != viewerBearer() { + t.Errorf("the mint presented bearer %q, want the viewer's", mints[0].Bearer) + } + if mints[0].WorkContext != credential.Token() { + t.Errorf("the mint presented work context %q, want this workload's own credential %q", mints[0].WorkContext, credential.Token()) + } +} + +// TestAMintIsRefusedWhenThisWorkloadCannotAttest is the blocker this PR was +// first refused for, inverted into a test. +// +// The attestation used to be best-effort: a workload whose renewal the issuer +// had started refusing kept minting viewer capabilities, with no execution +// binding on any of them, and a test asserted that three such mints succeeded. +// That is fail-open in the one case the attestation exists for — the issuer +// stops renewing precisely when the installation has moved, the build is no +// longer approved, or the principal's epoch has advanced — and "the host does +// not require it yet" is a statement about the counterpart's current state +// rather than a property of this runtime. +func TestAMintIsRefusedWhenThisWorkloadCannotAttest(t *testing.T) { + buf := &syncBuffer{} + log.SetOutput(buf) + defer log.SetOutput(os.Stderr) + + refusing := newHostMint(t, &hostMint{status: http.StatusForbidden}) + tokenFile := filepath.Join(t.TempDir(), "token") + writeFile(t, tokenFile, "projected") + gw := newModuleGateway(t, http.StatusOK, `{"entry_id":"e1"}`) + server := New(Manifest{ID: testSolutionID}).Credential(mintClientFor(t, refusing, tokenFile)) + server.cfg.gatewayURL = gw.URL + + header := http.Header{} + header.Set("authorization", viewerBearer()) + header.Set(orgHeader, "org-1") + header.Set(sessionHeader, "session-1") + header.Set(workcontext.InstallationIDHeaderName, testInstallation) + for range 3 { + _, err := server.gatewayFor(header).ForModule(context.Background(), "things", Scope{ResourceKind: "things", Actions: []string{"read"}}) + if err == nil { + t.Fatal("ForModule minted for a viewer while this workload could not attest which module was asking") + } + if !errors.Is(err, ErrNotAttested) { + t.Fatalf("ForModule error = %v, want one wrapping ErrNotAttested", err) + } + } + // Zero mints. Not fewer, not unattested ones: none. + if got := len(gw.observedMints()); got != 0 { + t.Errorf("observed %d viewer mints, want 0: a mint this solution cannot attest for must not be sent", got) + } + // The refusal reaches a page as this solution's condition, not the + // viewer's authority failing. + if code := connect.CodeOf(relayedError(fmt.Errorf("mint: %w", ErrNotAttested))); code != connect.CodeUnavailable { + t.Errorf("a page sees %v, want unavailable: the viewer's authority is not in question and one renewal fixes it", code) + } + // Still throttled: a renewal that is failing fails on every request, so + // the first occurrence must not be buried under the rest. + if got := strings.Count(buf.String(), "refusing to mint for a viewer"); got != 1 { + t.Errorf("the refusal was logged %d times for 3 calls, want 1:\n%s", got, buf.String()) + } +} + +// TestAServerWithNoCredentialSourceMintsNothing: the fail-closed rule has no +// hole for a server that was never given a source. A gateway built outside the +// boot — which is what the passthrough test seam does — mints nothing rather +// than minting unattested. +func TestAServerWithNoCredentialSourceMintsNothing(t *testing.T) { + gw := newModuleGateway(t, http.StatusOK, `{"entry_id":"e1"}`) + server := New(Manifest{ID: testSolutionID}) + server.cfg.gatewayURL = gw.URL + header := http.Header{} + header.Set("authorization", viewerBearer()) + header.Set(orgHeader, "org-1") + header.Set(sessionHeader, "session-1") + header.Set(workcontext.InstallationIDHeaderName, testInstallation) + _, err := server.gatewayFor(header).ForModule(context.Background(), "things", Scope{ResourceKind: "things", Actions: []string{"read"}}) + if !errors.Is(err, ErrNotAttested) { + t.Fatalf("ForModule error = %v, want ErrNotAttested", err) + } + if got := len(gw.observedMints()); got != 0 { + t.Errorf("observed %d mints with no credential source at all, want 0", got) + } +} + +// TestAnAbsentAuthorityValueIsNamed: the three values the platform provisions +// are refused by name. A boot that said only "could not mint" sent whoever read +// it looking through all three. +func TestAnAbsentAuthorityValueIsNamed(t *testing.T) { + for _, key := range []string{AuthorityPrincipalKey, AuthorityAudienceKey, AuthorityProjectionAudienceKey} { + t.Setenv("CODEFLY__WORKSPACE_CONFIGURATION__MODULE_AUTHORITY__"+key, "") + } + // The SDK answers from the snapshot it loaded, so an unprovisioned group + // has to be loaded as unprovisioned. + if err := codefly.LoadEnvironmentVariables(); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = codefly.LoadEnvironmentVariables() }) + _, err := readAuthority(context.Background()) + if err == nil { + t.Fatal("readAuthority accepted an unprovisioned authority group") + } + if !strings.Contains(err.Error(), AuthorityGroup) { + t.Errorf("refusal %q does not name the %q group a reader has to provision", err, AuthorityGroup) + } +} + +// stubCredentialSource is a consumer-supplied source that holds one credential +// it was handed. It exists for the boot tests, which care about which material +// a selected source needs rather than about minting. +type stubCredentialSource struct{ credential workcontext.Credential } + +func (s stubCredentialSource) Credential(context.Context) (workcontext.Credential, error) { + return s.credential, nil +} + +// mintClientFor is the SDK's mint client as the runtime configures it, pointed +// at a fake host over plain HTTP. The runtime has no mint of its own, so this +// is the only client any of these tests exercise. +// +// Which also says what the tests over it can and cannot prove, and it is worth +// being explicit because a reader naturally assumes otherwise: they pin the +// SDK's *contract* — one carrier per execution, the projected token re-read at +// every mint — and they reach no line of this repository, so no mutation of +// this package's own code can fail them. Their value is that an SDK bump +// changing either property fails here rather than in a deployment. What this +// runtime does with the client is covered by the boot and health tests, which +// drive start() and serve(). +// +// A boot's own client presents this workload's identity and verifies the host +// against the projected anchor (outboundClient); that path is exercised by the +// boot tests, which stand their fake host up under the cell's anchor. These +// tests drive the client directly, where the question is the mint's behaviour +// rather than who dialled it. +func mintClientFor(t *testing.T, mint *hostMint, tokenFile string) CredentialSource { + t.Helper() + client, err := workcontext.NewMintClient(workcontext.MintOptions{ + URL: mint.URL + credentialMintPath, + // The audience is named as a pinned value now, not passed as a string + // beside the pin: a string alongside meant the drift check guarded a + // value the mint did not use. + Audience: workcontext.AuthorityValue{Name: AuthorityGroup, Key: AuthorityAudienceKey}, + Authority: fixedAuthority{audience: testAudience}, + ProjectedToken: workcontext.ProjectedTokenFile(tokenFile), + ProjectionAudience: testProjectionAudience, + // The three readers sdk-go#51 takes, each asked during the handshake. + // The client still builds and owns the transport; what it no longer + // does is fix the anchor, present nothing, and admit any peer the + // anchor signed. + TrustAnchor: mint.roots, + ClientCertificate: func() (*tls.Certificate, error) { return mint.caller, nil }, + AdmittedPeers: func() ([]string, error) { return []string{testGatewayPrincipal}, nil }, + }) + if err != nil { + t.Fatal(err) + } + return client +} + +// fixedAuthority is an AuthorityPin whose values do not drift. The drift +// refusal is exercised by its own tests against the real reader; a fixture +// that drifted would fail every other test for a reason none of them is about. +type fixedAuthority struct{ audience string } + +func (fixedAuthority) Recheck(context.Context) error { return nil } + +func (a fixedAuthority) Value(name, key string) (string, error) { + switch { + case name == AuthorityGroup && key == AuthorityAudienceKey: + return a.audience, nil + case name == AuthorityGroup && key == AuthorityProjectionAudienceKey: + return testProjectionAudience, nil + case name == AuthorityGroup && key == AuthorityPrincipalKey: + return testPrincipal, nil + } + return "", fmt.Errorf("no pinned authority value %s/%s", name, key) +} + +// refusingSource is a credential source whose renewal the issuer has started +// refusing for good: the shape the boot already treats as terminal, happening +// after a successful boot instead of during it. +type refusingSource struct{ err error } + +func (r refusingSource) Credential(context.Context) (workcontext.Credential, error) { + return workcontext.Credential{}, r.err +} + +// TestATerminalRenewalRefusalEndsTheProcessRatherThanServing503Forever is the +// asymmetry a second reviewer found between the boot and the run. +// +// The boot separates the issuer's two answers and must: a refusal is a +// judgement on this build that no number of attempts changes, an unavailable +// mint is transient. At renewal both were collapsed into ErrNotAttested, which +// reaches a page as unavailable with the note that one renewal fixes it. For a +// refusal nothing fixes it — so the process answered 503 to every request +// forever while /health returned 200, which is the exact shape the +// delivered-presence model replaced the heartbeat to avoid: a solution that +// serves nothing and looks alive. +func TestATerminalRenewalRefusalEndsTheProcessRatherThanServing503Forever(t *testing.T) { + for _, tc := range []struct { + name string + err error + terminal bool + }{ + {"the host refuses this build", fmt.Errorf("mint: %w", workcontext.ErrMintRefused), true}, + // NOT terminal, and the earlier revision of this test asserted the + // opposite. ErrRevoked does not come back from the mint at all — it is + // a *callee* refusing a capability sealed to state that has moved — and + // the SDK's contract for it is a refresh and one retry. Ending the + // process on it would be ending on a condition the client recovers + // from. It is handled where it arrives: the far end's 409 drops the + // capability so the next call mints. + {"a capability a callee says is superseded", fmt.Errorf("mint: %w", workcontext.ErrRevoked), false}, + // The control, and the reason this is not just "exit on any error": an + // issuer that cannot reach its own policy log is behaving correctly, + // and exiting on it turns a dependency blip into a crash loop. + {"the issuer is briefly unavailable", fmt.Errorf("mint: %w", workcontext.ErrMintUnavailable), false}, + } { + t.Run(tc.name, func(t *testing.T) { + gw := newModuleGateway(t, http.StatusOK, `{"entry_id":"e1"}`) + server := New(Manifest{ID: testSolutionID}).Credential(refusingSource{err: tc.err}) + server.cfg.gatewayURL = gw.URL + + // Health asks the credential, so with a source that refuses it is + // already unhealthy before any request — which is the point of + // C5: a solution serving no module calls must not look alive while + // the issuer has stopped approving its build. + if status, _ := healthStatus(t, server); status != http.StatusServiceUnavailable { + t.Errorf("health = %d while the credential cannot be obtained, want 503: the probe is the only channel this runtime has, and lazy renewal means nothing else may ever ask", status) + } + + header := http.Header{} + header.Set("authorization", viewerBearer()) + header.Set(orgHeader, "org-1") + header.Set(sessionHeader, "session-1") + header.Set(workcontext.InstallationIDHeaderName, testInstallation) + _, err := server.gatewayFor(header).ForModule(context.Background(), "things", + Scope{ResourceKind: "things", Actions: []string{"read"}}) + if !errors.Is(err, ErrNotAttested) { + t.Fatalf("ForModule error = %v, want ErrNotAttested", err) + } + if got := len(gw.observedMints()); got != 0 { + t.Errorf("observed %d viewer mints, want 0", got) + } + + status, body := healthStatus(t, server) + if tc.terminal { + if status != http.StatusServiceUnavailable { + t.Errorf("health = %d after a refusal the issuer will not reverse, want 503: a 200 invites the host to keep routing to a binding that answers nothing", status) + } + if !strings.Contains(body, "refused") { + t.Errorf("the probe says %q, want it to name the condition", body) + } + select { + case <-server.credentialRefusedC(): + default: + t.Error("the process was not asked to end: a credential the host has stopped honouring cannot be waited out") + } + if stored := server.terminalErr.Load(); stored == nil || !errors.Is(*stored, tc.err) { + t.Error("the reason this process is ending was not recorded, so its exit would say nothing") + } + } else { + // Unhealthy, because this process cannot act for a viewer + // until the credential can be obtained — but NOT ending, which + // is the distinction that matters: the host stops routing to + // it, and it recovers when the issuer does. Exiting here would + // turn a dependency blip into a crash loop. + if status != http.StatusServiceUnavailable { + t.Errorf("health = %d while the credential cannot be obtained, want 503", status) + } + if strings.Contains(body, "refused") { + t.Errorf("the probe says %q, which reads as a judgement on this build: this condition is transient", body) + } + select { + case <-server.credentialRefusedC(): + t.Error("the process was ended on a transient condition") + default: + } + if server.terminalErr.Load() != nil { + t.Error("a transient condition was recorded as the reason this process is ending") + } + } + }) + } +} + +// healthStatus asks the health route what this process reports. +func healthStatus(t *testing.T, server *Server) (int, string) { + t.Helper() + recorder := httptest.NewRecorder() + server.handleHealth(recorder, httptest.NewRequest(http.MethodGet, HealthPath, nil)) + return recorder.Code, recorder.Body.String() +} + +// TestHealthIsHonestAboutTheCredential is the other direction of C5, so the +// check above is not just a probe that always fails: a process holding a +// current credential is healthy, and one that holds none at all (a server built +// outside a boot) does not pretend to consult one. +func TestHealthIsHonestAboutTheCredential(t *testing.T) { + t.Run("a current credential is healthy", func(t *testing.T) { + mint := newHostMint(t, &hostMint{}) + tokenFile := filepath.Join(t.TempDir(), "token") + writeFile(t, tokenFile, "projected") + server := New(Manifest{ID: testSolutionID}).Credential(mintClientFor(t, mint, tokenFile)) + if status, body := healthStatus(t, server); status != http.StatusOK { + t.Errorf("health = %d (%s) with a credential the issuer mints, want 200", status, body) + } + // And asking did not turn the probe into a mint per probe. + for range 5 { + healthStatus(t, server) + } + if got := mint.count(); got != 1 { + t.Errorf("six probes produced %d mints, want 1: the client holds one credential and the probe must not become a heartbeat", got) + } + }) + + t.Run("no credential source at all", func(t *testing.T) { + server := New(Manifest{ID: testSolutionID}) + if status, _ := healthStatus(t, server); status != http.StatusOK { + t.Errorf("health = %d on a server with no credential source, want 200: there is nothing to consult, and a gateway built outside a boot mints nothing anyway", status) + } + }) +} + +// failingCredentialSource answers every ask with one error, which is how a +// refused build and an unreachable issuer both look from a route's point of +// view. +type failingCredentialSource struct{ err error } + +func (f failingCredentialSource) Credential(context.Context) (workcontext.Credential, error) { + return workcontext.Credential{}, f.err +} + +// TestAPlainHandlerRefusesToActWithoutACredential: a plain handler mints +// nothing, which is exactly why it was the route that never consulted the +// credential — and it still receives a gateway carrying the viewer's bearer. +// With the credential refused, a booted server kept answering 200 and kept +// forwarding that bearer, with the credential consulted zero times. +// +// handleHealth already made this argument in full and it was applied to the +// probe alone: the probe answered honestly while the routes carried on. +func TestAPlainHandlerRefusesToActWithoutACredential(t *testing.T) { + for _, tc := range []struct { + name string + err error + ends bool + names error + }{ + { + "a refused build", + fmt.Errorf("%w: this build is not the one the presence document approved", workcontext.ErrMintRefused), + true, ErrCredentialRefused, + }, + { + "an issuer that cannot answer", + fmt.Errorf("%w: the issuer cannot reach its own dependencies", workcontext.ErrMintUnavailable), + false, ErrCredentialUnavailable, + }, + } { + t.Run(tc.name, func(t *testing.T) { + var reached atomic.Int64 + server := New(Manifest{ID: testSolutionID}). + Credential(failingCredentialSource{err: tc.err}) + handler := server.wrapRequest(func(*http.Request, *Gateway) (any, error) { + reached.Add(1) + return map[string]string{"ok": "yes"}, nil + }) + + request := httptest.NewRequest(http.MethodGet, "/thing", nil) + request.Header.Set("authorization", viewerBearer()) + recorder := httptest.NewRecorder() + handler(recorder, request) + + if recorder.Code != http.StatusServiceUnavailable { + t.Errorf("the handler answered %d while this process held no approved credential, want 503: it would forward the viewer's bearer to the gateway under an authority the issuer has withdrawn", recorder.Code) + } + if got := reached.Load(); got != 0 { + t.Errorf("the handler body ran %d times, want 0: the refusal has to come before anything acts for the viewer", got) + } + body := recorder.Body.String() + if !strings.Contains(body, "authority") { + t.Errorf("the 503 body %q does not say why this solution will not act", body) + } + // And it carries nothing the source said. The error wraps whatever + // came back — a mint URL, the issuer's own text, a dial failure + // naming an internal address — and this gate wrote it into the + // viewer's response verbatim, which is the disclosure + // handlerErrorResponse exists to prevent. A new gate in front of + // the handler put a path *around* the sanitization. + for _, leaked := range []string{"issuer", "presence document", "dependencies", "mint"} { + if strings.Contains(body, leaked) { + t.Errorf("the 503 body %q carries %q from the source's own error", body, leaked) + } + } + // Terminal and transient stay distinguishable: conflating them was + // a review blocker in both directions. + if err := server.actingForAViewer(context.Background()); !errors.Is(err, tc.names) { + t.Errorf("the route's refusal is %v, want one wrapping %v", err, tc.names) + } + if ended := server.terminalErr.Load() != nil; ended != tc.ends { + t.Errorf("the process ending = %v, want %v: a judgement about this build ends it, an issuer that cannot answer right now does not", ended, tc.ends) + } + }) + } +} + +// TestAPlainHandlerStillServesWithAnApprovedCredential is the survive-control. +// Without it, a route that refuses everything unconditionally passes the test +// above. +func TestAPlainHandlerStillServesWithAnApprovedCredential(t *testing.T) { + mint := newHostMint(t, &hostMint{}) + tokenFile := filepath.Join(t.TempDir(), "token") + writeFile(t, tokenFile, "projected") + server := New(Manifest{ID: testSolutionID}).Credential(mintClientFor(t, mint, tokenFile)) + handler := server.wrapRequest(func(*http.Request, *Gateway) (any, error) { + return map[string]string{"ok": "yes"}, nil + }) + request := httptest.NewRequest(http.MethodGet, "/thing", nil) + request.Header.Set("authorization", viewerBearer()) + recorder := httptest.NewRecorder() + handler(recorder, request) + if recorder.Code != http.StatusOK { + t.Fatalf("a handler answered %d while this process holds an approved credential, want 200: %s", recorder.Code, recorder.Body.String()) + } +} + +// slowCredentialSource answers, eventually. It is the shape of an issuer that +// is reachable and not responding. +type slowCredentialSource struct { + after time.Duration + credential workcontext.Credential +} + +func (s slowCredentialSource) Credential(ctx context.Context) (workcontext.Credential, error) { + select { + case <-time.After(s.after): + return s.credential, nil + case <-ctx.Done(): + return workcontext.Credential{}, ctx.Err() + } +} + +// TestARouteIsNotParkedByASlowCredentialSource: the bound on asking for the +// credential has to be imposed outside the SDK's client, because a context does +// not reach what blocks. +// +// The mint client takes a sync.Mutex and holds it across the network mint, and +// sync.Mutex.Lock ignores contexts — so a caller arriving during a slow renewal +// waits on the lock for as long as the renewal takes, whatever deadline it +// passed in. That made the two-second health deadline advisory, and it would +// have made every route's credential check park behind one slow issuer, which +// is a worse failure than the one the check was added for. +func TestARouteIsNotParkedByASlowCredentialSource(t *testing.T) { + slow := slowCredentialSource{after: 30 * time.Second} + + t.Run("with no credential in hand, the route refuses promptly", func(t *testing.T) { + server := New(Manifest{ID: testSolutionID}).Credential(slow) + started := time.Now() + done := make(chan error, 1) + go func() { done <- server.actingForAViewer(context.Background()) }() + select { + case err := <-done: + if err == nil { + t.Fatal("the route acted for a viewer without this process holding a credential") + } + if !errors.Is(err, ErrCredentialUnavailable) { + t.Errorf("the route refused with %v, want one wrapping %v", err, ErrCredentialUnavailable) + } + if elapsed := time.Since(started); elapsed > 20*routeCredentialTimeout { + t.Errorf("the route took %s to answer for a %s bound: the deadline it passes in does not reach the lock the source holds", elapsed, routeCredentialTimeout) + } + case <-time.After(20 * routeCredentialTimeout): + t.Fatalf("the route was still waiting %s after a %s bound: every request would park behind one slow issuer", 20*routeCredentialTimeout, routeCredentialTimeout) + } + }) + + // And the other half, which is why a timeout does not simply refuse: + // renewal begins while a valid credential is still held, so a slow renewal + // must not 503 every viewer. + t.Run("with an unexpired credential in hand, the route proceeds", func(t *testing.T) { + mint := newHostMint(t, &hostMint{}) + held := mintedCredential(t, mint, "held") + server := New(Manifest{ID: testSolutionID}).Credential(slow) + server.credentialHeld = held + done := make(chan error, 1) + go func() { done <- server.actingForAViewer(context.Background()) }() + select { + case err := <-done: + if err != nil { + t.Errorf("a route was refused while this process holds a credential valid for another hour and the source is merely slow: %v", err) + } + case <-time.After(20 * routeCredentialTimeout): + t.Fatal("the route parked despite holding a valid credential") + } + }) + + // An expired one is not "in hand" at all. + t.Run("with an expired credential, the route refuses", func(t *testing.T) { + mint := newHostMint(t, &hostMint{}) + tokenFile := filepath.Join(t.TempDir(), "token") + writeFile(t, tokenFile, "projected") + // Already past its expiry, which is the state a route must not act + // under. No waiting: see expiredCredential. + expired := expiredCredential(t, mint, tokenFile) + server := New(Manifest{ID: testSolutionID}).Credential(slow) + server.credentialHeld = expired + done := make(chan error, 1) + go func() { done <- server.actingForAViewer(context.Background()) }() + select { + case err := <-done: + if err == nil { + t.Error("a route acted for a viewer on an expired credential") + } + case <-time.After(20 * routeCredentialTimeout): + t.Fatal("the route parked") + } + }) +} + +// TestEverySourceIsHeldToTheFrozenAuthority: a supplied credential source +// replaces the mint client, and with it every option mintOptions carries — +// including the Authority the SDK rechecks before each renewal. So the one +// guarantee that stops a drifted authority value being re-sealed into a new +// credential applied to the platform path only, while this package's +// documentation promises it of the process. +func TestEverySourceIsHeldToTheFrozenAuthority(t *testing.T) { + // Two different credentials, so the second ask reads as a renewal: the + // token is the generation. They come from a plain-HTTP fake host of this + // test's own, because the boot's mint is served under the cell's anchor and + // this is only a way to obtain two real sealed credentials to hand out. + // Three: the boot consumes the first, the control renewal the second, and + // the ask after the drift the third. + fabricator := newHostMint(t, &hostMint{}) + handing := &handingSource{credentials: []workcontext.Credential{ + mintedCredential(t, fabricator, "first"), + mintedCredential(t, fabricator, "second"), + mintedCredential(t, fabricator, "third"), + }} + + mint := newHostMint(t, &hostMint{}) + bootEnvironment(t, mint) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + server := New(Manifest{ID: testSolutionID}).Credential(handing) + if _, err := takeListener(server).start(ctx); err != nil { + t.Fatalf("boot: %v", err) + } + + // The control: with the authority as this process froze it, a renewal is + // handed over. + if _, err := server.credential.Credential(ctx); err != nil { + t.Fatalf("a renewal was refused while the authority had not drifted: %v", err) + } + + // Now an authority-bearing value drifts under the running process, and the + // next renewal must be refused rather than re-sealed against it. + t.Setenv("CODEFLY__WORKSPACE_CONFIGURATION__MODULE_AUTHORITY__"+AuthorityPrincipalKey, + "spiffe://codefly.test/ns/solutions/sa/somebody-else") + if err := codefly.LoadEnvironmentVariables(); err != nil { + t.Fatal(err) + } + _, err := server.credential.Credential(ctx) + if err == nil { + t.Fatal("a renewal proceeded after an authority-bearing value drifted from the one this process froze: that renewal is the one moment the drift would be laundered into a differently-sealed credential") + } + if !errors.Is(err, workcontext.ErrMintRefused) { + t.Errorf("the refusal %v is not reported as a judgement, so the boot and the run would classify it differently", err) + } +} + +// handingSource hands out a prepared sequence of credentials, so a second ask +// reads as a renewal. +type handingSource struct { + mu sync.Mutex + credentials []workcontext.Credential + at int + asks int +} + +func (h *handingSource) Credential(context.Context) (workcontext.Credential, error) { + h.mu.Lock() + defer h.mu.Unlock() + h.asks++ + credential := h.credentials[h.at] + if h.at < len(h.credentials)-1 { + h.at++ + } + return credential, nil +} + +// count is how many times this source was reached, which is what shows whether +// a check ran before the ask or after it. +func (h *handingSource) count() int { + h.mu.Lock() + defer h.mu.Unlock() + return h.asks +} + +// mintedCredential is one credential from the fake host, so a test can hand out +// a real sealed one rather than a zero value. +func mintedCredential(t *testing.T, mint *hostMint, projection string) workcontext.Credential { + t.Helper() + tokenFile := filepath.Join(t.TempDir(), "token") + writeFile(t, tokenFile, projection) + credential, err := mintClientFor(t, mint, tokenFile).Credential(context.Background()) + if err != nil { + t.Fatalf("mint a credential for the test: %v", err) + } + return credential +} + +// unavailableAfter answers normally until `after` asks, then reports the issuer +// unavailable — the shape of a credential inside its renewal window against an +// issuer that has stopped answering. +type unavailableAfter struct { + mu sync.Mutex + credential workcontext.Credential + asks int + after int +} + +func (u *unavailableAfter) Credential(context.Context) (workcontext.Credential, error) { + u.mu.Lock() + defer u.mu.Unlock() + u.asks++ + if u.asks > u.after { + return workcontext.Credential{}, fmt.Errorf("%w: the issuer cannot answer right now", workcontext.ErrMintUnavailable) + } + return u.credential, nil +} + +func (u *unavailableAfter) count() int { + u.mu.Lock() + defer u.mu.Unlock() + return u.asks +} + +// TestAStreamOfViewerRequestsDoesNotMintPerRequest is the regression this +// round's own fix introduced, and it is the worst kind this runtime can have. +// +// With the credential inside its renewal window and the issuer answering 503, +// every viewer request asked, every ask minted, and every request answered 503 +// — while a perfectly valid credential sat in hand. Forty requests, forty +// audited mints. That is not a smaller version of the heartbeat this runtime +// deleted; it is a larger one, because the heartbeat minted on a timer and this +// minted on traffic. The PR body's own claim is that a client minting per +// request "would be worse than the heartbeat it replaced". +// +// The fast-failure path refused outright while the timeout path, three lines +// below, already served on a held credential. The two disagreed. +func TestAStreamOfViewerRequestsDoesNotMintPerRequest(t *testing.T) { + mint := newHostMint(t, &hostMint{}) + held := mintedCredential(t, mint, "held") + source := &unavailableAfter{credential: held, after: 1} + + server := New(Manifest{ID: testSolutionID}).Credential(source) + handler := server.wrapRequest(func(*http.Request, *Gateway) (any, error) { + return map[string]string{"ok": "yes"}, nil + }) + + // The first request takes the credential, which is valid for a while yet. + request := httptest.NewRequest(http.MethodGet, "/thing", nil) + request.Header.Set("authorization", viewerBearer()) + first := httptest.NewRecorder() + handler(first, request) + if first.Code != http.StatusOK { + t.Fatalf("the first request answered %d, want 200: %s", first.Code, first.Body.String()) + } + + // Now the issuer stops answering, and forty viewers arrive. + const viewers = 40 + for range viewers { + recorder := httptest.NewRecorder() + handler(recorder, httptest.NewRequest(http.MethodGet, "/thing", nil).WithContext(request.Context())) + served := httptest.NewRequest(http.MethodGet, "/thing", nil) + served.Header.Set("authorization", viewerBearer()) + recorder = httptest.NewRecorder() + handler(recorder, served) + if recorder.Code != http.StatusOK { + t.Fatalf("a viewer request answered %d while this process held a credential valid until %s: the issuer being unavailable is not a reason to refuse a viewer when the authority to act for them is already in hand", + recorder.Code, held.ExpiresAt().UTC().Format(time.RFC3339)) + } + } + + // And the asks were not one per request. One ask failed, which quiets the + // next ones; the renewal the credential's own expiry dictates is still + // ahead of us. + if got := source.count(); got > 4 { + t.Errorf("the source was asked %d times for %d viewer requests: a runtime that asks — and therefore mints — per request is worse than the heartbeat it replaced, which at least minted on a timer", got, viewers) + } +} + +// TestConcurrentViewerRequestsShareOneCredentialAsk is the other half, and the +// half a sequential test cannot see: without single-flight, N requests arriving +// together each start their own ask, and each ask that reaches a due credential +// mints. +// +// It is the shape production takes — the passthrough calls this from whatever +// goroutine the server hands it — and the saving this whole change buys is one +// mint plus a handful of renewals an hour against the heartbeat's 240. +func TestConcurrentViewerRequestsShareOneCredentialAsk(t *testing.T) { + // Slow enough that every caller arrives while the first ask is in flight, + // which is exactly when a per-caller ask would mint again. + source := &countedSlowSource{after: 400 * time.Millisecond} + server := New(Manifest{ID: testSolutionID}).Credential(source) + + const callers = 24 + var wg sync.WaitGroup + start := make(chan struct{}) + for range callers { + wg.Add(1) + go func() { + defer wg.Done() + <-start + _, _ = server.credentialWithin(context.Background(), 5*time.Second) + }() + } + close(start) + wg.Wait() + + if got := source.count(); got != 1 { + t.Errorf("%d concurrent callers produced %d asks, want 1: each ask that finds the credential due for renewal mints, so one per caller is one audited mint per request", callers, got) + } +} + +// countedSlowSource answers after a delay and counts how many times it was +// reached. +type countedSlowSource struct { + mu sync.Mutex + asks int + after time.Duration + held workcontext.Credential +} + +func (c *countedSlowSource) Credential(ctx context.Context) (workcontext.Credential, error) { + c.mu.Lock() + c.asks++ + c.mu.Unlock() + select { + case <-time.After(c.after): + case <-ctx.Done(): + return workcontext.Credential{}, ctx.Err() + } + // Deliberately unusable, so the test turns on the ask count alone and not + // on what came back. + return c.held, nil +} + +func (c *countedSlowSource) count() int { + c.mu.Lock() + defer c.mu.Unlock() + return c.asks +} + +// TestACredentialThatAuthorisesNothingIsRefused: a supplied source may hand +// back the zero credential, or an expired one, and this runtime held it, +// booted, answered /health 200 and acted for viewers under it. +// +// The authority wrapper could not see either: it keys on the token *changing*, +// and a token that is always "" never changes. +func TestACredentialThatAuthorisesNothingIsRefused(t *testing.T) { + mint := newHostMint(t, &hostMint{}) + tokenFile := filepath.Join(t.TempDir(), "token") + writeFile(t, tokenFile, "projected") + expiring := expiredCredential(t, mint, tokenFile) + + for _, tc := range []struct { + name string + credential workcontext.Credential + terminal bool + // names is the part of the refusal that has to be about this case, so + // a credential failing several checks at once does not stand in for + // each of them. + names string + }{ + {"the zero credential", workcontext.Credential{}, true, "empty token"}, + {"an expired credential", expiring, false, "expired at"}, + } { + t.Run(tc.name, func(t *testing.T) { + // At the boot. + boot := newHostMint(t, &hostMint{}) + server := New(Manifest{ID: testSolutionID}).Credential(stubCredentialSource{credential: tc.credential}) + server.firstMintWindow = 750 * time.Millisecond + if err := bootFailsWithin(t, server, boot, 10*time.Second); err == nil { + t.Fatal("the boot accepted a credential that authorises nothing: this process would serve, answer /health 200, and act for viewers under it") + } + + // And at a route, for a source that only degrades later. + serving := New(Manifest{ID: testSolutionID}).Credential(stubCredentialSource{credential: tc.credential}) + if err := serving.actingForAViewer(context.Background()); err == nil { + t.Fatal("a route acted for a viewer under a credential that authorises nothing") + } + // And classified: nothing to mint with is a judgement, an expiry + // that has passed is not. + err := usableCredential(tc.credential) + if got := terminalCredentialFailure(err); got != tc.terminal { + t.Errorf("terminal = %v, want %v for %v: the boot and the run have to classify these the same way", got, tc.terminal, err) + } + // Refused for its OWN reason. The zero credential fails three of + // these checks at once — no token, no seal, no expiry — so + // asserting only that it is refused left each individual check + // unprotected: deleting the empty-token check still refused it, on + // the seal. + if !strings.Contains(err.Error(), tc.names) { + t.Errorf("the refusal %q does not name %s, so this case does not cover that check", err, tc.names) + } + }) + } +} + +// slowThenRefused answers slowly, and then with a judgement — the shape where +// acquisition outlives the caller that started it. +type slowThenRefused struct { + after time.Duration + asked chan struct{} +} + +func (s *slowThenRefused) Credential(ctx context.Context) (workcontext.Credential, error) { + select { + case s.asked <- struct{}{}: + default: + } + select { + case <-time.After(s.after): + case <-ctx.Done(): + return workcontext.Credential{}, ctx.Err() + } + return workcontext.Credential{}, fmt.Errorf("%w: this build is not the one the presence document approved", workcontext.ErrMintRefused) +} + +// TestATerminalRefusalIsRecordedWhereverItLands is the blocker the static round +// six found in this round's own fix. +// +// The ask runs on a context detached from the caller's, so a request that gives +// up does not cancel work other callers are waiting on. That also means the +// answer can arrive after every caller has gone — and a terminal refusal +// classified only by whoever was still waiting was simply discarded. The route +// served on the held credential, nothing recorded the refusal, and the process +// went on acting for viewers after its own source had reported this execution +// refused. The classification belongs to the answer, not to the audience. +func TestATerminalRefusalIsRecordedWhereverItLands(t *testing.T) { + mint := newHostMint(t, &hostMint{}) + held := mintedCredential(t, mint, "held") + source := &slowThenRefused{after: 750 * time.Millisecond, asked: make(chan struct{}, 1)} + + server := New(Manifest{ID: testSolutionID}).Credential(source) + server.credentialHeld = held + + // A route asks, times out well before the source answers, and is served + // from the credential in hand — which is correct. + if err := server.actingForAViewer(context.Background()); err != nil { + t.Fatalf("a route was refused while this process held a valid credential and the source was merely slow: %v", err) + } + select { + case <-source.asked: + case <-time.After(2 * time.Second): + t.Fatal("the source was never asked, so this test does not exercise the detached ask") + } + + // The refusal lands with nobody waiting. + deadline := time.Now().Add(5 * time.Second) + for server.terminalErr.Load() == nil && time.Now().Before(deadline) { + time.Sleep(25 * time.Millisecond) + } + if server.terminalErr.Load() == nil { + t.Fatal("this execution's credential was refused and nothing recorded it, because the caller that started the ask had already been served: the process keeps acting for viewers under an authority its own source says is gone, and /health keeps answering 200") + } + + // And the process now refuses to act, rather than carrying on with the + // credential it still holds. + if err := server.actingForAViewer(context.Background()); !errors.Is(err, ErrCredentialRefused) { + t.Errorf("after a recorded refusal a route answered %v, want one wrapping %v: a held credential does not survive a judgement about the build that holds it", err, ErrCredentialRefused) + } +} + +// TestAWithdrawnAuthorityIsCaughtBeforeTheMint: the recheck ran after the ask, +// so a withdrawn authority was discovered one renewal too late — the mint had +// already happened, sealed to values nobody approved, and the caller that +// triggered it was served. +func TestAWithdrawnAuthorityIsCaughtBeforeTheMint(t *testing.T) { + fabricator := newHostMint(t, &hostMint{}) + handing := &handingSource{credentials: []workcontext.Credential{ + mintedCredential(t, fabricator, "first"), + mintedCredential(t, fabricator, "second"), + }} + + mint := newHostMint(t, &hostMint{}) + bootEnvironment(t, mint) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + server := New(Manifest{ID: testSolutionID}).Credential(handing) + if _, err := takeListener(server).start(ctx); err != nil { + t.Fatalf("boot: %v", err) + } + asksAtBoot := handing.count() + + // The authority drifts, and then something asks. + t.Setenv("CODEFLY__WORKSPACE_CONFIGURATION__MODULE_AUTHORITY__"+AuthorityPrincipalKey, + "spiffe://codefly.test/ns/solutions/sa/somebody-else") + if err := codefly.LoadEnvironmentVariables(); err != nil { + t.Fatal(err) + } + if _, err := server.credential.Credential(ctx); err == nil { + t.Fatal("an ask proceeded under a drifted authority") + } + // And the source was never reached, so nothing was minted under it. + if got := handing.count(); got != asksAtBoot { + t.Errorf("the source was asked %d times after the authority drifted (%d at boot): the recheck has to come before the ask, or the credential is already minted and sealed to a value nobody approved by the time the drift is noticed", + got-asksAtBoot, asksAtBoot) + } +} + +// TestNothingIsInferredFromAConflictOnAViewersMint replaces the two rounds +// spent building around a signal that was never sound. +// +// A 409 on a viewer's mint may mean the credential this workload presented is +// superseded, or it may mean that viewer lacks the authority they asked for. +// The host does not distinguish the two. A hook that dropped this process's +// belief on that answer, plus a remembered token so the drop would stick, +// turned one viewer's missing permission into a process-wide refusal to act for +// anybody — an escalating response to an ambiguous signal, which is worse than +// no response: before, a 409 cost that one call; after, it cost the process. +func TestNothingIsInferredFromAConflictOnAViewersMint(t *testing.T) { + // The host refuses every mint with a conflict, which is what a viewer + // lacking authority looks like from here. + // Refuses the viewer's mint for want of authority, which is exactly the + // answer that cannot be told apart from "the credential you presented is + // superseded". + gw := newModuleGateway(t, http.StatusOK, `{}`) + // 409, which is what this test is named after and what it was NOT doing. + // `deny` answers 403 — "this viewer lacks that authority" — so a + // 409-specific regression, inferring supersession from the one status that + // is genuinely ambiguous, would have survived this test. A reviewer named + // that twice. + gw.conflict = "read" + tokenFile := filepath.Join(t.TempDir(), "token") + writeFile(t, tokenFile, "projected") + mint := newHostMint(t, &hostMint{}) + + server := New(Manifest{ID: testSolutionID}). + Consumes(passthroughModule()). + Contract(ModuleContract{Ceilings: map[string]map[string][]Scope{ + localProfile: {"things": {{ResourceKind: "things", Actions: []string{"read"}}}}, + }}). + Credential(mintClientFor(t, mint, tokenFile)) + server.cfg.profile = localProfile + server.cfg.gatewayURL = gw.URL + contract, err := server.resolveContract() + if err != nil { + t.Fatalf("resolveContract: %v", err) + } + server.contract, server.contractResolved = contract, true + + header := http.Header{} + header.Set("authorization", viewerBearer()) + header.Set(orgHeader, "org-1") + header.Set(sessionHeader, "session-1") + header.Set(workcontext.InstallationIDHeaderName, testInstallation) + if _, err := server.gatewayFor(header).ForModule(context.Background(), "things", + Scope{ResourceKind: "things", Actions: []string{"read"}}); err == nil { + t.Fatal("a mint refused with a conflict was reported as a success") + } + + // That refusal cost the viewer their call and nothing else: this process + // still holds its credential and still acts for everybody else. + if _, ok := server.heldCredential(); !ok { + t.Error("a conflict on one viewer's mint dropped this process's own credential: the host does not say whether the conflict is about this workload's authority or that viewer's, so inferring the former makes one viewer's missing permission a process-wide outage") + } + if err := server.actingForAViewer(context.Background()); err != nil { + t.Errorf("this process stopped acting for viewers after one ambiguous conflict: %v", err) + } + if server.terminalErr.Load() != nil { + t.Error("an ambiguous conflict was recorded as a judgement about this build") + } + + // THE CONTROL a reviewer asked for: another viewer, asking for something + // the issuer does not conflict on, is still served. Without it the three + // assertions above hold for a process that has stopped working entirely + // for reasons none of them inspects. + gw.conflict = "" + another := http.Header{} + another.Set("authorization", viewerBearer()) + another.Set(orgHeader, "org-2") + another.Set(sessionHeader, "session-2") + another.Set(workcontext.InstallationIDHeaderName, testInstallation) + if _, err := server.gatewayFor(another).ForModule(context.Background(), "things", + Scope{ResourceKind: "things", Actions: []string{"read"}}); err != nil { + t.Errorf("a second viewer was refused after the first viewer's conflict, so one viewer's 409 did become a process-wide outage after all: %v", err) + } +} + +// TestAViewersMintGoesThroughTheOneController: the viewer's mint called the +// source directly, so the single-flight, the backoff and the held-credential +// fallback governed every path except the one that mints most. +// +// The controller is only a controller if it owns every acquisition. +func TestAViewersMintGoesThroughTheOneController(t *testing.T) { + fabricator := newHostMint(t, &hostMint{}) + held := mintedCredential(t, fabricator, "held") + source := &unavailableAfter{credential: held, after: 1} + + gw := newModuleGateway(t, http.StatusOK, `{"entry_id":"e1"}`) + server := New(Manifest{ID: testSolutionID}). + Consumes(passthroughModule()). + Contract(ModuleContract{Ceilings: map[string]map[string][]Scope{ + localProfile: {"things": {{ResourceKind: "things", Actions: []string{"read"}}}}, + }}). + Credential(source) + server.cfg.profile = localProfile + server.cfg.gatewayURL = gw.URL + contract, err := server.resolveContract() + if err != nil { + t.Fatalf("resolveContract: %v", err) + } + server.contract, server.contractResolved = contract, true + + header := http.Header{} + header.Set("authorization", viewerBearer()) + header.Set(orgHeader, "org-1") + header.Set(sessionHeader, "session-1") + header.Set(workcontext.InstallationIDHeaderName, testInstallation) + mintFor := func(t *testing.T) error { + t.Helper() + _, err := server.gatewayFor(header).ForModule(context.Background(), "things", + Scope{ResourceKind: "things", Actions: []string{"read"}}) + return err + } + + // The first mint takes the credential, which stays valid for a while. + if err := mintFor(t); err != nil { + t.Fatalf("the first viewer mint was refused: %v", err) + } + + // The issuer now fails. Forty viewers arrive, and every one of them is a + // mint for a *different* task, so the capability cache does not absorb + // them — only the controller can. + const viewers = 40 + for i := range viewers { + header.Set(sessionHeader, fmt.Sprintf("session-%d", i)) + header.Set(workcontext.InstallationIDHeaderName, testInstallation) + if err := mintFor(t); err != nil { + t.Fatalf("viewer %d was refused while this process held a credential valid until %s: %v", + i, held.ExpiresAt().UTC().Format(time.RFC3339), err) + } + } + if got := source.count(); got > 4 { + t.Errorf("the credential source was reached %d times for %d viewer mints: the viewer's mint is acquiring outside the controller, so the backoff and the held-credential fallback do not apply to the path that mints most", got, viewers) + } +} + +// TestTheBackoffAppliesWhenNothingIsHeld: the quiet window was consulted only +// when a credential WAS in hand, so the one state that should ask least — +// nothing held, issuer failing — asked on every request. The branch meant to +// back off was the branch that hammered. +func TestTheBackoffAppliesWhenNothingIsHeld(t *testing.T) { + // Fails every time, and holds nothing. + source := &unavailableAfter{after: 0} + server := New(Manifest{ID: testSolutionID}).Credential(source) + + const callers = 30 + for range callers { + if err := server.actingForAViewer(context.Background()); err == nil { + t.Fatal("a route acted for a viewer with no credential at all") + } + } + if got := source.count(); got > 3 { + t.Errorf("the source was asked %d times for %d calls with nothing held: each ask that finds a credential due mints, so the state with no credential is the one that must ask least, not most", got, callers) + } +} + +// TestAWaiterReadsItsOwnFlightsResult: waiters read the server's current state +// rather than the flight they waited on, so a caller could act on a later +// flight's answer — or on none, if the next flight had already cleared the +// fields. +func TestAWaiterReadsItsOwnFlightsResult(t *testing.T) { + mint := newHostMint(t, &hostMint{}) + first := mintedCredential(t, mint, "first") + second := mintedCredential(t, mint, "second") + if first.Token() == second.Token() { + t.Fatal("this test needs two distinguishable credentials") + } + + // Slow, so every caller lands on the SAME flight. With an instant source + // several flights happen and different answers are correct, which is why + // the first version of this test was asserting something untrue. + source := &slowHandingSource{ + handing: &handingSource{credentials: []workcontext.Credential{first, second}}, + after: 300 * time.Millisecond, + } + server := New(Manifest{ID: testSolutionID}).Credential(source) + + // Many callers on one flight: every one of them must come back with that + // flight's credential, not with whatever a later flight stored. + const callers = 16 + var wg sync.WaitGroup + got := make([]string, callers) + start := make(chan struct{}) + for i := range callers { + wg.Add(1) + go func(at int) { + defer wg.Done() + <-start + credential, err := server.credentialWithin(context.Background(), 5*time.Second) + if err == nil { + got[at] = credential.Token() + } + }(i) + } + close(start) + wg.Wait() + + for at, token := range got { + if token == "" { + t.Fatalf("caller %d came back with no credential at all: it read shared state a later flight had already cleared", at) + } + } + // One flight, so one answer. + for at, token := range got { + if token != got[0] { + t.Errorf("caller %d came back with a different credential from caller 0, though both waited on the same flight: waiters are reading the server's current state rather than the result of the flight they waited on", at) + } + } +} + +// slowHandingSource is handingSource with a delay, so concurrent callers all +// join one flight. +type slowHandingSource struct { + handing *handingSource + after time.Duration +} + +func (s *slowHandingSource) Credential(ctx context.Context) (workcontext.Credential, error) { + select { + case <-time.After(s.after): + case <-ctx.Done(): + return workcontext.Credential{}, ctx.Err() + } + return s.handing.Credential(ctx) +} + +// driftingSource changes an authority-bearing value while it is being asked, +// so the drift lands after the pre-ask recheck and before the answer is used. +type driftingSource struct { + credential workcontext.Credential + drift func() + once sync.Once +} + +func (d *driftingSource) Credential(context.Context) (workcontext.Credential, error) { + d.once.Do(d.drift) + return d.credential, nil +} + +// TestDriftDuringAnAskIsCaughtAfterIt is the half the pre-ask recheck cannot +// cover, and the half the post-check was skipping. +// +// The post-check was gated on the token changing, so it ran only on a renewal +// that produced a different credential. A drift landing *during* the ask — and +// a first credential, and an ask that returned the one already held — all went +// unchecked until some later renewal happened to change the token. +func TestDriftDuringAnAskIsCaughtAfterIt(t *testing.T) { + fabricator := newHostMint(t, &hostMint{}) + held := mintedCredential(t, fabricator, "held") + + mint := newHostMint(t, &hostMint{}) + bootEnvironment(t, mint) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + // The authority is intact when the ask begins and drifts while it runs, so + // the pre-ask recheck passes and only the one after it can refuse. + drifted := make(chan struct{}) + source := &driftingSource{ + credential: held, + drift: func() { + t.Setenv("CODEFLY__WORKSPACE_CONFIGURATION__MODULE_AUTHORITY__"+AuthorityPrincipalKey, + "spiffe://codefly.test/ns/solutions/sa/somebody-else") + _ = codefly.LoadEnvironmentVariables() + close(drifted) + }, + } + + server := New(Manifest{ID: testSolutionID}) + server.authority = authorityFor(t, ctx) + server.credential = heldToTheFrozenAuthority(source, server.authority) + + _, err := server.credential.Credential(ctx) + <-drifted + if err == nil { + t.Fatal("a credential obtained while an authority-bearing value drifted mid-ask was handed over: the recheck before the ask cannot see a drift that lands after it, so the one after the ask is what catches this — and gating that on the token changing means a first credential, an unchanged one, and a mid-ask drift all go unchecked") + } + if !errors.Is(err, workcontext.ErrMintRefused) { + t.Errorf("the refusal %v is not reported as a judgement, so the boot and the run would classify it differently", err) + } +} + +// authorityFor is this process's frozen authority, read through the SDK the way +// a boot reads it. +func authorityFor(t *testing.T, ctx context.Context) *codefly.Authority { + t.Helper() + server := New(Manifest{ID: testSolutionID}) + if err := server.openAuthority(ctx); err != nil { + t.Fatalf("open the authority: %v", err) + } + return server.authority +} + +// TestTheBootsCredentialIsHeldByTheController: the boot obtained the +// credential and nothing held it, so the first route after boot saw an empty +// controller — it asked again, and got none of the backoff or +// held-credential behaviour the controller exists to provide. +// +// "One credential per execution" starts at the boot's own credential or it +// starts one request late. +func TestTheBootsCredentialIsHeldByTheController(t *testing.T) { + mint := newHostMint(t, &hostMint{}) + bootEnvironment(t, mint) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + server := New(Manifest{ID: testSolutionID}) + ln, err := takeListener(server).start(ctx) + if err != nil { + t.Fatalf("boot: %v", err) + } + _ = ln.Close() + + held, ok := server.heldCredential() + if !ok { + t.Fatal("the boot obtained a credential and the controller holds nothing: the first route after boot therefore asks again, outside the single-flight and the backoff, which is the per-request minting this controller was built to stop") + } + if held.Token() == "" { + t.Fatal("the controller holds a credential with no token") + } + + // And the boot's mint is the only one: a route served immediately after + // boot must not produce a second. + before := mint.count() + if err := server.actingForAViewer(ctx); err != nil { + t.Fatalf("a route was refused immediately after a successful boot: %v", err) + } + if got := mint.count(); got != before { + t.Errorf("the host minted %d more times for the first route after boot: the boot's credential is what that route acts under", got-before) + } +} + +// TestAnAuthorityDriftDuringTheAskIsRefused closes a real gap my own mutation +// ledger found: removing the authority recheck that runs AFTER the ask was +// caught by nothing. +// +// The wrapper rechecks twice, and the two catch different things. The check +// before the ask refuses a value that had already drifted; only the check +// after it refuses a value that drifted *during* the mint — which is the one +// moment a credential could be sealed to values nobody approved and then held +// for its whole life. Every existing test drifted the value before the ask, so +// the pre-check answered all of them and the post-check could be deleted +// silently. That is the sixth time in this review a test has passed through a +// different check than the one it named. +func TestAnAuthorityDriftDuringTheAskIsRefused(t *testing.T) { + authorityValues(t) + ctx := context.Background() + authority, err := readAuthority(ctx) + if err != nil { + t.Fatalf("readAuthority: %v", err) + } + + mint := newHostMint(t, &hostMint{}) + minted := mintedCredential(t, mint, testProjectionAudience) + + // A source that drifts the principal while it is minting: the value is the + // approved one when the ask begins and another when it returns. + drifting := &driftingCredentialSource{t: t, credential: minted} + + _, err = heldToTheFrozenAuthority(drifting, authority).Credential(ctx) + if err == nil { + t.Fatal("a credential minted while an authority-bearing value drifted was accepted: the credential is sealed to the values it was minted under, so a drift that lands during the ask is laundered into one this process then holds for its whole life — the recheck after the ask is the only thing that sees it") + } + if !errors.Is(err, workcontext.ErrMintRefused) { + t.Errorf("the drift was refused with %v, want ErrMintRefused: a drifted authority is a refusal, never a retry", err) + } +} + +// driftingCredentialSource mints normally and changes an authority-bearing +// value as it does, which is the only way to reach the recheck that runs after +// the ask. +type driftingCredentialSource struct { + t *testing.T + credential workcontext.Credential +} + +func (d *driftingCredentialSource) Credential(context.Context) (workcontext.Credential, error) { + d.t.Setenv("CODEFLY__WORKSPACE_CONFIGURATION__MODULE_AUTHORITY__"+AuthorityPrincipalKey, + "spiffe://codefly.test/ns/apps/sa/somebody-else") + if err := codefly.LoadEnvironmentVariables(); err != nil { + return workcontext.Credential{}, err + } + return d.credential, nil +} + +// TestAnAlreadyDriftedAuthorityIsNotMintedAgainst is the pre-ask recheck's own +// property, which the post-ask one masks. +// +// Both rechecks refuse a value that drifted before the ask, so deleting the +// first changed no outcome and no test noticed. What it changes is whether the +// mint happens at all: without it a process whose authority has already +// drifted sends the projected service-account token and spends an audited mint +// to obtain a credential it will then throw away. "Before the ask, because the +// ask is what mints" is the comment on it; this is the test that makes the +// comment load-bearing. +func TestAnAlreadyDriftedAuthorityIsNotMintedAgainst(t *testing.T) { + authorityValues(t) + ctx := context.Background() + authority, err := readAuthority(ctx) + if err != nil { + t.Fatalf("readAuthority: %v", err) + } + mint := newHostMint(t, &hostMint{}) + minted := mintedCredential(t, mint, testProjectionAudience) + + // Drifted before anything asks. + t.Setenv("CODEFLY__WORKSPACE_CONFIGURATION__MODULE_AUTHORITY__"+AuthorityPrincipalKey, + "spiffe://codefly.test/ns/apps/sa/somebody-else") + if err := codefly.LoadEnvironmentVariables(); err != nil { + t.Fatal(err) + } + + counted := &countingCredentialSource{credential: minted} + _, err = heldToTheFrozenAuthority(counted, authority).Credential(ctx) + if err == nil { + t.Fatal("a credential was obtained while an authority-bearing value had already drifted") + } + if !errors.Is(err, workcontext.ErrMintRefused) { + t.Errorf("the drift was refused with %v, want ErrMintRefused", err) + } + if asks := counted.count(); asks != 0 { + t.Errorf("the source was asked %d times while the authority had already drifted: the recheck before the ask exists so that no projected token is sent and no audited mint is spent for a credential that cannot be kept", asks) + } +} + +// countingCredentialSource records how many times it was asked, so a test can +// assert that it was not. +type countingCredentialSource struct { + mu sync.Mutex + asks int + credential workcontext.Credential +} + +func (c *countingCredentialSource) Credential(context.Context) (workcontext.Credential, error) { + c.mu.Lock() + c.asks++ + c.mu.Unlock() + return c.credential, nil +} + +func (c *countingCredentialSource) count() int { + c.mu.Lock() + defer c.mu.Unlock() + return c.asks +} + +// TestAViewerMintNamesTheInstallationItActsUnder is the installation seam. +// +// The host requires an installation on every mint (core's StartInput) and +// refuses one naming none. An organization is not a substitute: one org may +// hold several installations, so a capability minted without naming one is +// attributable to the org and to no deployment inside it. +// +// The viewer's installation comes from the SEAL of the capability they arrived +// with, not from the x-codefly-installation-id header beside it. That order is +// the SDK's own rule, in its words: the installation that governs a call is the +// one inside the signature, and a header is caller-controlled while a seal is +// not. This runtime reads neither itself — SealedInstallation is the SDK's. +func TestAViewerMintNamesTheInstallationItActsUnder(t *testing.T) { + t.Run("the gateway's stamped installation reaches the mint", func(t *testing.T) { + gw := newWorkContextGateway(t, &workContextGateway{}) + solution := serveHandler(t, gw.URL, func(ctx context.Context, g *Gateway) (any, error) { + _, err := g.ForModule(ctx, "documents", Scope{ResourceKind: "documents", Actions: []string{"read"}}) + return nil, err + }) + resp := viewerRequest(t, solution.URL) + _ = resp.Body.Close() + + if got := gw.mintCount(); got != 1 { + t.Fatalf("the viewer's mint ran %d times, want 1", got) + } + mint := <-gw.mints + if mint.InstallationID != corework.FixtureInstallation { + t.Errorf("the mint named installation %q, want %q — the one sealed into the capability the viewer arrived with", + mint.InstallationID, corework.FixtureInstallation) + } + }) + + t.Run("a request with no readable installation mints nothing", func(t *testing.T) { + gw := newWorkContextGateway(t, &workContextGateway{}) + server := New(Manifest{ID: "notes"}).Credential(attestingSource(t)) + server.cfg = config{gatewayURL: gw.URL} + + // Everything a mint needs except an installation: the bearer is an + // opaque string, which is what a caller arriving without a sealed + // capability looks like. + header := http.Header{} + header.Set("authorization", "Bearer an-opaque-session-token") + header.Set(orgHeader, viewerOrg) + header.Set(sessionHeader, viewerSession) + // No installation, which is the point: the gateway stamped none and no + // capability was carried. + + _, err := server.gatewayFor(header).ForModule(context.Background(), "documents", + Scope{ResourceKind: "documents", Actions: []string{"read"}}) + if err == nil { + t.Fatal("a work context was minted for a request naming no installation: the host refuses such a mint, and an org is not an installation") + } + if !strings.Contains(err.Error(), "installation") { + t.Errorf("the refusal %q does not name what is missing", err) + } + // Not a ClientError: no caller can seal an installation, so this is not + // a status a viewer could act on. + var client *ClientError + if errors.As(err, &client) { + t.Errorf("the refusal carries %d, telling the caller to fix something no caller can seal", client.StatusCode) + } + if got := gw.mintCount(); got != 0 { + t.Errorf("the host saw %d mints for a request naming no installation, want 0", got) + } + }) + + t.Run("a caller-supplied capability does not decide the installation", func(t *testing.T) { + // Inbound, x-codefly-work-context is caller-controlled: a browser can + // send one, and this runtime's standing property is that such a + // capability authenticates nothing and is never forwarded. Reading the + // installation out of it would let a caller name the installation + // their own mint is attributed to. + header := http.Header{} + header.Set("authorization", "Bearer an-opaque-session-token") + header.Set(workcontext.InstallationIDHeaderName, testInstallation) + header.Set(workcontext.HeaderName, capability("forged-by-the-caller")) + got, err := viewerInstallation(header) + if err != nil { + t.Fatalf("a request carrying a caller-supplied capability was refused: %v", err) + } + if got != testInstallation { + t.Errorf("installation = %q, want the stamped %q", got, testInstallation) + } + + // And a malformed one cannot decide whether the mint happens either. + header.Set(workcontext.HeaderName, "not-a-capability-at-all") + if got, err := viewerInstallation(header); err != nil || got != testInstallation { + t.Errorf("a caller's malformed capability changed the answer (%q, %v): it is not read at all", got, err) + } + }) + + t.Run("the stamped header is the source, and absent is a refusal", func(t *testing.T) { + // The ordinary inbound shape: the gateway stamps identity and this + // runtime mints the capability itself, so nothing arrives carrying + // one. There the stamped header is the only source there is. + header := http.Header{} + header.Set("authorization", "Bearer an-opaque-session-token") + header.Set(workcontext.InstallationIDHeaderName, "installation-from-a-header") + got, err := viewerInstallation(header) + if err != nil { + t.Fatalf("a stamped installation was refused with no capability carried: %v", err) + } + if got != "installation-from-a-header" { + t.Errorf("installation = %q, want the stamped one", got) + } + + // And neither is a refusal that names both carriers. + bare := http.Header{} + bare.Set("authorization", "Bearer an-opaque-session-token") + _, err = viewerInstallation(bare) + if err == nil { + t.Fatal("a request naming no installation at all was accepted") + } + for _, named := range []string{workcontext.InstallationIDHeaderName, orgHeader, sessionHeader} { + if !strings.Contains(err.Error(), named) { + t.Errorf("the refusal %q does not name %s", err, named) + } + } + }) +} + +// TestARenewalToADifferentInstallationIsRefused: usableCredential refuses a +// credential sealed to NO installation; this refuses one sealed to a DIFFERENT +// installation than the credential this process has been acting under. +// +// That is not a renewal, it is a different identity arriving through the +// renewal path, and every viewer mint names the installation this execution +// acts under — so the mints either side of it would name different deployments +// with nothing recording that they did. +func TestARenewalToADifferentInstallationIsRefused(t *testing.T) { + authorityValues(t) + ctx := context.Background() + authority, err := readAuthority(ctx) + if err != nil { + t.Fatalf("readAuthority: %v", err) + } + mint := newHostMint(t, &hostMint{}) + first := mintedCredential(t, mint, testProjectionAudience) + + source := &movingInstallationSource{credential: first} + held := heldToTheFrozenAuthority(source, authority) + + if _, err := held.Credential(ctx); err != nil { + t.Fatalf("the first credential was refused: %v", err) + } + + // The same process, renewing, and the host answers with a credential + // sealed to another installation. + moved := newHostMint(t, &hostMint{installation: "installation-somewhere-else"}) + source.credential = mintedCredential(t, moved, testProjectionAudience) + _, err = held.Credential(ctx) + if err == nil { + t.Fatal("a renewal sealed to a different installation was accepted: the mints before and after it would name different deployments") + } + if !errors.Is(err, workcontext.ErrMintRefused) { + t.Errorf("the refusal is %v, want ErrMintRefused: the host has answered about a different installation than this build was approved for, which no retry changes", err) + } +} + +// movingInstallationSource hands out whatever credential the test last set, so +// a renewal can answer with one sealed elsewhere. +type movingInstallationSource struct { + mu sync.Mutex + credential workcontext.Credential +} + +func (m *movingInstallationSource) Credential(context.Context) (workcontext.Credential, error) { + m.mu.Lock() + defer m.mu.Unlock() + return m.credential, nil +} + +// expiredCredential is a credential that has already expired in real time, +// without waiting for it. +// +// Minting a short-lived one no longer works: the SDK floors the renewal lead +// at five seconds (minRenewalLeadTime), so it refuses to install a credential +// whose whole remaining lifetime is inside that lead — installing one would +// put the client straight back into a renewal, which is a mint loop and one +// audit event per iteration. That refusal is right, and it means a test cannot +// obtain a nearly-dead credential from the client at all. +// +// So the clock moves instead of the lifetime. MintOptions.Now is the SDK's own +// hook for exactly this: the client mints and installs against a backdated +// clock, where the credential is comfortably valid, and every check in THIS +// runtime reads the real time, where it expired twenty minutes ago. +func expiredCredential(t *testing.T, mint *hostMint, tokenFile string) workcontext.Credential { + t.Helper() + // Both clocks move, not one. The ISSUER stamps not-before and expiry, so + // backdating only the client makes the credential not-yet-valid rather + // than expired; backdating only the issuer makes the client refuse it as + // expired on arrival. Both are held twenty minutes back, where the + // credential is comfortably valid, and every check in THIS runtime reads + // real time, where it expired ten minutes ago. + backdated := time.Now().Add(-20 * time.Minute) + // Its OWN authority, not the shared stand-in. standInAuthority returns a + // singleton, so setting Now on it would backdate every other test's + // issuer too — a fixture that poisons the tests around it is worse than + // the wait this replaces. + _, key := corework.FixtureKeyPair() + mint.authority = &corework.Authority{ + Issuer: corework.FixtureIssuer, + KeyID: corework.FixtureKeyID, + Key: key, + Revisions: corework.FixtureRevisions(), + Seals: corework.FixtureSeals(), + Now: func() time.Time { return backdated }, + } + mint.ttl = 10 * time.Minute + client, err := workcontext.NewMintClient(workcontext.MintOptions{ + URL: mint.URL + credentialMintPath, + Audience: workcontext.AuthorityValue{Name: AuthorityGroup, Key: AuthorityAudienceKey}, + Authority: fixedAuthority{audience: testAudience}, + ProjectedToken: workcontext.ProjectedTokenFile(tokenFile), + ProjectionAudience: testProjectionAudience, + TrustAnchor: mint.roots, + ClientCertificate: func() (*tls.Certificate, error) { return mint.caller, nil }, + AdmittedPeers: func() ([]string, error) { return []string{testGatewayPrincipal}, nil }, + Now: func() time.Time { return backdated }, + }) + if err != nil { + t.Fatal(err) + } + credential, err := client.Credential(context.Background()) + if err != nil { + t.Fatalf("mint a credential against a backdated clock: %v", err) + } + if !time.Now().After(credential.ExpiresAt()) { + t.Fatalf("the credential expires at %s, which is not in the past", credential.ExpiresAt()) + } + return credential +} + +// TestAnExecutionFreeCredentialIsRefusedEverywhere is round eleven's blocker, +// and a regression the sdk-go v0.3.0 migration introduced silently. +// +// usableCredential never checked for a sealed execution and never had to: the +// previous SDK's own reader refused a seal whose build incarnation was zero, +// so a credential reaching it carried one by construction. v0.3.0 moved +// structural validation to core, and core LEGITIMATELY mints a credential +// sealing no execution — for a principal recorded as bearing none, a person at +// a terminal. That is core's own execution-missing shape. The guarantee was +// inherited, the inheritance ended, and nothing here noticed. +// +// Reproduced before it was fixed: the runtime booted on such a credential, +// held it, passed actingForAViewer, and served handlers with a credential +// attesting no approved build. +// +// A workload credential is the one kind that may never be execution-free: its +// purpose is attesting which build is asking, so a host can hold a mint to the +// build its presence document approved. +func TestAnExecutionFreeCredentialIsRefusedEverywhere(t *testing.T) { + t.Run("the boot refuses it, terminally", func(t *testing.T) { + mint := newHostMint(t, &hostMint{executionFree: true}) + bootEnvironment(t, mint) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + server := New(Manifest{ID: testSolutionID}) + // Short, so a refusal that were wrongly classified as transient shows + // up as a bounded wait rather than hanging this test. + server.firstMintWindow = 2 * time.Second + ln, err := takeListener(server).start(ctx) + if ln != nil { + _ = ln.Close() + } + if err == nil { + t.Fatal("the boot accepted a credential sealing no execution: it attests no approved build, so the host has nothing to hold a mint to and the ceiling it is held to is nobody's") + } + if !errors.Is(err, workcontext.ErrMintRefused) { + t.Errorf("the boot failed with %v, want ErrMintRefused: the issuer will seal the same thing next time, so this is a judgement and not an outage", err) + } + if !strings.Contains(err.Error(), "seals no execution") { + t.Errorf("the refusal %q does not say what is missing", err) + } + // Terminal means it asked ONCE. A bounded wait here would be the + // audited-mint loop this runtime was changed to remove. + if got := mint.count(); got != 1 { + t.Errorf("the boot asked the issuer %d times, want exactly 1", got) + } + }) + + t.Run("no handler runs and no module is called", func(t *testing.T) { + // The route gate's half: even handed such a credential directly, a + // route must not act for a viewer and nothing author-written may run. + mint := newHostMint(t, &hostMint{executionFree: true}) + tokenFile := filepath.Join(t.TempDir(), "token") + writeFile(t, tokenFile, "projected") + gw := newWorkContextGateway(t, &workContextGateway{}) + + var ran atomic.Bool + server := New(Manifest{ID: testSolutionID}). + Credential(mintClientFor(t, mint, tokenFile)) + server.cfg = config{gatewayURL: gw.URL} + // wrapRequest is what Serve mounts each route through + // (mux.HandleFunc(path, withCORS(s.wrapRequest(handler)))), so this + // drives the route's own gate rather than calling the gate directly. + route := server.wrapRequest(func(*http.Request, *Gateway) (any, error) { + ran.Store(true) + return map[string]string{"ok": "yes"}, nil + }) + + header := http.Header{} + header.Set("authorization", viewerBearer()) + header.Set(orgHeader, viewerOrg) + header.Set(sessionHeader, viewerSession) + header.Set(workcontext.InstallationIDHeaderName, testInstallation) + + // THROUGH THE REGISTERED ROUTE, not by calling the gate. + // + // This called actingForAViewer directly, which proves the gate + // refuses and nothing about whether a route consults it: removing the + // gate from the route left the handler uncalled and every count at + // zero, because nothing ever reached the route at all. A reviewer + // named it twice. + recorder := httptest.NewRecorder() + request := httptest.NewRequest(http.MethodGet, "/thing", nil) + request.Header = header + route(recorder, request) + + if recorder.Code != http.StatusServiceUnavailable { + t.Errorf("the route answered %d under a credential sealing no execution, want 503: it attests no approved build, so there is nothing for a host to hold a mint to", recorder.Code) + } + if ran.Load() { + t.Error("the handler ran under a credential sealing no execution: the refusal has to come before anything author-written") + } + if got := gw.mintCount(); got != 0 { + t.Errorf("the gateway saw %d viewer mints, want 0", got) + } + if got := len(gw.calls); got != 0 { + t.Errorf("a module was called %d times under a credential attesting no build, want 0", got) + } + + // And the gate itself, named, because the route's 503 does not say + // WHICH refusal produced it. + if err := server.actingForAViewer(context.Background()); err == nil { + t.Fatal("a route acted for a viewer under a credential sealing no execution") + } else if !errors.Is(err, ErrCredentialRefused) { + t.Errorf("the route refused with %v, want one wrapping ErrCredentialRefused", err) + } + + // THE POSITIVE CONTROL: the same route, the same headers, a credential + // that does seal an execution. Without it the 503 above is satisfied + // by a route that answers 503 to everything. + sound := newHostMint(t, &hostMint{}) + healthy := New(Manifest{ID: testSolutionID}). + Credential(mintClientFor(t, sound, tokenFile)) + healthy.cfg = config{gatewayURL: gw.URL} + controlRoute := healthy.wrapRequest(func(*http.Request, *Gateway) (any, error) { + return map[string]string{"ok": "yes"}, nil + }) + control := httptest.NewRecorder() + controlRequest := httptest.NewRequest(http.MethodGet, "/thing", nil) + controlRequest.Header = header + controlRoute(control, controlRequest) + if control.Code != http.StatusOK { + t.Errorf("the control route answered %d with a sound credential, want 200: so the 503 above says nothing about the execution-free one", control.Code) + } + }) + + t.Run("the carrier still accepts it as structurally valid", func(t *testing.T) { + // The division of labour stays intact. An execution-free capability is + // not malformed — core mints it on purpose for a principal bearing no + // execution — so the CARRIER must not refuse it. What refuses it is + // this runtime holding its OWN credential to a higher bar than it + // holds a capability it merely carries. + if _, classified := boundary["execution-missing"]; !classified { + t.Fatal("core's execution-missing fixture is unclassified, so this says nothing") + } + if boundary["execution-missing"] != carried { + t.Error("the carrier refuses core's execution-missing fixture: a capability core mints legitimately is not the carrier's to reject, and refusing it here would be the silent-downgrade failure this boundary exists to prevent") + } + }) +} + +// TestTheMintHopIsHeldToTheSameThingsTheGatewayHopIs is round thirteen's +// blocker, closed where it had to be closed: in the SDK. +// +// Round eleven asked for this and I could not do it. v0.3.0 took RootCAs as a +// VALUE and built its transport from it on the spot, presented no client +// certificate, and offered no peer hook — so the mint hop had no identity of +// its own going out, admitted any peer the anchor signed, and fixed its anchor +// for the client's lifetime. Reimplementing the transport here would have been +// the hack this repo's rule 2 forbids, so it went out as an sdk-go issue and +// came back as sdk-go#51: three READERS, each consulted during the handshake. +// +// So these are the negative controls round thirteen named. Each is about the +// mint hop specifically, driven through a real TLS handshake against the fake +// endpoint rather than by inspecting the options struct. +func TestTheMintHopIsHeldToTheSameThingsTheGatewayHopIs(t *testing.T) { + t.Run("an unreadable anchor refuses rather than dialling under the last good one", func(t *testing.T) { + mint := newHostMint(t, &hostMint{}) + tokenFile := filepath.Join(t.TempDir(), "token") + writeFile(t, tokenFile, "projected") + + var fail atomic.Bool + source := mintClientWith(t, mint, tokenFile, func(o *workcontext.MintOptions) { + o.TrustAnchor = func() (*x509.CertPool, error) { + if fail.Load() { + return nil, errors.New("the projected bundle is unreadable") + } + pool, _ := mint.roots() + return pool, nil + } + }) + held, err := source.Credential(context.Background()) + if err != nil { + t.Fatalf("the first ask was refused: %v", err) + } + // The anchor goes away. A client that captured the last good pool goes + // on dialling the endpoint that receives the projected + // service-account token under trust that no longer exists. + fail.Store(true) + if _, err := freshMint(t, source, held); err == nil { + t.Fatal("a mint succeeded while the trust anchor could not be read: the projected service-account token went to an endpoint verified against nothing this process can currently vouch for") + } + }) + + t.Run("a peer the anchor signed but the peer set does not name is refused", func(t *testing.T) { + // THE finding. Every workload in the trust domain holds a certificate + // from this anchor, the consumed modules included, so "signed by the + // right CA" is authentication and not authorisation. Before sdk-go#51 + // a neighbouring workload presenting a certificate valid for the + // mint's hostname received the projected token. + mint := newHostMint(t, &hostMint{}) + tokenFile := filepath.Join(t.TempDir(), "token") + writeFile(t, tokenFile, "projected") + + source := mintClientWith(t, mint, tokenFile, func(o *workcontext.MintOptions) { + // The endpoint's certificate is valid and issued by the accepted + // anchor. It simply is not the party this runtime talks to. + o.AdmittedPeers = func() ([]string, error) { return []string{rivalPrincipal}, nil } + }) + if _, err := source.Credential(context.Background()); err == nil { + t.Fatal("the mint was dialled and the projected service-account token handed over to a peer the admitted set does not name: a certificate from the cell's anchor says which trust domain answered, not which party") + } else if mint.count() != 0 { + t.Errorf("the endpoint answered %d requests before the identity was refused, want 0: the refusal has to happen in the handshake, not after the token is on the wire", mint.count()) + } + }) + + t.Run("peer withdrawal takes effect with the anchor unchanged", func(t *testing.T) { + // The second half of the same finding, and the one an anchor check + // cannot do: the CA never changes here. Only the provisioned set does. + mint := newHostMint(t, &hostMint{}) + tokenFile := filepath.Join(t.TempDir(), "token") + writeFile(t, tokenFile, "projected") + + var withdrawn atomic.Bool + source := mintClientWith(t, mint, tokenFile, func(o *workcontext.MintOptions) { + o.AdmittedPeers = func() ([]string, error) { + if withdrawn.Load() { + return []string{rivalPrincipal}, nil + } + return []string{testGatewayPrincipal}, nil + } + }) + held, err := source.Credential(context.Background()) + if err != nil { + t.Fatalf("the first ask was refused: %v", err) + } + withdrawn.Store(true) + if _, err := freshMint(t, source, held); err == nil { + t.Fatal("a mint succeeded after the endpoint's identity was withdrawn from the admitted set while its certificate and the anchor stayed exactly as they were: this is the revocation an anchor check cannot express, and the reason the set is read per handshake rather than once") + } + }) + + t.Run("the mint request presents this workload's own identity", func(t *testing.T) { + // The fixture REQUIRES a client certificate now, so a mint that + // completes at all proves one was presented — and the identity is + // asserted rather than expected to be empty, which is what the + // fixture was weakened to expect while the SDK presented nothing. + mint := newHostMint(t, &hostMint{}) + tokenFile := filepath.Join(t.TempDir(), "token") + writeFile(t, tokenFile, "projected") + + if _, err := mintClientFor(t, mint, tokenFile).Credential(context.Background()); err != nil { + t.Fatalf("the mint refused a caller presenting this workload's certificate: %v", err) + } + if len(mint.callers) == 0 { + t.Fatal("the endpoint recorded no caller at all") + } + if got := mint.callers[0]; got != testPrincipal { + t.Errorf("the mint endpoint saw the caller as %q, want %q: the projected service-account token is no longer the whole of what attests which workload is asking", got, testPrincipal) + } + }) + + t.Run("a certificate reader that cannot answer refuses the mint", func(t *testing.T) { + mint := newHostMint(t, &hostMint{}) + tokenFile := filepath.Join(t.TempDir(), "token") + writeFile(t, tokenFile, "projected") + + source := mintClientWith(t, mint, tokenFile, func(o *workcontext.MintOptions) { + o.ClientCertificate = func() (*tls.Certificate, error) { + return nil, errors.New("the projected pair is half-written") + } + }) + if _, err := source.Credential(context.Background()); err == nil { + t.Fatal("the mint was dialled with no certificate to present: an endpoint that requires one would refuse, and one that does not would attribute this request to nobody") + } + }) +} + +// freshMint forces a NEW handshake against the endpoint. +// +// Calling Credential again would not: a mint client holds the credential it +// has until renewal is due, which is the one-per-execution property this +// runtime wants everywhere else, so "ask again" dials nothing and a withdrawal +// test that re-asked would pass against a client that never re-checked +// anything. Refresh is the call that re-mints, so it is the call that reaches +// the readers under test. +func freshMint(t *testing.T, source CredentialSource, held workcontext.Credential) (workcontext.Credential, error) { + t.Helper() + client, ok := source.(*workcontext.MintClient) + if !ok { + t.Fatalf("the source under test is %T, not the SDK's mint client, so this would not exercise the transport at all", source) + } + return client.Refresh(context.Background(), held) +} + +// mintClientWith is a mint client against the fake endpoint, with one or more +// of the three handshake readers replaced. +func mintClientWith(t *testing.T, mint *hostMint, tokenFile string, with ...func(*workcontext.MintOptions)) CredentialSource { + t.Helper() + options := workcontext.MintOptions{ + URL: mint.URL + credentialMintPath, + Audience: workcontext.AuthorityValue{Name: AuthorityGroup, Key: AuthorityAudienceKey}, + Authority: fixedAuthority{audience: testAudience}, + ProjectedToken: workcontext.ProjectedTokenFile(tokenFile), + ProjectionAudience: testProjectionAudience, + TrustAnchor: mint.roots, + ClientCertificate: func() (*tls.Certificate, error) { return mint.caller, nil }, + AdmittedPeers: func() ([]string, error) { return []string{testGatewayPrincipal}, nil }, + } + for _, apply := range with { + apply(&options) + } + client, err := workcontext.NewMintClient(options) + if err != nil { + t.Fatal(err) + } + return client +} + +// TestARenewalSealedToADifferentExecutionIsRefused is the half of round +// eleven's blocker my own fix did not cover. +// +// usableCredential refuses a credential sealed to no installation and one +// sealed to no execution. This is the other axis: a renewal that is +// well-formed, carries both, and carries DIFFERENT ones. The executed round +// found an installation move caught and a build move — incarnation 11 to 99 — +// accepted silently, which is a different execution's credential arriving +// through the renewal path. +// +// It matters for the reason the installation does: every work context this +// process mints for a viewer is attested by this credential, so the mints +// either side of a silent change name a different deployment or a different +// build — including one the host's approved-build ceiling was never applied +// to. +func TestARenewalSealedToADifferentExecutionIsRefused(t *testing.T) { + authorityValues(t) + ctx := context.Background() + authority, err := readAuthority(ctx) + if err != nil { + t.Fatalf("readAuthority: %v", err) + } + tokenFile := filepath.Join(t.TempDir(), "token") + writeFile(t, tokenFile, "projected") + + for _, tc := range []struct { + name string + moved *hostMint + }{ + {"a different build incarnation", &hostMint{incarnation: 99}}, + // A new digest comes with a new incarnation, because core refuses it + // otherwise: "a new build is a new run, so advance the incarnation — + // reusing it lets a swap back to the earlier digest re-admit every + // capability sealed to it". So this is the realistic redeploy shape + // rather than a digest swapped under a fixed incarnation. + {"a different image digest", &hostMint{digest: "sha256:" + strings.Repeat("b", 64), incarnation: 12}}, + {"a different installation", &hostMint{installation: "installation-somewhere-else"}}, + } { + t.Run(tc.name, func(t *testing.T) { + first := newHostMint(t, &hostMint{}) + source := &movingInstallationSource{credential: mintedCredential(t, first, testProjectionAudience)} + held := heldToTheFrozenAuthority(source, authority) + if _, err := held.Credential(ctx); err != nil { + t.Fatalf("the first credential was refused: %v", err) + } + + // The same process, renewing, and the issuer answers with a + // credential sealed to a different execution. + moved := newHostMint(t, tc.moved) + source.credential = mintedCredential(t, moved, testProjectionAudience) + _, err := held.Credential(ctx) + if err == nil { + t.Fatal("a renewal sealed to a different execution was accepted: the mints before and after it would be attributed to different builds, and the ceiling the host approved for one would have been applied to the other") + } + if !errors.Is(err, workcontext.ErrMintRefused) { + t.Errorf("the refusal is %v, want ErrMintRefused: a process whose approved build has moved under it should end and be restarted against the delivery as it now stands", err) + } + if !strings.Contains(err.Error(), "different execution") { + t.Errorf("the refusal %q does not say what changed", err) + } + }) + } + + t.Run("an unchanged renewal is accepted", func(t *testing.T) { + // The control: the freeze must not refuse an ordinary renewal, which + // is the mistake that would turn this into a crash loop. + mint := newHostMint(t, &hostMint{}) + source := &movingInstallationSource{credential: mintedCredential(t, mint, testProjectionAudience)} + held := heldToTheFrozenAuthority(source, authority) + if _, err := held.Credential(ctx); err != nil { + t.Fatalf("the first credential was refused: %v", err) + } + source.credential = mintedCredential(t, mint, testProjectionAudience) + if _, err := held.Credential(ctx); err != nil { + t.Fatalf("an ordinary renewal from the same issuer was refused: %v", err) + } + }) +} + +// TestTheMintRefusesToPresentARotatedIdentity is round fourteen's blocker, on +// the production path rather than through a hand-built client. +// +// platformCredentialSource handed outboundLeaf — the SDK's reloader over the +// projected files — straight to MintOptions.ClientCertificate. The gateway's +// configuration gets holdPresentedCertificate; this reader got nothing. So +// after a same-CA rotation to a neighbouring workload's certificate, the mint +// hop presented THAT identity while the frozen principal was still this one, +// and the SDK cannot catch it: it checks that a certificate is non-empty and +// has a key, which is the right division of labour and not a statement about +// whose certificate it is. +// +// The three steps the reviewer asked for, in order: A mints; a rotation to B +// produces NO mint request; a fresh certificate for A mints again. The middle +// one is the finding — and "no request" rather than "an error" is the +// assertion, because the refusal has to happen before the projected +// service-account token is on the wire. +func TestTheMintRefusesToPresentARotatedIdentity(t *testing.T) { + mint := newHostMint(t, &hostMint{}) + // The boot's own environment and projected material, under the anchor the + // fake host is served with — this test is about the production path, so it + // uses what a boot uses rather than a hand-built client. + certFile, keyFile, bundleFile, _, _, c := bootIdentityIn(t, mint, testPrincipal) + tokenFile := filepath.Join(t.TempDir(), "token") + writeFile(t, tokenFile, "projected") + + server := New(Manifest{ID: testSolutionID}) + server.cfg = config{ + identityCertFile: certFile, + identityKeyFile: keyFile, + trustBundleFile: bundleFile, + allowedCallersFile: identitiesFile(t, testGatewayPrincipal), + mintPeersFile: identitiesFile(t, testGatewayPrincipal), + gatewayPeersFile: identitiesFile(t, testGatewayPrincipal), + mintURL: mint.URL + credentialMintPath, + projectedTokenPath: tokenFile, + } + server.principal = testPrincipal + // The frozen authority, read through the SDK the way a boot reads it. + server.authority = authorityFor(t, context.Background()) + source, err := server.platformCredentialSource() + if err != nil { + t.Fatalf("configure the production credential source: %v", err) + } + + // A mints. + held, err := source.Credential(context.Background()) + if err != nil { + t.Fatalf("the mint refused this workload's own identity: %v", err) + } + if got := mint.count(); got != 1 { + t.Fatalf("the issuer served %d requests for the first credential, want 1", got) + } + + // rotate replaces the projected pair in place, as a projected Secret + // update does, and advances the modification time so the SDK's reloader + // observes it (it refreshes on mtime, per handshake). + rotate := func(spiffeID string) { + t.Helper() + leafPEM, keyPEM, _ := issueLeaf(t, c.anchor, c.anchorKey, spiffeID) + writeFile(t, certFile, string(leafPEM)) + writeFile(t, keyFile, string(keyPEM)) + ahead := time.Now().Add(time.Minute) + for _, path := range []string{certFile, keyFile} { + if err := os.Chtimes(path, ahead, ahead); err != nil { + t.Fatal(err) + } + } + } + + // THE FINDING: a valid, same-CA certificate for another workload. + rotate(rivalPrincipal) + before := mint.count() + if _, err := freshMint(t, source, held); err == nil { + t.Fatal("the mint was dialled presenting a certificate issued for another workload: the host holds this process to the identity its presence document approved, and this hop is the one that carries the projected service-account token") + } else if !strings.Contains(err.Error(), "another workload") && !strings.Contains(err.Error(), "issued for") { + t.Errorf("the refusal %q does not say the certificate names another workload", err) + } + if got := mint.count(); got != before { + t.Errorf("the issuer served %d more requests while this workload was presenting another identity: the refusal has to come before the projected service-account token is on the wire, not after", got-before) + } + + // And a rotation that is ordinary — a fresh certificate for the same + // principal — still works, so the check is about the identity and not + // about the pair having changed. + rotate(testPrincipal) + // Through the SDK's hold-off, not around it. + // + // The refusal above is a failed ask, and the mint client backs off after + // one — "held off until …" — so the next attempt is refused by the backoff + // rather than by anything about the certificate. That is correct behaviour + // and it is also the reason this step waits: asserting the recovery + // against a brand-new client would prove that a fresh client works, which + // is not what is in question. + var recovered error + deadline := time.Now().Add(30 * time.Second) + for { + if _, recovered = freshMint(t, source, held); recovered == nil { + break + } + if !strings.Contains(recovered.Error(), "held off until") || time.Now().After(deadline) { + t.Fatalf("an ordinary rotation to a fresh certificate for this workload was refused, which would make every projected-pair renewal fatal: %v", recovered) + } + time.Sleep(250 * time.Millisecond) + } + if got := mint.count(); got <= before { + t.Errorf("the issuer served %d requests after a valid rotation, want more than %d", got, before) + } +} + +// TestEachSealedFieldIsComparedOnItsOwn isolates the three comparisons that +// hold this execution's credential to what it was first sealed to. +// +// TestARenewalSealedToADifferentExecutionIsRefused drives real credentials, and +// its digest case advances the incarnation as well — not an oversight but +// core's own rule: a new digest at a reused incarnation is refused outright +// ("a new build is a new run, so advance the incarnation — reusing it lets a +// swap back to the earlier digest re-admit every capability sealed to it", +// core@v0.9.0/workcontext/seal.go:861). So a credential differing ONLY in +// digest is not a thing core will mint, and a reviewer's suggested fixture +// cannot be built without inventing a shape core refuses. +// +// What can be isolated is the comparison, which is where the finding actually +// lands: removing the digest field from sealedIdentity leaves the integration +// test green because the incarnation moved too. This drives +// holdSealedIdentity directly, one field at a time. +func TestEachSealedFieldIsComparedOnItsOwn(t *testing.T) { + const digest = "sha256:" + "a" + seal := func(installation, image string, incarnation uint64) *workcontext.SealedValues { + // InstallationId is a plain string on the message; the other two are + // pointer scalars, which is why the production code reads all three + // through the Get accessors rather than the fields. + return &workcontext.SealedValues{ + InstallationId: installation, + ImageDigest: &image, + BuildIncarnation: &incarnation, + } + } + first := seal(testInstallation, digest, 11) + + for _, tc := range []struct { + field string + moved *workcontext.SealedValues + }{ + {"the image digest alone", seal(testInstallation, "sha256:b", 11)}, + {"the build incarnation alone", seal(testInstallation, digest, 12)}, + {"the installation alone", seal("installation-somewhere-else", digest, 11)}, + } { + t.Run(tc.field, func(t *testing.T) { + held := &authorityHeldSource{} + if err := held.holdSealedIdentity(first); err != nil { + t.Fatalf("the first seal was refused: %v", err) + } + err := held.holdSealedIdentity(tc.moved) + if err == nil { + t.Fatalf("a renewal that moved %s was accepted: every work context this process mints is attested by this credential, so the mints either side would name a different deployment or a different build", tc.field) + } + if !errors.Is(err, workcontext.ErrMintRefused) { + t.Errorf("the refusal is %v, want ErrMintRefused", err) + } + }) + } + + t.Run("and the same seal twice is a renewal", func(t *testing.T) { + // The control: this refuses on CHANGE, not on every renewal. + held := &authorityHeldSource{} + if err := held.holdSealedIdentity(first); err != nil { + t.Fatalf("the first seal was refused: %v", err) + } + if err := held.holdSealedIdentity(seal(testInstallation, digest, 11)); err != nil { + t.Errorf("an ordinary renewal to the same execution was refused, which would end every process at its first renewal: %v", err) + } + }) +} + +// TestTheCredentialBackoffGrowsAndBreaksPhase is round sixteen's second major. +// +// The quiet period after a failed acquisition was a flat second, reset to the +// same value every time. Under sustained traffic against a fast-failing issuer +// that is one audited mint attempt per replica per second for the length of the +// outage — and every replica on the same period, so they arrive together. +// Single-flight bounds how many asks one replica makes at once; it says +// nothing about the rate, and nothing about the fleet. +func TestTheCredentialBackoffGrowsAndBreaksPhase(t *testing.T) { + // It grows, and it is capped. + previous := time.Duration(0) + for failures := 1; failures <= 12; failures++ { + quiet := credentialAskQuiet(failures) + if quiet <= 0 { + t.Fatalf("after %d failures the quiet period is %s, which asks again immediately", failures, quiet) + } + if quiet > credentialAskBackoffCeiling { + t.Errorf("after %d failures the quiet period is %s, past the %s ceiling: a backoff longer than a renewal lead would stop a held credential renewing on its own schedule", + failures, quiet, credentialAskBackoffCeiling) + } + if failures <= 5 && failures > 1 && quiet <= previous { + t.Errorf("the quiet period did not grow between %d and %d failures (%s then %s): a flat period is one attempt per replica per interval for as long as the outage lasts", + failures-1, failures, previous, quiet) + } + previous = quiet + } + // The ceiling is reached and held. + if got := credentialAskQuiet(20); got > credentialAskBackoffCeiling || got < credentialAskBackoffCeiling/2 { + t.Errorf("after 20 failures the quiet period is %s, want it pinned near the %s ceiling", got, credentialAskBackoffCeiling) + } + // And it is jittered, so replicas that failed together stop being in + // phase. Without jitter every one of these is identical. + seen := map[time.Duration]bool{} + for range 40 { + seen[credentialAskQuiet(8)] = true + } + if len(seen) < 2 { + t.Errorf("40 backoffs at the same failure count produced %d distinct value(s): un-jittered, every replica retries on the same tick and an issuer outage becomes an issuer stampede", len(seen)) + } + // The jitter only ever shortens, so the ceiling is a real ceiling. + for quiet := range seen { + if quiet > credentialAskBackoffCeiling { + t.Errorf("jitter produced %s, past the %s ceiling", quiet, credentialAskBackoffCeiling) + } + } +} + +// TestASuccessfulAcquisitionResetsTheBackoff: an outage that ends must not +// leave the next failure starting from the ceiling. +func TestASuccessfulAcquisitionResetsTheBackoff(t *testing.T) { + server := New(Manifest{ID: testSolutionID}) + server.credentialAskFailures = 7 + mint := newHostMint(t, &hostMint{}) + tokenFile := filepath.Join(t.TempDir(), "token") + writeFile(t, tokenFile, "projected") + server.credential = mintClientFor(t, mint, tokenFile) + + if _, err := server.acquireCredential(context.Background()); err != nil { + t.Fatalf("the acquisition failed, so this says nothing about the reset: %v", err) + } + if got := server.credentialAskFailures; got != 0 { + t.Errorf("after a successful acquisition the failure count is %d, want 0: an issuer that recovers and fails again would start from the ceiling rather than from a second", got) + } + if !server.credentialQuietUntil.IsZero() { + t.Errorf("a successful acquisition left a quiet period until %s, so the next ask is held off for no reason", server.credentialQuietUntil) + } +} diff --git a/deployed_registration_test.go b/deployed_registration_test.go deleted file mode 100644 index 8dcf95c..0000000 --- a/deployed_registration_test.go +++ /dev/null @@ -1,307 +0,0 @@ -package solution - -import ( - "context" - "io" - "net" - "net/http" - "net/http/httptest" - "os" - "path/filepath" - "strconv" - "strings" - "testing" - "testing/fstest" - "time" -) - -// clearSelfEnvironment unsets every carrier these tests key on, so a variable -// exported in the shell running the suite cannot decide an assertion. -func clearSelfEnvironment(t *testing.T) { - t.Helper() - for _, key := range []string{"SELF_UPSTREAM", "PUBLIC_URL", "CODEFLY__RUNTIME_CONTEXT", - "CODEFLY__MODULE", "CODEFLY__SERVICE", "CODEFLY__ENVIRONMENT", - "CODEFLY__SELF_ENDPOINT__LASTLOGIN_GO__BACKEND__HTTP__HTTP"} { - t.Setenv(key, "") - } -} - -// TestSelfUpstreamIsTheReachableSelfEndpoint proves the upstream registered -// with the gateway is the address Codefly injects for reaching this service, -// not its own listen address. A deployed solution registered -// "http://localhost:8080" — its listen address, via the public URL default — -// and the gateway, dialling its own localhost, proxied the solution to itself. -func TestSelfUpstreamIsTheReachableSelfEndpoint(t *testing.T) { - const reachable = "http://backend.solutions-lastlogin.svc.cluster.local:8080" - for _, tc := range []struct { - name string - carrier string - override string - want string - }{ - {name: "the injected self endpoint", carrier: reachable, want: reachable}, - {name: "the listen address when none is injected", want: "http://localhost:8080"}, - {name: "an explicit override over the carrier", carrier: reachable, override: "http://upstream.example.com:9000/", want: "http://upstream.example.com:9000"}, - } { - t.Run(tc.name, func(t *testing.T) { - clearSelfEnvironment(t) - t.Setenv("PORT", "8080") - t.Setenv("CODEFLY__MODULE", "lastlogin-go") - t.Setenv("CODEFLY__SERVICE", "backend") - // The key is normalised as CODEFLY__ENDPOINT__ keys are: upper - // case, dashes to underscores. - t.Setenv("CODEFLY__SELF_ENDPOINT__LASTLOGIN_GO__BACKEND__HTTP__HTTP", tc.carrier) - t.Setenv("SELF_UPSTREAM", tc.override) - cfg := loadConfig(context.Background(), "lastlogin-go") - if cfg.selfUpstream != tc.want { - t.Fatalf("selfUpstream = %q, want %q", cfg.selfUpstream, tc.want) - } - // PUBLIC_URL is the browser's origin, not the gateway's upstream: - // it no longer feeds the registered upstream at all. - t.Setenv("PUBLIC_URL", "https://solutions.example.com") - if got := loadConfig(context.Background(), "lastlogin-go").selfUpstream; got != tc.want { - t.Fatalf("with PUBLIC_URL set, selfUpstream = %q, want %q", got, tc.want) - } - }) - } -} - -// TestGatewayRegistrationCarriesTheReachableUpstream proves the carrier reaches -// the wire: the body the gateway is sent names the injected self endpoint. -func TestGatewayRegistrationCarriesTheReachableUpstream(t *testing.T) { - const reachable = "http://backend.solutions-lastlogin.svc.cluster.local:8080" - bodies := make(chan string, 16) - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.URL.Path == solutionRegisterPath { - body, _ := io.ReadAll(r.Body) - send(bodies, string(body)) - } - answeringTheExchange(w, r) - })) - defer srv.Close() - ln, err := net.Listen("tcp", "127.0.0.1:0") - if err != nil { - t.Fatal(err) - } - s := New(Manifest{ID: "lastlogin-go", Title: "Last Login"}) - s.cfg = config{ - port: strconv.Itoa(ln.Addr().(*net.TCPAddr).Port), - hostRegisterURL: srv.URL + "/api/solutions/register", - gatewayRegisterURL: srv.URL + solutionRegisterPath, - moduleRegisterURL: srv.URL + moduleRegisterPath, - moduleTokenURL: srv.URL + moduleRegistrationTokenPath, - solutionTokenURL: srv.URL + solutionRegistrationTokenPath, - internalToken: internalTokenTest, - solutionSecret: "s3cret", - selfUpstream: reachable, - } - ctx, cancel := context.WithCancel(context.Background()) - done := make(chan struct{}) - go func() { _ = s.serve(ctx, ln); close(done) }() - defer func() { cancel(); <-done }() - - select { - case body := <-bodies: - if !strings.Contains(body, `"upstream":"`+reachable+`"`) { - t.Fatalf("gateway registration body = %s, want upstream %q", body, reachable) - } - case <-time.After(5 * time.Second): - t.Fatal("no gateway registration within timeout") - } -} - -// TestValidateRefusesLoopbackInADeployedRuntime proves a deployed process -// cannot boot registering an address only it can reach — and that the signal -// is the explicit runtime context, never the environment's name. -func TestValidateRefusesLoopbackInADeployedRuntime(t *testing.T) { - base := config{ - port: "8080", - gatewayURL: "http://gateway:42152", - hostRegisterURL: "http://frontend:21931/api/solutions/register", - gatewayRegisterURL: "http://gateway:42152/solutions/_register", - moduleRegisterURL: "http://gateway:42152/modules/_register", - moduleTokenURL: "http://gateway:42152/modules/_registration-token", - solutionTokenURL: "http://gateway:42152/solutions/_registration-token", - solutionSecret: "s3cret", - selfUpstream: "http://backend.solutions-lastlogin.svc.cluster.local:8080", - } - loopbacks := []string{"http://localhost:8080", "http://127.0.0.1:8080", "http://[::1]:8080", - "http://0.0.0.0:8080", "http://app.localhost:8080", "http://LOCALHOST.:8080"} - - for _, upstream := range loopbacks { - c := base - c.runtimeContext = "kubernetes" - c.selfUpstream = upstream - err := c.validate() - if err == nil || !strings.Contains(err.Error(), "self upstream") || !strings.Contains(err.Error(), selfEndpointCarrierPrefix) { - t.Errorf("deployed with upstream %q: validate() = %v, want a refusal naming the self upstream and its carrier", upstream, err) - } - // Every runtime context `codefly run` uses locally accepts it: there - // the gateway shares the machine. - for _, local := range []string{"", "native", "nix", "container", "free"} { - c.runtimeContext = local - if err := c.validate(); err != nil { - t.Errorf("runtime context %q with upstream %q: validate() = %v, want nil", local, upstream, err) - } - } - } - - c := base - c.runtimeContext = "kubernetes" - c.publicURL = "http://localhost:8080" - if err := c.validate(); err == nil || !strings.Contains(err.Error(), "PUBLIC_URL") { - t.Errorf("deployed with a loopback PUBLIC_URL: validate() = %v, want a refusal naming PUBLIC_URL", err) - } - c.publicURL = "https://solutions.example.com" - if err := c.validate(); err != nil { - t.Errorf("deployed with a reachable upstream and public URL: validate() = %v, want nil", err) - } - c.selfUpstream = "/relative" - if err := c.validate(); err == nil || !strings.Contains(err.Error(), "SELF_UPSTREAM") { - t.Errorf("relative upstream: validate() = %v, want a refusal naming SELF_UPSTREAM", err) - } -} - -// TestLoopbackRefusalIgnoresTheEnvironmentName proves the environment's name -// decides nothing: an environment named like a local one still refuses a -// loopback upstream when the runtime context says it is deployed, and one -// named like a deployment accepts it when nothing says so. -func TestLoopbackRefusalIgnoresTheEnvironmentName(t *testing.T) { - for _, tc := range []struct { - environment, runtimeContext string - refused bool - }{ - {environment: "local-dogfood", runtimeContext: "kubernetes", refused: true}, - {environment: "local", runtimeContext: "kubernetes", refused: true}, - {environment: "staging", runtimeContext: "native", refused: false}, - {environment: "local-dogfood", runtimeContext: "container", refused: false}, - } { - clearSelfEnvironment(t) - t.Setenv("PORT", "8080") - t.Setenv("GATEWAY_URL", "http://gateway:42152") - t.Setenv("HOST_REGISTER_URL", "http://frontend:21931/api/solutions/register") - t.Setenv(SolutionRegistrationSecretEnvironmentVariable, "s3cret") - t.Setenv("CODEFLY__ENVIRONMENT", tc.environment) - t.Setenv("CODEFLY__RUNTIME_CONTEXT", tc.runtimeContext) - err := loadConfig(context.Background(), "lastlogin-go").validate() - if refused := err != nil && strings.Contains(err.Error(), "loopback"); refused != tc.refused || (err != nil && !refused) { - t.Errorf("environment %q, runtime context %q: validate() = %v, want refused=%v", - tc.environment, tc.runtimeContext, err, tc.refused) - } - } -} - -// TestAssetsServeFromTheManifestFS proves a solution that ships its frontend -// in Manifest.Assets serves it with nothing on disk — ASSETS_DIR points -// nowhere, as on a pod with a read-only root filesystem — and that the -// root-relative manifestUrl it registers resolves to a file that FS holds. -func TestAssetsServeFromTheManifestFS(t *testing.T) { - assets := fstest.MapFS{ - "mf-manifest.json": {Data: []byte(`{"name":"lastlogin"}`)}, - "static/js/async/123.3f9a1c2b.js": {Data: []byte(`export default 1`)}, - } - base := bootWithAssets(t, Manifest{ID: "lastlogin-go", Assets: assets}, filepath.Join(t.TempDir(), "absent")) - - manifestURL := frontendManifestURL(t, base) - if manifestURL != federationManifestPath { - t.Fatalf("manifestUrl = %q, want %q", manifestURL, federationManifestPath) - } - resp := get(t, base+manifestURL) - if resp.status != http.StatusOK || resp.body != `{"name":"lastlogin"}` { - t.Fatalf("GET %s = %d %q, want the manifest from Manifest.Assets", manifestURL, resp.status, resp.body) - } - if !strings.HasPrefix(resp.header.Get("content-type"), "application/json") { - t.Errorf("manifest content-type = %q, want application/json", resp.header.Get("content-type")) - } - if resp.header.Get("access-control-allow-origin") != "*" { - t.Errorf("manifest served without CORS") - } - if got := resp.header.Get("cache-control"); got != "no-cache" { - t.Errorf("manifest cache-control = %q, want no-cache: its name is fixed, so a cached copy outlives a redeploy", got) - } - - chunk := get(t, base+"/assets/static/js/async/123.3f9a1c2b.js") - if chunk.status != http.StatusOK || !strings.Contains(chunk.header.Get("content-type"), "javascript") { - t.Fatalf("hashed chunk = %d %q", chunk.status, chunk.header.Get("content-type")) - } - if got := chunk.header.Get("cache-control"); got != "public, max-age=31536000, immutable" { - t.Errorf("hashed chunk cache-control = %q, want immutable", got) - } - - missing := get(t, base+"/assets/static/js/async/456.0badc0de.js") - if missing.status != http.StatusNotFound { - t.Fatalf("missing chunk = %d, want 404", missing.status) - } - if got := missing.header.Get("cache-control"); got != "no-cache" { - t.Errorf("missing hashed chunk cache-control = %q, want no-cache: a 404 must not be cached for a year under the name the next build serves", got) - } -} - -// TestAssetsServeFromAssetsDirWithoutAnFS proves the directory path is kept -// for a solution that sets no Manifest.Assets, with the same cache policy. -func TestAssetsServeFromAssetsDirWithoutAnFS(t *testing.T) { - dir := t.TempDir() - if err := os.WriteFile(filepath.Join(dir, "mf-manifest.json"), []byte(`{"name":"disk"}`), 0o644); err != nil { - t.Fatal(err) - } - base := bootWithAssets(t, Manifest{ID: "lastlogin-go"}, dir) - resp := get(t, base+federationManifestPath) - if resp.status != http.StatusOK || resp.body != `{"name":"disk"}` { - t.Fatalf("GET manifest = %d %q, want the file from ASSETS_DIR", resp.status, resp.body) - } - if got := resp.header.Get("cache-control"); got != "no-cache" { - t.Errorf("manifest cache-control = %q, want no-cache", got) - } -} - -func TestAssetCacheControl(t *testing.T) { - for name, want := range map[string]string{ - "/mf-manifest.json": "no-cache", - "/remoteEntry.js": "no-cache", - "/static/js/123.3f9a1c2b.js": "public, max-age=31536000, immutable", - "/static/css/index-3f9a1c2b0d.css": "public, max-age=31536000, immutable", - "/static/js/lib-react.js": "no-cache", - "/static/js/deadbeef.js": "no-cache", - } { - if got := assetCacheControl(name); got != want { - t.Errorf("assetCacheControl(%q) = %q, want %q", name, got, want) - } - } -} - -type response struct { - status int - header http.Header - body string -} - -func get(t *testing.T, target string) response { - t.Helper() - resp, err := http.Get(target) - if err != nil { - t.Fatalf("GET %s: %v", target, err) - } - defer resp.Body.Close() - body, _ := io.ReadAll(resp.Body) - return response{status: resp.StatusCode, header: resp.Header, body: string(body)} -} - -// bootWithAssets serves m on an ephemeral port with assetsDir as ASSETS_DIR -// and returns its base URL. Registration targets a closed port: these tests -// are about what the solution serves, not whom it registers with. -func bootWithAssets(t *testing.T, m Manifest, assetsDir string) string { - t.Helper() - ln, err := net.Listen("tcp", "127.0.0.1:0") - if err != nil { - t.Fatal(err) - } - s := New(m) - s.cfg = config{port: strconv.Itoa(ln.Addr().(*net.TCPAddr).Port), assetsDir: assetsDir} - ctx, cancel := context.WithCancel(context.Background()) - done := make(chan struct{}) - go func() { _ = s.serve(ctx, ln); close(done) }() - t.Cleanup(func() { cancel(); <-done }) - base := "http://" + ln.Addr().String() - waitFor(t, "health", func() bool { return getStatus(t, base+"/health") == http.StatusOK }) - return base -} diff --git a/environment_boundary_test.go b/environment_boundary_test.go index 1d56323..f1be770 100644 --- a/environment_boundary_test.go +++ b/environment_boundary_test.go @@ -11,63 +11,305 @@ import ( "testing" ) -// serveTimeEnvironmentRead is the one function that reads the environment -// outside boot resolution: the consumed-API projection is parsed when the -// server starts, never reaches validate(), and a malformed value disables the -// federation with a log line instead of refusing the boot. -const serveTimeEnvironmentRead = "registerConsumedAPIs" - // AGENTS.md tells an agent that configuration is resolved in one place and -// refused at boot, naming the single serve-time exception and the silent -// failure it produces. Prose cannot notice a second exception appearing -// underneath it, and an agent-context file is followed literally, so the claim -// is pinned here: every environment read must sit in loadConfig's call tree, -// where validate() governs the result, or be the documented exception. +// refused at boot, with no exception. Prose cannot notice an exception +// appearing underneath it, and an agent-context file is followed literally, so +// the claim is pinned here: every environment read must sit in loadConfig's +// call tree, where validate() governs the result. +// +// There used to be one exception, and deleting it is part of this runtime +// ceasing to register itself. registerConsumedAPIs read the api.consumes +// projection at serve time to register each consumed module's upstream with +// the gateway; a malformed projection disabled the whole federation with a log +// line while the solution served on, so every consumed facade 404'd at the +// gateway and the solution looked healthy. Nothing registers now — the routes +// are delivered — and the projection is resolved at boot like everything else, +// so the refusal is a refusal and not a log line. func TestEnvironmentIsReadOnlyWhereAgentsFileSaysItIs(t *testing.T) { pkg := parsePackage(t) boot := pkg.reachableFrom("loadConfig") var undocumented []string for _, fn := range pkg.environmentReaders() { - if !boot[fn] && fn != serveTimeEnvironmentRead { + if !boot[fn] { undocumented = append(undocumented, fn) } } if len(undocumented) > 0 { t.Errorf("these functions read the environment outside loadConfig's call tree: %s\n"+ - "AGENTS.md states that boot configuration is resolved in loadConfig and refused by validate(), with %q as the only exception. "+ + "AGENTS.md states that boot configuration is resolved in loadConfig and refused by validate(), with no exception. "+ "Either resolve the value in loadConfig, or document the new exception in AGENTS.md — a value read after boot is never refused, it degrades while the solution serves.", - strings.Join(undocumented, ", "), serveTimeEnvironmentRead) + strings.Join(undocumented, ", ")) } } -// The documented exception has to keep being one. If it stops reading the -// environment, or starts being called during boot resolution, AGENTS.md -// describes a hazard that no longer exists where it says it does. -func TestServeTimeEnvironmentReadIsStillTheException(t *testing.T) { - pkg := parsePackage(t) +// And the read has to happen at boot, not merely be written there. +// +// Static reachability is not the claim AGENTS.md makes, and the difference was +// exploitable without meaning to: a closure *defined* inside loadConfig that +// reads the environment is in loadConfig's call tree for this gate's purposes +// while running per handshake, long after validate() has had its say. Two of +// them existed, resolving the caller and peer admission sets, and the gate +// reported the boundary intact — the reads were lexically where the rule wanted +// them and temporally nowhere near it. A value genuinely resolved at boot does +// not need a closure to do it, so the whole shape is refused rather than +// inspected for intent. +func TestNoClosureDefersAnEnvironmentReadPastBoot(t *testing.T) { + // Indirectly too. The closures this gate exists to catch did not call + // os.Getenv: they called workloadPath, which calls env, which does — so a + // rule that looked only for a direct read would have passed the exact code + // it was written for. A probe below pins that. + readers := parsePackage(t).environmentReachers() + fset := token.NewFileSet() + entries, err := os.ReadDir(".") + if err != nil { + t.Fatalf("read package directory: %v", err) + } + for _, entry := range entries { + name := entry.Name() + if filepath.Ext(name) != ".go" || strings.HasSuffix(name, "_test.go") { + continue + } + file, err := parser.ParseFile(fset, name, nil, 0) + if err != nil { + t.Fatalf("parse %s: %v", name, err) + } + for _, line := range deferredEnvironmentReads(fset, file, readers) { + t.Errorf("%s:%d defines a function literal that reads the environment: a read inside a closure happens when the closure is called, which is not boot, so validate() never governs it. Resolve the value in loadConfig's own body and carry the result.", + name, line) + } + for _, line := range environmentValuesEscaping(fset, file, readers) { + t.Errorf("%s:%d takes a reference to a configuration resolver without calling it: a method value or a function value is the same deferred read as a closure, with none of the syntax, so it happens whenever the value is called rather than at boot.", + name, line) + } + } +} - reads := false - for _, fn := range pkg.environmentReaders() { - if fn == serveTimeEnvironmentRead { - reads = true +// deferredEnvironmentReads is every environment read inside a function literal +// in one file, by line. +// +// Separated from the test so the gate can be run against a file the test +// writes. A gate that only ever sees conforming code reports success whether +// or not it works, which is how the shape it exists to catch got in. +func deferredEnvironmentReads(fset *token.FileSet, file *ast.File, readers map[string]bool) []int { + var lines []int + ast.Inspect(file, func(node ast.Node) bool { + lit, ok := node.(*ast.FuncLit) + if !ok { + return true + } + ast.Inspect(lit.Body, func(inner ast.Node) bool { + call, ok := inner.(*ast.CallExpr) + if !ok { + return true + } + if readsEnvironment(call) { + lines = append(lines, fset.Position(call.Pos()).Line) + return true + } + if callee, ok := calleeName(call); ok && readers[callee] { + lines = append(lines, fset.Position(call.Pos()).Line) + } + return true + }) + return true + }) + return lines +} + +// environmentValuesEscaping is every *reference* to a configuration resolver +// that is taken without being called — a method value or a function value, +// which is the same deferred read as a closure with none of the syntax. +// +// `cfg.resolve = s.readTheSet` reads the environment whenever `resolve` is +// called, and the closure gate sees no function literal at all. A review drove +// exactly that shape past both gates. +func environmentValuesEscaping(fset *token.FileSet, file *ast.File, readers map[string]bool) []int { + var lines []int + ast.Inspect(file, func(node ast.Node) bool { + switch held := node.(type) { + // A call's own Fun is a call, not an escaping reference — but its + // arguments are. + case *ast.CallExpr: + for _, arg := range held.Args { + lines = append(lines, referencedReaders(fset, arg, readers)...) + } + case *ast.AssignStmt: + for _, rhs := range held.Rhs { + lines = append(lines, referencedReaders(fset, rhs, readers)...) + } + // `var r = env`, which is an assignment with no AssignStmt. + case *ast.ValueSpec: + for _, value := range held.Values { + lines = append(lines, referencedReaders(fset, value, readers)...) + } + // `return workloadPath`, handing the reader to a caller. + case *ast.ReturnStmt: + for _, result := range held.Results { + lines = append(lines, referencedReaders(fset, result, readers)...) + } + // `config{resolve: workloadPath}`, which is how the shape this gate + // exists for was actually written. + case *ast.CompositeLit: + for _, element := range held.Elts { + if pair, ok := element.(*ast.KeyValueExpr); ok { + lines = append(lines, referencedReaders(fset, pair.Value, readers)...) + continue + } + lines = append(lines, referencedReaders(fset, element, readers)...) + } + } + return true + }) + return lines +} + +// referencedReaders is the lines at which expr names a reader without calling +// it. +func referencedReaders(fset *token.FileSet, expr ast.Expr, readers map[string]bool) []int { + var lines []int + switch named := expr.(type) { + case *ast.Ident: + if readers[named.Name] { + lines = append(lines, fset.Position(named.Pos()).Line) + } + case *ast.SelectorExpr: + if readers[named.Sel.Name] { + lines = append(lines, fset.Position(named.Pos()).Line) } } - if !reads { - t.Errorf("%s no longer reads the environment, so AGENTS.md documents an exception that does not exist: remove it there and here", - serveTimeEnvironmentRead) + return lines +} + +// environmentReachers is the package's configuration resolvers: the functions +// that read the environment themselves, and the ones whose whole job is to call +// those (workloadPath over env). +// +// One hop, not the transitive closure. The closure marks most of the package — +// enough of it that this gate failed two function literals doing nothing but +// holding a certificate to a principal — and a rule with false positives gets +// relaxed rather than obeyed. One hop is what the shape under test needed: the +// closures called workloadPath, which calls env, which reads. A resolver added +// later is caught the same way, because a new resolver reading the environment +// is itself a direct reader. +func (p pkgFuncs) environmentReachers() map[string]bool { + reaches := map[string]bool{} + for name := range p.reads { + reaches[name] = true } - if pkg.reachableFrom("loadConfig")[serveTimeEnvironmentRead] { - t.Errorf("%s is now reachable from loadConfig, so its value is resolved at boot: AGENTS.md describes it as read at serve time and unseen by validate()", - serveTimeEnvironmentRead) + for caller, callees := range p.calls { + for _, callee := range callees { + if p.reads[callee] { + reaches[caller] = true + break + } + } } + return reaches +} +// TestTheClosureGateCatchesTheShapeThatEvadedIt runs the gate against the two +// closures that passed the boundary check while reading the environment per +// handshake — they resolved the caller and peer admission sets — and against +// the conforming shape, so the gate is not simply refusing every closure. +func TestTheClosureGateCatchesTheShapeThatEvadedIt(t *testing.T) { + for _, tc := range []struct { + name string + source string + caught bool + }{ + { + "the admission-set resolver as it was written", + `package solution +func loadConfig() config { + return config{ + resolveAllowedCallers: func() string { + return workloadPath(ctx, IdentityAllowedCallersEnvironmentVariable, WorkloadIdentityAllowedCallersKey) + }, + } +}`, + true, + }, + { + "a closure reading the environment directly", + `package solution +func f() func() string { + return func() string { return os.Getenv("ANYTHING") } +}`, + true, + }, + { + "the SDK accessor called directly in a closure, the exact N1 shape", + `package solution +func loadConfig() config { + return config{ + resolve: func() string { + v, _ := codefly.For(ctx).WorkspaceConfiguration("workload-identity", "ALLOWED_CALLERS") + return v + }, + } +}`, + true, + }, + { + "codefly.Environment() in a closure", + `package solution +func f() func() string { + return func() string { return codefly.Environment() } +}`, + true, + }, + { + "a closure that reads a file, which is what a live value needs", + `package solution +func f() func() ([]byte, error) { + return func() ([]byte, error) { return os.ReadFile(path) } +}`, + false, + }, + { + "a read in a function body, where boot can govern it", + `package solution +func loadConfig() config { + return config{port: env("PORT", "")} +}`, + false, + }, + } { + t.Run(tc.name, func(t *testing.T) { + fset := token.NewFileSet() + file, err := parser.ParseFile(fset, "probe.go", tc.source, 0) + if err != nil { + t.Fatalf("parse the probe: %v", err) + } + // The helpers the real closures reached the environment through. + found := len(deferredEnvironmentReads(fset, file, map[string]bool{"workloadPath": true, "env": true})) > 0 + if found != tc.caught { + if tc.caught { + t.Error("the gate did not see an environment read deferred into a closure, so a value read per handshake can be written as if it were resolved at boot") + } else { + t.Error("the gate flagged a closure that reads no environment: resolving a value from a file per use is how a live value is supposed to work here, and refusing it would refuse the fix") + } + } + }) + } +} + +// AGENTS.md has to keep saying so. A file that stopped describing the boundary +// would leave the next agent to infer it from whichever call site they opened +// first, which is how the serve-time exception came to exist. +func TestTheEnvironmentBoundaryIsDocumented(t *testing.T) { agents, err := os.ReadFile("AGENTS.md") if err != nil { t.Fatalf("read AGENTS.md: %v", err) } - if !strings.Contains(string(agents), serveTimeEnvironmentRead) { - t.Errorf("AGENTS.md does not name %q, the one configuration read that is not refused at boot", serveTimeEnvironmentRead) + for _, claim := range []string{"loadConfig", "validate()"} { + if !strings.Contains(string(agents), claim) { + t.Errorf("AGENTS.md does not name %q, half of where configuration is resolved and refused", claim) + } + } + if strings.Contains(string(agents), "registerConsumedAPIs") { + t.Error("AGENTS.md still names registerConsumedAPIs, the serve-time exception this runtime deleted along with its registrations") } } @@ -156,11 +398,31 @@ func readsEnvironment(call *ast.CallExpr) bool { if !ok { return false } - pkg, ok := selector.X.(*ast.Ident) - if !ok || pkg.Name != "os" { - return false + // os.Getenv and friends. + if pkg, ok := selector.X.(*ast.Ident); ok && (pkg.Name == "os" || pkg.Name == "syscall") { + switch selector.Sel.Name { + // ExpandEnv reads every variable it finds in the string, and + // syscall.Getenv is the same read one package down; the rule named + // neither. + case "Getenv", "LookupEnv", "Environ", "ExpandEnv": + return true + } + } + // And the SDK's own readers, which is the gap that mattered: the rule + // matched only `os.*`, so a closure calling + // codefly.For(ctx).WorkspaceConfiguration(...) passed — and that is the + // *precise* shape of the defect the whole round-four finding was about. + // A gate that misses the exact code it was written for is worse than no + // gate, because it reports the boundary intact. + switch selector.Sel.Name { + case "WorkspaceConfiguration", "Environment", "Endpoint", "Secret": + return true } - return selector.Sel.Name == "Getenv" || selector.Sel.Name == "LookupEnv" || selector.Sel.Name == "Environ" + // Not LoadEnvironmentVariables: that is the loader, and start() calling it + // before loadConfig is the documented shape rather than a read to govern. + // Not For() either — it returns the query; the read is the accessor called + // on it, which the names above cover. + return false } // calleeName is the called function's own name, whether it is called plainly @@ -175,3 +437,102 @@ func calleeName(call *ast.CallExpr) (string, bool) { } return "", false } + +// TestTheGateCatchesAReaderTakenAsAValue: a method value or a function value is +// the same deferred read as a closure, with none of the syntax — and a review +// drove exactly that past both gates. +func TestTheGateCatchesAReaderTakenAsAValue(t *testing.T) { + readers := map[string]bool{"workloadPath": true, "env": true, "readTheSet": true} + for _, tc := range []struct { + name string + source string + caught bool + }{ + { + "a method value assigned for later", + `package solution +func (s *Server) wire() { + s.cfg.resolve = s.readTheSet +}`, + true, + }, + { + "a function value assigned for later", + `package solution +func wire() config { + c := config{} + c.resolve = workloadPath + return c +}`, + true, + }, + { + "a reader passed as an argument, to be called later", + `package solution +func wire() { + hold(workloadPath) +}`, + true, + }, + { + "a composite-literal field, which is how the real one was written", + `package solution +func loadConfig() config { + return config{resolve: workloadPath} +}`, + true, + }, + { + "returned to a caller", + `package solution +func resolver() func() string { return workloadPath }`, + true, + }, + { + "a package-level var holding the reader", + `package solution +var resolve = env`, + true, + }, + { + "handed to a goroutine", + `package solution +func wire() { go readTheSet() }`, + false, + }, + { + "calling it, which is a read at boot and fine", + `package solution +func loadConfig() config { + return config{port: env("PORT", "")} +}`, + false, + }, + { + "assigning its result, which is also fine", + `package solution +func loadConfig() config { + c := config{} + c.port = workloadPath(ctx, "A", "B") + return c +}`, + false, + }, + } { + t.Run(tc.name, func(t *testing.T) { + fset := token.NewFileSet() + file, err := parser.ParseFile(fset, "probe.go", tc.source, 0) + if err != nil { + t.Fatalf("parse the probe: %v", err) + } + found := len(environmentValuesEscaping(fset, file, readers)) > 0 + if found != tc.caught { + if tc.caught { + t.Error("the gate did not see a configuration resolver taken as a value: it is called whenever the value is called, which is not boot, and no function literal appears for the closure gate to find") + } else { + t.Error("the gate flagged a resolver that is called rather than referenced, which is a read at boot and exactly what the rule wants") + } + } + }) + } +} diff --git a/gateway_error.go b/gateway_error.go index c5ab365..b8f0f08 100644 --- a/gateway_error.go +++ b/gateway_error.go @@ -4,8 +4,10 @@ import ( "errors" "fmt" "net/http" + "net/url" "connectrpc.com/connect" + "github.com/codefly-dev/sdk-go/workcontext" ) // GatewayError carries a non-success HTTP status received through the host @@ -66,6 +68,44 @@ func handlerErrorResponse(err error) (int, string) { } return status, http.StatusText(status) } + // This runtime's own credential conditions, before the untyped fallthrough + // below reaches them. + // + // These errors are not a handler's to classify and they are not generic: + // they are produced inside this package, and they carry the mint URL, the + // gateway URL and the issuer's own message, because that is what a boot + // refusal needs to name. Left to the fallthrough, a Handle or HandleRequest + // handler that returned one — which is exactly what a handler does when + // ForModule fails — answered the browser 502 with that text verbatim. + // + // The status was wrong in the same breath as the disclosure. The passthrough + // path already maps these through relayedError, where ErrNotAttested is + // unavailable because the condition is this process's and one renewal fixes + // it; a 502 told the page a module was broken. The two paths now agree, and + // the message says whose problem it is and nothing about where. + switch { + // ErrRevoked first: ErrNotAttested wraps it when a renewal is refused + // because the state the credential is sealed to has moved, and the first + // matching branch is what the page is told. It arrives from a *callee* or + // from a renewal, never from the mint client itself, and it is not terminal + // — the SDK's contract for it is a refresh and a retry (see + // terminalCredentialFailure). + case errors.Is(err, workcontext.ErrRevoked): + return http.StatusConflict, "the authority this solution presented has been superseded; the call was not made" + case errors.Is(err, ErrNotAttested): + return http.StatusServiceUnavailable, "this solution cannot currently act for the viewer against the module" + case errors.Is(err, workcontext.ErrNotACoreToken), errors.Is(err, workcontext.ErrInvalid): + return http.StatusBadGateway, "this solution could not present a usable credential for the module" + } + // A transport failure carries the URL it was dialling. *url.Error puts the + // destination in Error(), so a mint or gateway call that could not connect + // reached the browser as a 502 naming an internal address — the same + // disclosure the sentinels above were mapped to stop, arriving by a + // different route and not caught by matching on them. + var urlErr *url.Error + if errors.As(err, &urlErr) { + return http.StatusBadGateway, "this solution could not reach the platform" + } // Preserve the existing untyped-handler contract. Callers must not put // private details in ordinary errors returned to the runtime. return http.StatusBadGateway, err.Error() diff --git a/gateway_error_test.go b/gateway_error_test.go index da63f0b..5bd2f29 100644 --- a/gateway_error_test.go +++ b/gateway_error_test.go @@ -4,9 +4,11 @@ import ( "context" "errors" "fmt" + "github.com/codefly-dev/sdk-go/workcontext" "io" "net/http" "net/http/httptest" + "net/url" "strings" "sync/atomic" "testing" @@ -34,7 +36,7 @@ func TestUnaryErrorStatusAcrossHTTPBoundary(t *testing.T) { upstream := httptest.NewServer(connect.NewUnaryHandler("/example.Service/Read", func(_ context.Context, req *connect.Request[emptypb.Empty]) (*connect.Response[emptypb.Empty], error) { calls.Add(1) - if req.Header().Get("Authorization") != "Bearer viewer-token" { + if req.Header().Get("Authorization") != viewerBearer() { t.Error("viewer credential lost") } return nil, connect.NewError(tc.code, errors.New("private upstream diagnostic")) @@ -78,3 +80,65 @@ func TestGatewayErrorStatusValidation(t *testing.T) { }) } } + +// TestAHandlerPathCredentialFailureDisclosesNothing is the disclosure a second +// reviewer found behind the "a page sees unavailable" answer, which was only +// ever true of the passthrough. +// +// A Handle or HandleRequest handler returns whatever ForModule gave it. Those +// errors are produced inside this package and deliberately name the mint URL, +// the gateway URL and the issuer's own text, because that is what a boot +// refusal has to say. handlerErrorResponse ended in `err.Error()`, so the +// browser got a 502 carrying all of it — and 502 is the wrong answer anyway: +// the condition is this process's and one renewal fixes it, which is what the +// passthrough path has always reported. +func TestAHandlerPathCredentialFailureDisclosesNothing(t *testing.T) { + const secretish = "https://mint.internal.example/platform/_credential" + for _, tc := range []struct { + name string + err error + status int + }{ + {"a workload this solution cannot attest for", fmt.Errorf("obtain this execution's credential from %s: 403 forbidden: build 11 is not approved: %w", secretish, ErrNotAttested), http.StatusServiceUnavailable}, + {"a superseded authority", fmt.Errorf("mint at %s: %w", secretish, workcontext.ErrRevoked), http.StatusConflict}, + {"a capability this solution cannot carry", fmt.Errorf("mint at %s: %w", secretish, workcontext.ErrInvalid), http.StatusBadGateway}, + } { + t.Run(tc.name, func(t *testing.T) { + status, message := handlerErrorResponse(tc.err) + if status != tc.status { + t.Errorf("status = %d, want %d: the handler path must answer what the passthrough path answers for the same condition", status, tc.status) + } + if strings.Contains(message, secretish) { + t.Errorf("the message handed to the browser names an internal destination:\n%s", message) + } + if strings.Contains(message, "403") || strings.Contains(message, "not approved") { + t.Errorf("the message handed to the browser carries the issuer's own text:\n%s", message) + } + if message == "" { + t.Error("the message says nothing at all: a page still needs to know whose problem it is") + } + }) + } +} + +// TestATransportFailureDoesNotNameTheDestination is the disclosure that got +// past the sentinel mapping by a different route. +// +// *url.Error carries the URL it was dialling in Error(), so a mint or gateway +// call that could not connect reached the browser as a 502 naming an internal +// address — the same leak the credential sentinels were mapped to stop, arriving +// through the untyped fallthrough and not matched by any of them. +func TestATransportFailureDoesNotNameTheDestination(t *testing.T) { + const internal = "https://gateway.internal.svc.cluster.local:42152" + err := &url.Error{Op: "Post", URL: internal + "/platform/_credential", Err: errors.New("dial tcp 10.0.0.5:42152: connect: connection refused")} + status, message := handlerErrorResponse(fmt.Errorf("mint: %w", err)) + if status != http.StatusBadGateway { + t.Errorf("status = %d, want 502", status) + } + if strings.Contains(message, "internal") || strings.Contains(message, "10.0.0.5") { + t.Errorf("the message handed to the browser names an internal destination:\n%s", message) + } + if message == "" { + t.Error("the message says nothing at all") + } +} diff --git a/go.mod b/go.mod index ecec0ba..d4d8234 100644 --- a/go.mod +++ b/go.mod @@ -4,10 +4,12 @@ go 1.27.0 require ( connectrpc.com/connect v1.21.0 - github.com/codefly-dev/core v0.3.41 - github.com/codefly-dev/sdk-go v0.1.65 + github.com/codefly-dev/core v0.9.0 + github.com/codefly-dev/sdk-go v0.3.0 + github.com/codefly-dev/sdk-go/workcontext v0.0.0-20261005062635-dd6ca019c685 github.com/google/uuid v1.6.0 github.com/modelcontextprotocol/go-sdk v1.8.0 + golang.org/x/tools v0.51.0 google.golang.org/genproto/googleapis/api v0.0.0-20260819154853-08b0e4226688 google.golang.org/protobuf v1.36.12 gopkg.in/yaml.v3 v3.0.1 @@ -37,10 +39,12 @@ require ( github.com/go-openapi/swag/typeutils v0.29.1 // indirect github.com/go-openapi/swag/yamlutils v0.29.1 // indirect github.com/go-viper/mapstructure/v2 v2.5.0 // indirect + github.com/gofrs/flock v0.13.1 // indirect + github.com/google/go-github/v89 v89.0.0 // indirect + github.com/google/go-querystring v1.2.0 // indirect github.com/google/jsonschema-go v0.4.3 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect - github.com/kr/text v0.2.0 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/oklog/ulid/v2 v2.1.2 // indirect github.com/pkg/errors v0.9.1 // indirect @@ -50,6 +54,7 @@ require ( github.com/yosida95/uritemplate/v3 v3.0.2 // indirect go.yaml.in/yaml/v3 v3.0.5 // indirect golang.org/x/exp v0.0.0-20260820142414-ca536658362e // indirect + golang.org/x/mod v0.41.0 // indirect golang.org/x/net v0.59.0 // indirect golang.org/x/oauth2 v0.36.0 // indirect golang.org/x/sync v0.23.0 // indirect diff --git a/go.sum b/go.sum index f00909f..ba3bc8e 100644 --- a/go.sum +++ b/go.sum @@ -30,17 +30,18 @@ github.com/cheggaaa/pb/v3 v3.2.1 h1:aprZbFRG+B7+ug76S8QZ6Y1PW168UHzOmbC3wa+aU6I= github.com/cheggaaa/pb/v3 v3.2.1/go.mod h1:U9hSVxoKqJrZIE3PkFG1xXXNaK/Ilzg+EF/scpXzEdw= github.com/clipperhouse/uax29/v2 v2.7.0 h1:+gs4oBZ2gPfVrKPthwbMzWZDaAFPGYK72F0NJv2v7Vk= github.com/clipperhouse/uax29/v2 v2.7.0/go.mod h1:EFJ2TJMRUaplDxHKj1qAEhCtQPW2tJSwu5BF98AuoVM= -github.com/codefly-dev/core v0.3.41 h1:Etae1jOIWEl524U7tnCT3TaDPUtU5WGsf1XhlFALvt8= -github.com/codefly-dev/core v0.3.41/go.mod h1:uG0HQy5iWnkuWUPa1LbbiPfEgzuxi1SzEKwcFPC6eoA= +github.com/codefly-dev/core v0.9.0 h1:fZ+jnP33uQ4JqYBrABjOvdqF/A6K/hLcQumykjHWrgY= +github.com/codefly-dev/core v0.9.0/go.mod h1:2wt5DdL+OnJbIQT0w4cc7dS2sAGVK3QjnwW43r5cQQg= github.com/codefly-dev/gortk v0.2.0 h1:7bOlS5valYz2zil+fZctQNcPCYBcPj86abcw9N8h1hQ= github.com/codefly-dev/gortk v0.2.0/go.mod h1:dDWUMFgAP063OCGhTEpV7FFUj9+zTcxxO/nV80VKEwg= -github.com/codefly-dev/sdk-go v0.1.65 h1:/AKs/XzaVW5P5VAPXSi8ZOUZHMP3N5bUq92DN6/Y2RM= -github.com/codefly-dev/sdk-go v0.1.65/go.mod h1:IBVdaC5quw571pOELRCqZdhh6Y6pO5WE2F+jFG0EUEA= +github.com/codefly-dev/sdk-go v0.3.0 h1:EBU/hKRmmJEjesOSBgNBSeO6fPEJXr7FKSbEpXZR8aw= +github.com/codefly-dev/sdk-go v0.3.0/go.mod h1:5YzonaTyqB060UvXJ2b1+4c8bZUgNthcIOFkVxKXi10= +github.com/codefly-dev/sdk-go/workcontext v0.0.0-20261005062635-dd6ca019c685 h1:yk/xDIcWasCQNrETwHJpLmZriCAOUS/9mQoYL9lANoQ= +github.com/codefly-dev/sdk-go/workcontext v0.0.0-20261005062635-dd6ca019c685/go.mod h1:MychiSpNDcTy+aI5zGDuiPnE+jgNyYkmWJOCg3EICrc= github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI= github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M= github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE= github.com/containerd/errdefs/pkg v0.3.0/go.mod h1:NJw6s9HwNuRhnjJhM7pylWwMyAkmCQvQ4GpJHEqRLVk= -github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= @@ -119,6 +120,7 @@ github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/gnostic-models v0.6.9 h1:MU/8wDLif2qCXZmzncUQ/BOfxWfthHi63KqpoNbWqVw= github.com/google/gnostic-models v0.6.9/go.mod h1:CiWsm0s6BSQd1hRn8/QmxqB6BesYcbSZxsz9b0KuDBw= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/go-github/v89 v89.0.0 h1:35bEK5XoEcF3PZrlVbl9XN63f5BcJRA/UGkxeC9xPg0= @@ -134,10 +136,6 @@ github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= -github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= -github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= -github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= -github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/lufia/plan9stats v0.0.0-20260330125221-c963978e514e h1:Q6MvJtQK/iRcRtzAscm/zF23XxJlbECiGPyRicsX+Ak= github.com/lufia/plan9stats v0.0.0-20260330125221-c963978e514e/go.mod h1:autxFIvghDt3jPTLoqZ9OZ7s9qTGNAWmYCjVFWPX/zg= github.com/mattn/go-colorable v0.1.15 h1:+u9SLTRGnXv73cEsnsmoZBom+dMU88B2M0aDcWy0/jY= @@ -169,8 +167,6 @@ github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 h1:o4JXh1EVt github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/rodaine/protogofakeit v0.1.1 h1:ZKouljuRM3A+TArppfBqnH8tGZHOwM/pjvtXe9DaXH8= github.com/rodaine/protogofakeit v0.1.1/go.mod h1:pXn/AstBYMaSfc1/RqH3N82pBuxtWgejz1AlYpY1mI0= -github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= -github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/santhosh-tekuri/jsonschema/v6 v6.0.3 h1:1EYB5IzjZawrrnELUi78f9fPu57HuXjmddZPjrls/28= github.com/santhosh-tekuri/jsonschema/v6 v6.0.3/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU= github.com/segmentio/asm v1.1.3 h1:WM03sfUOENvvKexOLp+pCqgb/WDjsi7EK8gIsICtzhc= @@ -183,10 +179,10 @@ github.com/shoenig/go-m1cpu v0.2.1 h1:yqRB4fvOge2+FyRXFkXqsyMoqPazv14Yyy+iyccT2E github.com/shoenig/go-m1cpu v0.2.1/go.mod h1:KkDOw6m3ZJQAPHbrzkZki4hnx+pDRR1Lo+ldA56wD5w= github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE= github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg= -github.com/tklauser/go-sysconf v0.3.16 h1:frioLaCQSsF5Cy1jgRBrzr6t502KIIwQ0MArYICU0nA= -github.com/tklauser/go-sysconf v0.3.16/go.mod h1:/qNL9xxDhc7tx3HSRsLWNnuzbVfh3e7gh/BmM179nYI= -github.com/tklauser/numcpus v0.11.0 h1:nSTwhKH5e1dMNsCdVBukSZrURJRoHbSEQjdEbY+9RXw= -github.com/tklauser/numcpus v0.11.0/go.mod h1:z+LwcLq54uWZTX0u/bGobaV34u6V7KNlTZejzM6/3MQ= +github.com/tklauser/go-sysconf v0.4.0 h1:7H0uAN+7RkwWRaxhYXDLqa5V3LPrJeV8wmD9dRUgPQU= +github.com/tklauser/go-sysconf v0.4.0/go.mod h1:8mTNWyog7H+MpKijp4VmKJAd2bbYQ2zuUwkYRbUArPI= +github.com/tklauser/numcpus v0.12.0 h1:NR85qdvHA9pFse3x3weVZ0r0ST8R6l5RHbZrlRaqob4= +github.com/tklauser/numcpus v0.12.0/go.mod h1:ABHeXzJnr/qqwguhClkZKT1/8VABcYrsyUiUGobwWJg= github.com/xlab/treeprint v1.2.0 h1:HzHnuAF1plUN2zGlAFHbSQP2qJ0ZAD3XF5XD7OesXRQ= github.com/xlab/treeprint v1.2.0/go.mod h1:gj5Gd3gPdKtR1ikdDK6fnFLdmIS0X30kTTuNd/WEJu0= github.com/yoheimuta/go-protoparser/v4 v4.14.2 h1:/P/LlX1CF9NaTWEltGcIZVvNlPbhABuAnBtAWpb3+74= @@ -217,6 +213,8 @@ go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= golang.org/x/exp v0.0.0-20260820142414-ca536658362e h1:01Ju2A/fZKkci4zqx0eZxw//DnRYOnBiGJG14hFBhO8= golang.org/x/exp v0.0.0-20260820142414-ca536658362e/go.mod h1:zeBbvyFKDaLwa7CH/zI8KXt7gTl14SF7sO08Pl5jBCM= +golang.org/x/mod v0.41.0 h1:qJmnOUb4YB+FsEuM3HcWucdZASCPGhsX6uljO6pog0c= +golang.org/x/mod v0.41.0/go.mod h1:Ek9pY8RKWXwsWvd3rQiHYtMqkjSUV+s1Rj7j4H5Ur6o= golang.org/x/net v0.59.0 h1:5zfYln+w5XCxwrnMMJPufRgNoXEaGxl0wo5GqPXyues= golang.org/x/net v0.59.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg= golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= @@ -231,8 +229,8 @@ golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI= golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E= golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= -golang.org/x/tools v0.49.0 h1:3NI7VXzL9+1WZD52Dx2ttoPwD5DWrFGpl9mFZDlmisI= -golang.org/x/tools v0.49.0/go.mod h1:SJNXV9DBKT0UbdttsQjbfJlAE/q+y36++zo3uL3N0Oo= +golang.org/x/tools v0.51.0 h1:k4Xc/1Om9jwkBJBo4NVLMSARBoWtK10mx+W5BnXCeAI= +golang.org/x/tools v0.51.0/go.mod h1:9eEncMayCV6zRMGhR5eZEC2iBx98qWcF1HZ9Z7wJOoA= gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= google.golang.org/genproto/googleapis/api v0.0.0-20260819154853-08b0e4226688 h1:ax2KzoSRIZU/M0cIxri3pKxy99vniH1PVxWC6si/eZI= @@ -243,9 +241,8 @@ google.golang.org/grpc v1.83.2 h1:EManeRomTObA0BU7I8vXgg/78uE5MJ9M8B39EX2WscU= google.golang.org/grpc v1.83.2/go.mod h1:YPI1hK3kDked6iHvgX3tR0y+nX/qpMFKhPgFsokw1S8= google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc= google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/evanphx/json-patch.v4 v4.13.0 h1:czT3CmqEaQ1aanPc5SdlgQrrEIb8w/wwCvWWnfEbYzo= gopkg.in/evanphx/json-patch.v4 v4.13.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/handshake_test.go b/handshake_test.go new file mode 100644 index 0000000..c57b67a --- /dev/null +++ b/handshake_test.go @@ -0,0 +1,2337 @@ +package solution + +import ( + "context" + "crypto/ed25519" + "crypto/rand" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "errors" + "io" + "math/big" + "net" + "net/http" + "net/http/httptest" + "net/http/httptrace" + "net/url" + "os" + "path/filepath" + "strings" + "sync/atomic" + "testing" + "time" +) + +// What this file is for: every test here completes, or fails to complete, a +// real TLS 1.3 mutual-authentication handshake against the configuration this +// runtime would actually serve or dial with, and then asks what identity was on +// the wire. +// +// That is the only way to answer the question the posture checks kept getting +// wrong. A check that reads a *tls.Config and approves it is answering "is this +// configuration's description acceptable", and four separate shapes were +// demonstrated describing one identity and serving another: an SNI-keyed +// callback, a NameToCertificate map, a static certificate Go reaches without +// consulting the callback, and a tls.Certificate.Leaf that did not match its +// own DER. Nothing short of a handshake distinguishes those from a conforming +// configuration, so the tests do handshakes. +// +// net.Pipe rather than a socket, for the inbound ones: the subject is which +// certificate crypto/tls selected, which is settled before a byte of +// application data moves, and an in-memory pipe keeps it that way with no port +// to bind. The outbound ones use a real httptest TLS server, because what they +// assert is what the production http.Transport does with a connection. + +// rivalPrincipal is a second workload in the same cell: a valid identity, under +// the same anchor, that this workload must never be able to present or serve. +const rivalPrincipal = "spiffe://codefly.test/ns/solutions/sa/rival" + +// handshook is one handshake's outcome from both ends. Both are needed: in TLS +// 1.3 the client finishes before it learns whether the server accepted its +// certificate, so "the server refused this caller" is only visible on the +// server's side, while "which certificate was served" is only visible on the +// client's. +type handshook struct { + served *x509.Certificate + serverErr, cliErr error +} + +// refused reports that the handshake did not complete as an authenticated +// connection, from whichever side noticed. +func (h handshook) refused() bool { return h.serverErr != nil || h.cliErr != nil } + +func (h handshook) reason() string { + switch { + case h.serverErr != nil: + return h.serverErr.Error() + case h.cliErr != nil: + return h.cliErr.Error() + } + return "" +} + +// servedTo performs one handshake against config, as a caller presenting +// caller, and reports the identity the caller was shown. +// +// The caller skips verification deliberately: the question is which certificate +// this listener selected, not whether a test client trusts it, and a client +// that verified would refuse before reporting what it saw. An empty serverName +// is a no-SNI hello — what a peer addressing a pod by IP sends, and the case +// the previous synthetic "localhost" probe never exercised. +func servedTo(t *testing.T, config *tls.Config, caller *tls.Certificate, serverName string) handshook { + t.Helper() + callerSide, listenerSide := net.Pipe() + done := make(chan error, 1) + go func() { + listener := tls.Server(listenerSide, config) + done <- listener.Handshake() + _ = listener.Close() + }() + + client := tls.Client(callerSide, &tls.Config{ + InsecureSkipVerify: true, + Certificates: []tls.Certificate{*caller}, + ServerName: serverName, + MinVersion: tls.VersionTLS13, + }) + outcome := handshook{cliErr: client.Handshake()} + if outcome.cliErr == nil { + if state := client.ConnectionState(); len(state.PeerCertificates) > 0 { + outcome.served = state.PeerCertificates[0] + } + // Drained while the listener finishes, because net.Pipe is unbuffered: + // a TLS 1.3 server writes session tickets after the client's Finished, + // and with nobody reading them its Handshake never returns. + go func() { _, _ = io.Copy(io.Discard, client) }() + } + outcome.serverErr = <-done + _ = callerSide.Close() + return outcome +} + +// identityOf is the SPIFFE ID a served certificate names, or "" for none. +func identityOf(leaf *x509.Certificate) string { + if leaf == nil || len(leaf.URIs) == 0 { + return "" + } + return leaf.URIs[0].String() +} + +// shapeShifter is a consumer's IdentitySource that describes one identity and +// can serve another. Each field turns on one of the four demonstrated shapes; +// none of them is exotic, and all four passed the posture checks. +type shapeShifter struct { + approved, rival *tls.Certificate + roots *x509.CertPool + + // keyedOnSNI serves approved for a hello naming "localhost" and rival for + // anything else, including the no-SNI hello a peer addressed by IP sends. + keyedOnSNI bool + // rivalForNamed is the same trick the other way round: conforming for the + // no-SNI hello a boot-time check can ask with, and the rival for a hello + // that names anything. It is the case no probe can catch, whatever name the + // probe uses, and the reason the selection is wrapped rather than sampled. + rivalForNamed bool + // byName puts both certificates in NameToCertificate, which crypto/tls + // consults when there is more than one. + byName bool + // staticRivalBeside lists the rival first among several certificates and + // returns approved from GetCertificate. Go consults the callback only when + // the hello carries a name, and picks the first compatible chain from the + // list otherwise — so this is a configuration whose served identity is + // decided by selection, which is why selection is refused outright. + staticRivalBeside bool + // staticRival carries rival in Certificates while GetCertificate returns + // approved. Go consults GetCertificate only when there is no certificate + // list or the hello carries a name, so a no-SNI caller gets the static one. + staticRival bool + // lyingLeaf hands back a pair whose Leaf field describes approved and whose + // DER is rival's. + lyingLeaf bool + // perConnection moves whichever shape is selected behind + // GetConfigForClient, the callback this runtime's own default source uses. + perConnection bool +} + +func (s shapeShifter) ServerTLSConfig() (*tls.Config, error) { + config := &tls.Config{ + MinVersion: tls.VersionTLS13, + ClientAuth: tls.RequireAndVerifyClientCert, + ClientCAs: s.roots, + } + s.apply(config) + if !s.perConnection { + return config, nil + } + // The conforming shape at the top level, the shifting one per connection. + base := &tls.Config{ + MinVersion: tls.VersionTLS13, + ClientAuth: tls.RequireAndVerifyClientCert, + ClientCAs: s.roots, + GetCertificate: func(*tls.ClientHelloInfo) (*tls.Certificate, error) { return s.approved, nil }, + } + base.GetConfigForClient = func(*tls.ClientHelloInfo) (*tls.Config, error) { return config, nil } + return base, nil +} + +// PeerAnchor is what makes the per-connection cases of +// TestTheCertificateServedIsTheCertificateChecked run at all. +// +// Without it, a source setting GetConfigForClient is refused at boot for not +// implementing PeerAnchorSource — and that refusal contains the words "identity +// source", which mentionsTheCertificate accepted, so the two perConnection +// cases took the early return and asserted nothing. Mutant C2-2 (the +// per-connection answer's certificate is not held to the frozen principal) +// survived the whole suite because of it. +// +// This source's defect is which CERTIFICATE it serves for which hello; it has +// no quarrel with the anchor, so answering the real one here leaves the +// certificate shape as the only thing under test. +func (s shapeShifter) PeerAnchor() (*x509.CertPool, error) { return s.roots, nil } + +func (s shapeShifter) apply(config *tls.Config) { + switch { + case s.byName: + config.Certificates = []tls.Certificate{*s.approved, *s.rival} + //nolint:staticcheck // Deliberately sets the deprecated field, to prove the listener refuses a source that does. + config.NameToCertificate = map[string]*tls.Certificate{"localhost": s.rival} + case s.staticRivalBeside: + config.Certificates = []tls.Certificate{*s.rival, *s.approved} + config.GetCertificate = func(*tls.ClientHelloInfo) (*tls.Certificate, error) { return s.approved, nil } + case s.staticRival: + config.Certificates = []tls.Certificate{*s.rival} + config.GetCertificate = func(*tls.ClientHelloInfo) (*tls.Certificate, error) { return s.approved, nil } + case s.lyingLeaf: + config.GetCertificate = func(*tls.ClientHelloInfo) (*tls.Certificate, error) { + lying := *s.rival + lying.Leaf = leafOf(s.approved) + return &lying, nil + } + case s.keyedOnSNI: + config.GetCertificate = func(hello *tls.ClientHelloInfo) (*tls.Certificate, error) { + if hello.ServerName == "localhost" { + return s.approved, nil + } + return s.rival, nil + } + case s.rivalForNamed: + config.GetCertificate = func(hello *tls.ClientHelloInfo) (*tls.Certificate, error) { + if hello.ServerName == "" { + return s.approved, nil + } + return s.rival, nil + } + default: + config.GetCertificate = func(*tls.ClientHelloInfo) (*tls.Certificate, error) { return s.approved, nil } + } +} + +func leafOf(pair *tls.Certificate) *x509.Certificate { + leaf, err := x509.ParseCertificate(pair.Certificate[0]) + if err != nil { + panic(err) + } + return leaf +} + +// TestTheCertificateServedIsTheCertificateChecked is the blocker, reproduced. +// +// The check it replaces called GetCertificate with a hello naming "localhost", +// parsed what came back, compared it to the frozen principal, and left the +// configuration untouched. Four handshakes completed under configurations that +// had passed it, serving a neighbouring workload's leaf — which is the identity +// the host, the gateway and the issuer hold this process to, and the identity a +// presence document names. +// +// Each case must now either be refused at boot or refuse the handshake. What it +// must never do is complete a handshake presenting the rival: a listener whose +// served identity depends on the hello it receives is a listener whose identity +// is the caller's choice. +func TestTheCertificateServedIsTheCertificateChecked(t *testing.T) { + c := newCell(t) + approved := c.identity(t, testPrincipal) + rival := c.identity(t, rivalPrincipal) + caller := c.identity(t, testGatewayPrincipal) + + for _, tc := range []struct { + name string + shape shapeShifter + // sni is the hello this shape serves the rival for. + sni string + }{ + { + name: "a callback keyed on the server name, asked with the name nothing dials by", + shape: shapeShifter{keyedOnSNI: true}, + }, + { + name: "a callback keyed on the server name, behind the per-connection callback", + shape: shapeShifter{keyedOnSNI: true, perConnection: true}, + }, + { + name: "a callback that answers any boot-time probe correctly and the real hello otherwise", + shape: shapeShifter{rivalForNamed: true}, + sni: "localhost", + }, + { + name: "the same, behind the per-connection callback", + shape: shapeShifter{rivalForNamed: true, perConnection: true}, + sni: "localhost", + }, + { + name: "name selection among several certificates", + shape: shapeShifter{byName: true}, + sni: "localhost", + }, + { + name: "selection among several certificates beside a conforming callback", + shape: shapeShifter{staticRivalBeside: true}, + }, + { + name: "a static certificate Go reaches without consulting the callback", + shape: shapeShifter{staticRival: true}, + }, + { + name: "a cached Leaf that does not describe its own DER", + shape: shapeShifter{lyingLeaf: true}, + }, + } { + t.Run(tc.name, func(t *testing.T) { + shape := tc.shape + shape.approved, shape.rival, shape.roots = approved, rival, c.roots + server := New(Manifest{ID: testSolutionID}).Identity(shape) + server.cfg = config{allowedCallersFile: identitiesFile(t, testGatewayPrincipal), mintPeersFile: identitiesFile(t, testGatewayPrincipal), + gatewayPeersFile: identitiesFile(t, testGatewayPrincipal)} + server.principal = testPrincipal + + config, err := server.serverIdentity() + if err != nil { + // Refused at boot, which is the better of the two outcomes: + // nothing is served and the mint has not happened yet. But + // only for the right reason — returning on *any* boot error + // made every case here pass the moment serverIdentity failed + // over an unresolved path or an unreadable admission set, + // which is a test reporting success for a cause it never + // exercised. + if !mentionsTheCertificate(err) { + t.Fatalf("the boot refused this source for something other than the certificate it would serve, so this case never ran: %v", err) + } + return + } + outcome := servedTo(t, config, caller, tc.sni) + if got := identityOf(outcome.served); got == rivalPrincipal { + t.Fatalf("the listener served %s: the configuration passed every posture check describing %s, and the certificate on the wire is the identity the host holds this process to", + got, testPrincipal) + } + if !outcome.refused() && identityOf(outcome.served) != testPrincipal { + t.Fatalf("a handshake completed serving %q, which is neither this workload's principal nor a refusal", identityOf(outcome.served)) + } + if outcome.refused() && !strings.Contains(outcome.reason(), "certificate") { + t.Errorf("the handshake was refused with %q, which does not say it was about the certificate", outcome.reason()) + } + }) + } + + t.Run("and the conforming shape still serves", func(t *testing.T) { + // The control. Without it, refusing every handshake would pass every + // case above, which is not a listener. + shape := shapeShifter{approved: approved, rival: rival, roots: c.roots} + server := New(Manifest{ID: testSolutionID}).Identity(shape) + server.cfg = config{allowedCallersFile: identitiesFile(t, testGatewayPrincipal), mintPeersFile: identitiesFile(t, testGatewayPrincipal), + gatewayPeersFile: identitiesFile(t, testGatewayPrincipal)} + server.principal = testPrincipal + config, err := server.serverIdentity() + if err != nil { + t.Fatalf("a conforming source was refused: %v", err) + } + outcome := servedTo(t, config, caller, "") + if outcome.refused() { + t.Fatalf("a conforming listener refused a handshake from an admitted caller: %v", outcome.reason()) + } + if got := identityOf(outcome.served); got != testPrincipal { + t.Errorf("the listener served %q, want this workload's own %q", got, testPrincipal) + } + }) +} + +// TestANilPerConnectionAnswerCannotDropTheTrustAnchor: a callback answering nil +// means "serve the base configuration", and the base was admitted at boot only +// because a callback would resolve the anchor per handshake. A source that +// answers for one hello and not another therefore served a listener that +// requires a caller's certificate and verifies it against this host's system +// roots — mutual TLS in every log line, and a client certificate from any +// public CA admitted. +func TestANilPerConnectionAnswerCannotDropTheTrustAnchor(t *testing.T) { + c := newCell(t) + approved := c.identity(t, testPrincipal) + caller := c.identity(t, testGatewayPrincipal) + + server := New(Manifest{ID: testSolutionID}).Identity(conditionalAnchor{approved: approved, roots: c.roots}) + server.cfg = config{allowedCallersFile: identitiesFile(t, testGatewayPrincipal), mintPeersFile: identitiesFile(t, testGatewayPrincipal), + gatewayPeersFile: identitiesFile(t, testGatewayPrincipal)} + server.principal = testPrincipal + config, err := server.serverIdentity() + // THIS FIXTURE MUST BOOT. The early return here read "refused at boot is + // also fail-closed" and let ANY boot error pass the test — which is how + // the missing PeerAnchor made every assertion below unreachable for + // several rounds. That fixture defect is fixed; this closes the door it + // came through, so an unrelated boot regression fails here instead of + // quietly passing. + if err != nil { + t.Fatalf("this source boots: it implements PeerAnchorSource and carries a usable anchor, so a refusal here means something unrelated broke and every assertion below would otherwise be skipped: %v", err) + } + outcome := servedTo(t, config, caller, "") + if !outcome.refused() { + t.Fatalf("a handshake completed under a configuration with no trust anchor: the callback answered nothing for this hello, so the base configuration served — and it verifies a caller's certificate against this host's system roots") + } + if !strings.Contains(outcome.reason(), "anchor") { + t.Errorf("the refusal %q does not name the missing anchor", outcome.reason()) + } + // The answered case still works, or this is a listener that refuses + // everything rather than one that refuses the hole. + if outcome := servedTo(t, config, caller, "localhost"); outcome.refused() { + t.Errorf("the handshake the source does answer for was refused: %v", outcome.reason()) + } +} + +// conditionalAnchor resolves the anchor per handshake for a hello naming +// "localhost" and answers nothing for any other, with no anchor on the base +// configuration to fall back to. +type conditionalAnchor struct { + approved *tls.Certificate + roots *x509.CertPool +} + +func (a conditionalAnchor) ServerTLSConfig() (*tls.Config, error) { + base := &tls.Config{ + MinVersion: tls.VersionTLS13, + ClientAuth: tls.RequireAndVerifyClientCert, + GetCertificate: func(*tls.ClientHelloInfo) (*tls.Certificate, error) { return a.approved, nil }, + } + base.GetConfigForClient = func(hello *tls.ClientHelloInfo) (*tls.Config, error) { + if hello.ServerName != "localhost" { + return nil, nil + } + answered := base.Clone() + answered.ClientCAs = a.roots + answered.GetConfigForClient = nil + return answered, nil + } + return base, nil +} + +// PeerAnchor makes this source usable at boot, which is the whole reason +// TestANilPerConnectionAnswerCannotDropTheTrustAnchor can assert anything. +// +// Without it serverIdentity refuses the boot — a source resolving its +// configuration per handshake has to be able to answer what its anchor is +// without a handshake — and that test's "refused at boot is also fail-closed" +// early return swallowed it, so every assertion below the return had been dead +// code since PeerAnchorSource landed. Mutant C2-1 (a nil per-connection answer +// serves a base configuration with no anchor) survived the whole suite because +// of it. +// +// The anchor is the real one: this source's defect is what it answers PER +// HANDSHAKE, not what it can say about its trust, so answering honestly here +// is what leaves the per-handshake hole as the only thing under test. +func (a conditionalAnchor) PeerAnchor() (*x509.CertPool, error) { + return a.roots, nil +} + +// TestTheAdmittedCallerSetIsResolvedPerHandshake: the set was read once at +// boot, and a snapshotted admission decision has the defect this runtime +// already refuses to accept for the trust anchor. Removing a compromised +// consumed module from the provisioned set left this listener admitting it +// until somebody restarted the process. +// +// The drift here is a rewrite of the provisioned file, which is what a platform +// rotating an admission set actually does. The previous version of this test +// drifted the set by mutating a closure the test itself supplied, and passed +// against a runtime where no deployment could drift it at all: the set was read +// through the SDK's value accessor, which is fixed at process start, so the +// production answer never changed and only the test's closure did. +func TestTheAdmittedCallerSetIsResolvedPerHandshake(t *testing.T) { + c := newCell(t) + approved := c.identity(t, testPrincipal) + caller := c.identity(t, testGatewayPrincipal) + + provisioned := identitiesFile(t, testGatewayPrincipal) + server := New(Manifest{ID: testSolutionID}).Identity(shapeShifter{approved: approved, roots: c.roots}) + server.cfg = config{ + allowedCallersFile: provisioned, + mintPeersFile: identitiesFile(t, testGatewayPrincipal), + gatewayPeersFile: identitiesFile(t, testGatewayPrincipal), + } + server.principal = testPrincipal + config, err := server.serverIdentity() + if err != nil { + t.Fatalf("boot: %v", err) + } + if outcome := servedTo(t, config, caller, ""); outcome.refused() { + t.Fatalf("an admitted caller was refused: %v", outcome.reason()) + } + + // The platform removes that caller from the file. No restart, no new + // listener. + writeFile(t, provisioned, "spiffe://codefly.test/ns/platform/sa/somebody-else\n") + outcome := servedTo(t, config, caller, "") + if !outcome.refused() { + t.Fatal("a caller removed from the provisioned set was still admitted: the listener is judging by the set it booted with, so revocation needs a restart somebody has to remember to perform") + } + + // A set that resolves to nothing refuses, rather than falling back to the + // set this process booted with. + writeFile(t, provisioned, "\n") + if outcome := servedTo(t, config, caller, ""); !outcome.refused() { + t.Fatal("a caller was admitted while the provisioned set resolved to nothing: an admission decision that cannot be resolved is not one to guess at") + } + + // And so does one that cannot be read at all, for the same reason: an + // unreadable set is not an empty set and it is not the boot set either. + if err := os.Remove(provisioned); err != nil { + t.Fatal(err) + } + if outcome := servedTo(t, config, caller, ""); !outcome.refused() { + t.Fatal("a caller was admitted while the provisioned set could not be read: the only safe reading of an unreadable admission set is to refuse") + } +} + +// TestTheProvisionedCallerSetIsReadThroughLoadConfig is the other half: the +// path production reads has to be the one loadConfig resolves, through the +// documented override, and the content has to be re-read rather than captured. +// +// It is the test that could not have been written before. The set was a +// configuration *value*, and a configuration value is fixed at process start — +// inline or file-carried, the SDK reads it once and keeps it (sdk-go +// file_carrier.go) — so "re-resolved per handshake" returned the boot answer +// forever, and the only way to make a test show drift was to drift something no +// platform touches. Here the test changes what the platform changes. +func TestTheProvisionedCallerSetIsReadThroughLoadConfig(t *testing.T) { + callers := identitiesFile(t, testGatewayPrincipal) + t.Setenv(IdentityAllowedCallersFileEnvironmentVariable, callers) + peers := identitiesFile(t, testGatewayPrincipal) + t.Setenv(IdentityMintPeersFileEnvironmentVariable, peers) + + cfg := loadConfig(context.Background(), testSolutionID, nil) + if cfg.allowedCallersFile != callers { + t.Fatalf("loadConfig resolved the caller set from %q, want the provisioned %q", cfg.allowedCallersFile, callers) + } + if cfg.mintPeersFile != peers { + t.Fatalf("loadConfig resolved the mint peer set from %q, want the provisioned %q", cfg.mintPeersFile, peers) + } + + server := New(Manifest{ID: testSolutionID}) + server.cfg = cfg + admits := server.admittedCallers() + got, err := admits() + if err != nil { + t.Fatalf("resolve the admitted callers: %v", err) + } + if len(got) != 1 || got[0] != testGatewayPrincipal { + t.Fatalf("the admitted callers resolved to %v, want %q", got, testGatewayPrincipal) + } + + // The platform rewrites it. Nothing restarts. + writeFile(t, callers, "spiffe://codefly.test/ns/platform/sa/somebody-else\n") + got, err = admits() + if err != nil { + t.Fatalf("resolve the admitted callers after the set was rewritten: %v", err) + } + if len(got) != 1 || got[0] != "spiffe://codefly.test/ns/platform/sa/somebody-else" { + t.Fatalf("the admitted callers resolved to %v after the provisioned set was rewritten: it is reading something it captured", got) + } +} + +// TestAnAdmissionSetIsNeitherCachedNorDefaulted pins the two answers that are +// not an identity list, since each one has a plausible-looking wrong behaviour: +// falling back to the boot set, or treating an empty file as "admit nobody" and +// carrying on. +func TestAnAdmissionSetIsNeitherCachedNorDefaulted(t *testing.T) { + for _, tc := range []struct { + name string + content string + write bool + }{ + {"an empty file", "", true}, + {"a file of only separators", ",,\n\n", true}, + {"a file of only comments", "# the gateway, once\n", true}, + {"no file at all", "", false}, + } { + t.Run(tc.name, func(t *testing.T) { + path := filepath.Join(t.TempDir(), "identities") + if tc.write { + writeFile(t, path, tc.content) + } + if _, err := resolvedIdentities(path, "unresolved")(); err == nil { + t.Error("an unusable admission set resolved without an error, so a handshake would be judged by something other than the provisioned set") + } + }) + } + // And the path itself being unset is refused, not read as "no restriction". + if _, err := resolvedIdentities("", "unresolved")(); err == nil { + t.Error("an unset admission path resolved without an error") + } +} + +// platformHost is a platform endpoint — the mint, the gateway — served under +// the cell's anchor, requiring the caller's certificate, recording who called. +type platformHost struct { + *httptest.Server + callers chan string +} + +func newPlatformHost(t *testing.T, c *cell, identity string) *platformHost { + t.Helper() + host := &platformHost{callers: make(chan string, 64)} + host.Server = httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + select { + case host.callers <- callerIdentity(r): + default: + } + writeJSON(w, http.StatusOK, map[string]string{"ok": "yes"}) + })) + host.TLS = &tls.Config{ + MinVersion: tls.VersionTLS13, + Certificates: []tls.Certificate{*c.identity(t, identity)}, + ClientCAs: c.roots, + ClientAuth: tls.RequireAndVerifyClientCert, + } + host.StartTLS() + t.Cleanup(host.Close) + return host +} + +func (h *platformHost) called() []string { + var seen []string + for { + select { + case caller := <-h.callers: + seen = append(seen, caller) + default: + return seen + } + } +} + +// TestTheOutboundLeafIsHeldWhenItIsPresented: a projection that rotates to a +// neighbouring workload's identity before anything is dialled is refused, and +// the rival never reaches the platform. +// +// What this does NOT prove, despite its name, is that the hold runs at the +// moment of presentation. The outbound configuration is rebuilt per dial, so +// the check made while building it is already at-use for this case, and +// removing the wrapper around the callback leaves this test passing. The +// at-use half — a rotation landing between that check and the handshake — is +// TestARotationBetweenTheCheckAndTheHandshakeIsRefused, and the wrapper is +// driven directly by TestTheOutboundHoldRefusesTheAnswerAHandshakeGets. +// Saying so here because a test whose name claims the stronger property is +// how the weaker one gets mistaken for coverage. +func TestTheOutboundLeafIsHeldWhenItIsPresented(t *testing.T) { + c := newCell(t) + certFile, keyFile, bundleFile, _, _ := c.workload(t, testPrincipal) + host := newPlatformHost(t, c, testGatewayPrincipal) + + server := New(Manifest{ID: testSolutionID}) + server.cfg = config{identityCertFile: certFile, identityKeyFile: keyFile, trustBundleFile: bundleFile, + mintPeersFile: identitiesFile(t, testGatewayPrincipal), + gatewayPeersFile: identitiesFile(t, testGatewayPrincipal), + // The destination this test dials, so the authorization set for it is + // the mint's: the set is chosen per destination now. + mintURL: host.URL + credentialMintPath} + server.principal = testPrincipal + client, err := server.outboundClient(nil) + if err != nil { + t.Fatalf("outboundClient: %v", err) + } + + // The projection is replaced after the client was built and before anything + // was dialled: the window the preflight left open. + rivalCert, rivalKey, _, _, _ := c.workload(t, rivalPrincipal) + writeFile(t, certFile, readFile(t, rivalCert)) + writeFile(t, keyFile, readFile(t, rivalKey)) + + resp, err := client.Get(host.URL + credentialMintPath) + if err == nil { + _ = resp.Body.Close() + t.Fatal("the mint was called after the projected pair rotated to another workload's identity: the check ran before the handshake, and the handshake presented whatever the reloader held by then") + } + if !strings.Contains(err.Error(), rivalPrincipal) || !strings.Contains(err.Error(), testPrincipal) { + t.Errorf("the refusal %q does not name both the identity that would have been presented and this workload's frozen principal", err) + } + for _, caller := range host.called() { + if caller == rivalPrincipal { + t.Error("the platform saw a request from the rival identity") + } + } +} + +// TestTheOutboundPeerMustBeAProvisionedPlatformIdentity: verifying the chain +// and the hostname says a destination holds a certificate this cell issued, +// which every workload in the cell does. A neighbouring workload answering at +// the gateway's address under a certificate valid for that address completed +// this handshake and was handed the projected token, the viewer's bearer and +// this workload's own credential. +func TestTheOutboundPeerMustBeAProvisionedPlatformIdentity(t *testing.T) { + c := newCell(t) + certFile, keyFile, bundleFile, _, _ := c.workload(t, testPrincipal) + + build := func(t *testing.T, host *platformHost) *http.Client { + t.Helper() + server := New(Manifest{ID: testSolutionID}) + server.cfg = config{identityCertFile: certFile, identityKeyFile: keyFile, trustBundleFile: bundleFile, + mintPeersFile: identitiesFile(t, testGatewayPrincipal), + gatewayPeersFile: identitiesFile(t, testGatewayPrincipal), + mintURL: host.URL + credentialMintPath} + server.principal = testPrincipal + client, err := server.outboundClient(nil) + if err != nil { + t.Fatalf("outboundClient: %v", err) + } + return client + } + + t.Run("the provisioned platform identity is dialled", func(t *testing.T) { + host := newPlatformHost(t, c, testGatewayPrincipal) + resp, err := build(t, host).Get(host.URL + credentialMintPath) + if err != nil { + t.Fatalf("the provisioned platform destination was refused: %v", err) + } + _ = resp.Body.Close() + }) + + t.Run("another workload at the same address is not", func(t *testing.T) { + // A valid certificate from the same anchor, valid for this address, + // naming a workload that is not the platform. + host := newPlatformHost(t, c, rivalPrincipal) + resp, err := build(t, host).Get(host.URL + credentialMintPath) + if err == nil { + _ = resp.Body.Close() + t.Fatal("this workload's credentials were presented to a destination that is not a provisioned platform identity: a chain and a hostname do not distinguish the gateway from any other workload in the cell") + } + if !strings.Contains(err.Error(), rivalPrincipal) { + t.Errorf("the refusal %q does not name the identity that answered", err) + } + }) +} + +// TestABusyConnectionDoesNotOutliveTheTrustThatAuthenticatedIt is the second +// blocker, reproduced. +// +// Per-dial reloading was documented as bounding outbound trust staleness to +// IdleConnTimeout, and that bound only exists for a connection nobody is using. +// A connection carrying a request every 100ms was demonstrated still answering +// 31 seconds after the server's root was removed from the bundle, over a single +// handshake: traffic held the connection, and the trust decision behind it, +// open indefinitely. The committed test for per-dial reloading closed idle +// connections, which is exactly the case this one does not. +func TestABusyConnectionDoesNotOutliveTheTrustThatAuthenticatedIt(t *testing.T) { + c := newCell(t) + certFile, keyFile, bundleFile, _, _ := c.workload(t, testPrincipal) + host := newPlatformHost(t, c, testGatewayPrincipal) + + server := New(Manifest{ID: testSolutionID}) + server.cfg = config{identityCertFile: certFile, identityKeyFile: keyFile, trustBundleFile: bundleFile, + mintPeersFile: identitiesFile(t, testGatewayPrincipal), + gatewayPeersFile: identitiesFile(t, testGatewayPrincipal), + // The destination this test dials, so the authorization set for it is + // the mint's: the set is chosen per destination now. + mintURL: host.URL + credentialMintPath} + server.principal = testPrincipal + client, err := server.outboundClient(nil) + if err != nil { + t.Fatalf("outboundClient: %v", err) + } + + // The control first, and it is the half that was missing: while trust is + // intact, a connection kept this busy has to keep working across several + // recheck intervals. Without this, a recheck that closed every connection + // unconditionally — or one that closed them for an unrelated reason — + // satisfied the assertion below and the test reported the guarantee. + staysUsable(t, client, host.URL+credentialMintPath, 3*outboundTrustRecheckInterval) + + // The root is removed and replaced by an unrelated one: the bundle still + // parses and no longer contains the issuer of the certificate the platform + // presents. Nothing closes the connection, and traffic keeps arriving. + other := newCell(t) + writeFile(t, bundleFile, string(other.anchorPEM)) + + err = busyUntilRefused(t, client, host.URL+credentialMintPath, 3*outboundTrustRecheckInterval) + if err == nil { + t.Fatalf("requests still succeeded %s after the platform's root was removed from the bundle, on a connection kept busy throughout: an inactivity timeout cannot bound a connection that is never inactive, so the documented staleness window held only for a quiet process", + 3*outboundTrustRecheckInterval) + } + // And it ended for the reason under test. Any-error was what let two + // mutants through: a recheck broken in a way that fails requests for some + // other cause looks identical to one that works. + if !mentionsTrustFailure(err) { + t.Errorf("the connection stopped being usable, but not over the peer's trust: %v", err) + } +} + +// staysUsable keeps ONE connection busy for d and fails if a request stops +// succeeding — or if the connection underneath is replaced. +// +// Counting connections is the half that was missing, and without it the control +// proves much less than it looks: net/http re-dials transparently, so a recheck +// that closed every connection once a second kept every request succeeding and +// satisfied this. The defect would have been invisible in exactly the test +// written to rule it out. +func staysUsable(t *testing.T, client *http.Client, url string, d time.Duration) { + t.Helper() + // Any connection dialled after the first request is a replacement. + // + // Counting non-reused dials outright is not enough: the caller may already + // hold a connection, so the first request contributes nothing, and a + // recheck closing it once would then look like the single dial a fresh + // client makes. Replacements are counted from after the first observation, + // which is the only way to tell "nothing changed" from "it was replaced". + var seen, replaced atomic.Int64 + trace := &httptrace.ClientTrace{ + GotConn: func(info httptrace.GotConnInfo) { + if seen.Add(1) > 1 && !info.Reused { + replaced.Add(1) + } + }, + } + deadline := time.Now().Add(d) + for time.Now().Before(deadline) { + request, err := http.NewRequestWithContext(httptrace.WithClientTrace(context.Background(), trace), http.MethodGet, url, nil) + if err != nil { + t.Fatal(err) + } + resp, err := client.Do(request) + if err != nil { + t.Fatalf("a request failed while trust was intact, so nothing this test goes on to assert is about trust being withdrawn: %v", err) + } + _ = resp.Body.Close() + time.Sleep(outboundTrustRecheckInterval / 20) + } + if got := replaced.Load(); got > 0 { + t.Fatalf("the connection was replaced %d times over %s while trust was intact: it is being closed and re-dialled underneath, which keeps every request succeeding and makes the recheck's own behaviour invisible", got, d) + } + if seen.Load() < 2 { + t.Fatalf("only %d requests were observed over %s, which is too few for the replacement check to mean anything", seen.Load(), d) + } +} + +// busyUntilRefused keeps one connection busy until a request fails, and returns +// that error — or nil if requests were still succeeding after d, which is the +// defect these tests exist to catch. +func busyUntilRefused(t *testing.T, client *http.Client, url string, d time.Duration) error { + t.Helper() + deadline := time.Now().Add(d) + for { + resp, err := client.Get(url) + if err != nil { + return err + } + _ = resp.Body.Close() + if time.Now().After(deadline) { + return nil + } + time.Sleep(outboundTrustRecheckInterval / 20) + } +} + +// mentionsTrustFailure reports whether an error is the recheck's own judgement +// about the peer, rather than any other way a request can fail. +func mentionsTrustFailure(err error) bool { + // Both the recheck's own judgement and the refusal a *re-dial* makes + // count: once the recheck closes the connection, net/http dials again, and + // that dial is refused by the same anchor and peer set. What must not pass + // is an error from anywhere else. + for _, named := range []string{ + "no longer verifies", + "no longer one this runtime presents", + "no longer one this solution admits", + "refusing to present this workload's credentials", + "certificate signed by unknown authority", + "certificate is not trusted", + } { + if strings.Contains(err.Error(), named) { + return true + } + } + return false +} + +// TestEveryPlatformConnectionIsWatched is the narrow half of the test above: it +// is the production transport's own dialler that has to attach the recheck, not +// a wrapper a test builds. +// +// It asserts the recheck's *behaviour* on the dialled connection, not its type. +// A type assertion on the wrapper proves the transport returned something +// named right, which survives a recheck that never fires — and a mutant that +// stopped the ticker left this test green. +func TestEveryPlatformConnectionIsWatched(t *testing.T) { + c := newCell(t) + certFile, keyFile, bundleFile, _, _ := c.workload(t, testPrincipal) + host := newPlatformHost(t, c, testGatewayPrincipal) + + server := New(Manifest{ID: testSolutionID}) + server.cfg = config{identityCertFile: certFile, identityKeyFile: keyFile, trustBundleFile: bundleFile, + mintPeersFile: identitiesFile(t, testGatewayPrincipal), + gatewayPeersFile: identitiesFile(t, testGatewayPrincipal), + // The destination this test dials, so the authorization set for it is + // the mint's: the set is chosen per destination now. + mintURL: host.URL + credentialMintPath} + server.principal = testPrincipal + client, err := server.outboundClient(nil) + if err != nil { + t.Fatalf("outboundClient: %v", err) + } + transport, ok := client.Transport.(*http.Transport) + if !ok { + t.Fatalf("outbound transport is %T, want *http.Transport", client.Transport) + } + if transport.DialTLSContext == nil { + t.Fatal("the outbound transport has no TLS dialler of its own, so every connection it makes is configured once and never re-judged") + } + address := strings.TrimPrefix(host.URL, "https://") + conn, err := transport.DialTLSContext(context.Background(), "tcp", address) + if err != nil { + t.Fatalf("dial the platform: %v", err) + } + defer func() { _ = conn.Close() }() + + // The connection is usable, and stays usable while trust holds. + // + // HTTP/1.1 with keep-alive, and the response drained, because the point of + // the exchange is to leave the connection OPEN. This asked over HTTP/1.0, + // so the server closed it as soon as it had answered — and the loop below + // returns on io.EOF, which means it returned on the server's own close and + // passed identically with no watcher installed at all. Mutant C2-3 (an + // established outbound connection is not watched) survived the whole + // suite on exactly that. + request := "GET " + credentialMintPath + " HTTP/1.1\r\nHost: " + address + "\r\nConnection: keep-alive\r\n\r\n" + if _, err := conn.Write([]byte(request)); err != nil { + t.Fatalf("the dialled connection was not usable: %v", err) + } + // Drain what the host answered. A read that times out means there is + // nothing more to read AND the connection is still open, which is the + // state this test needs before it can attribute a later close. + answered := 0 + for { + _ = conn.SetReadDeadline(time.Now().Add(outboundTrustRecheckInterval)) + var buf [512]byte + read, err := conn.Read(buf[:]) + answered += read + if errors.Is(err, os.ErrDeadlineExceeded) { + break + } + if err != nil { + t.Fatalf("the connection closed while the host was answering (%v), so a close observed after the root is removed would not be the watcher's doing and this test could not tell the two apart", err) + } + } + if answered == 0 { + t.Fatal("the host answered nothing, so this connection was never a working one and what closes it later is unattributable") + } + time.Sleep(2 * outboundTrustRecheckInterval) + + // Now the root goes, and this connection — which no transport is polling, + // so nothing but its own watcher can notice — has to close itself. + other := newCell(t) + writeFile(t, bundleFile, string(other.anchorPEM)) + + deadline := time.Now().Add(5 * outboundTrustRecheckInterval) + for { + _ = conn.SetReadDeadline(time.Now().Add(outboundTrustRecheckInterval / 4)) + var buf [1]byte + _, err := conn.Read(buf[:]) + if errors.Is(err, net.ErrClosed) || errors.Is(err, io.EOF) { + return // the watcher closed it, which is the whole point + } + if time.Now().After(deadline) { + t.Fatalf("a connection the transport dialled was still open %s after the platform's root was removed: a connection whose trust is never re-verified is bounded only by how long it stays idle, which traffic prevents", + 5*outboundTrustRecheckInterval) + } + } +} + +// rotatingIdentity is a consumer's source whose leaf changes between calls: +// conforming the first time it is asked, a neighbouring workload's after that. +// +// It is the shape of a real rotation — the platform replaces the projected pair +// under a running process, deliberately and often — compressed so a test can +// land it in the window between a check and the handshake that check was meant +// to cover. Nothing about it is adversarial; the first call being the one a +// preflight makes is the whole defect. +type rotatingIdentity struct { + approved, rival *tls.Certificate + roots *x509.CertPool + asked *int +} + +func (r rotatingIdentity) ServerTLSConfig() (*tls.Config, error) { + return &tls.Config{ + MinVersion: tls.VersionTLS13, + ClientAuth: tls.RequireAndVerifyClientCert, + ClientCAs: r.roots, + GetCertificate: func(*tls.ClientHelloInfo) (*tls.Certificate, error) { + *r.asked++ + if *r.asked == 1 { + return r.approved, nil + } + return r.rival, nil + }, + }, nil +} + +// TestARotationBetweenTheCheckAndTheHandshakeIsRefused is the outbound +// check/use gap on its own. +// +// The outbound configuration is rebuilt per dial, so its preflight runs close +// to the handshake — close enough that a file rotation rarely lands between +// them, which is why the busy-connection and rotation tests above cannot tell +// the preflight from the wrapper. A source asked twice can: the first answer is +// the one the check reads, the second is the one the handshake sends. Only +// holding the pair at presentation refuses that. +func TestARotationBetweenTheCheckAndTheHandshakeIsRefused(t *testing.T) { + c := newCell(t) + host := newPlatformHost(t, c, testGatewayPrincipal) + asked := 0 + source := rotatingIdentity{approved: c.identity(t, testPrincipal), rival: c.identity(t, rivalPrincipal), roots: c.roots, asked: &asked} + + server := New(Manifest{ID: testSolutionID}).Identity(source) + server.cfg = config{allowedCallersFile: identitiesFile(t, testGatewayPrincipal), mintPeersFile: identitiesFile(t, testGatewayPrincipal), + gatewayPeersFile: identitiesFile(t, testGatewayPrincipal), + mintURL: host.URL + credentialMintPath} + server.principal = testPrincipal + identity, err := server.serverIdentity() + if err != nil { + t.Fatalf("the first answer is this workload's own, so the boot must not be refused: %v", err) + } + client, err := server.outboundClient(identity) + if err != nil { + t.Fatalf("outboundClient: %v", err) + } + resp, err := client.Get(host.URL + credentialMintPath) + if err == nil { + _ = resp.Body.Close() + t.Fatalf("the platform was dialled with the certificate the source produced *after* the one that was checked (asked %d times): a check that samples the source and leaves its callback installed is a check a rotation walks past", + asked) + } + if !strings.Contains(err.Error(), rivalPrincipal) { + t.Errorf("the refusal %q does not name the identity that would have been presented", err) + } + for _, caller := range host.called() { + if caller == rivalPrincipal { + t.Error("the platform saw a request from the rival identity") + } + } +} + +// TestTheOutboundHoldRefusesTheAnswerAHandshakeGets is the same property as a +// unit, because the end-to-end window is too small to wedge a test into on the +// default path: the outbound configuration is rebuilt per dial, so its +// boot-time refusal runs microseconds before the handshake it covers. What +// closes the window is that the callback itself is wrapped, so the pair a +// handshake asks for is the pair that is compared — and that is checkable +// directly. +func TestTheOutboundHoldRefusesTheAnswerAHandshakeGets(t *testing.T) { + c := newCell(t) + approved, rival := c.identity(t, testPrincipal), c.identity(t, rivalPrincipal) + asked := 0 + config := &tls.Config{ + MinVersion: tls.VersionTLS13, + RootCAs: c.roots, + GetClientCertificate: func(*tls.CertificateRequestInfo) (*tls.Certificate, error) { + asked++ + if asked == 1 { + return approved, nil + } + return rival, nil + }, + } + if err := holdPresentedCertificate(config, testPrincipal); err != nil { + t.Fatalf("the first answer is this workload's own, so the hold must accept it: %v", err) + } + pair, err := config.GetClientCertificate(&tls.CertificateRequestInfo{}) + if err == nil { + t.Fatalf("the certificate a handshake asked for (%s) was returned unchecked: the hold read an earlier answer and left the source's callback installed, which is a check a rotation walks past", + identityOf(leafOf(pair))) + } + if !strings.Contains(err.Error(), rivalPrincipal) { + t.Errorf("the refusal %q does not name the identity that would have been presented", err) + } +} + +// TestAnEstablishedConnectionFollowsTheProvisionedPeerSet: the recheck is +// supposed to apply the *current* allowed-peer set, not only the current +// anchor. Without that, a destination removed from the provisioned set keeps +// every connection it already has — which is the same staleness the anchor half +// of this was fixed for, one field along. +func TestAnEstablishedConnectionFollowsTheProvisionedPeerSet(t *testing.T) { + c := newCell(t) + certFile, keyFile, bundleFile, _, _ := c.workload(t, testPrincipal) + host := newPlatformHost(t, c, testGatewayPrincipal) + + provisioned := identitiesFile(t, testGatewayPrincipal) + server := New(Manifest{ID: testSolutionID}) + server.cfg = config{identityCertFile: certFile, identityKeyFile: keyFile, trustBundleFile: bundleFile, + mintPeersFile: provisioned, + gatewayPeersFile: provisioned, + mintURL: host.URL + credentialMintPath} + server.principal = testPrincipal + client, err := server.outboundClient(nil) + if err != nil { + t.Fatalf("outboundClient: %v", err) + } + staysUsable(t, client, host.URL+credentialMintPath, 3*outboundTrustRecheckInterval) + + // That destination is removed from the provisioned set, and traffic keeps + // arriving on the connection it already has. + writeFile(t, provisioned, "spiffe://codefly.test/ns/platform/sa/somebody-else\n") + err = busyUntilRefused(t, client, host.URL+credentialMintPath, 3*outboundTrustRecheckInterval) + if err == nil { + t.Fatalf("this workload kept presenting its credentials for %s after the destination was removed from the provisioned set, on the connection it already held", + 3*outboundTrustRecheckInterval) + } + if !mentionsTrustFailure(err) { + t.Errorf("the connection ended for something other than the peer set it is held to: %v", err) + } +} + +// TestALeafReadFailureDoesNotCloseATrustedConnection: the recheck may only +// close a connection over a judgement about the PEER. +// +// Rebuilding the whole outbound configuration on every recheck, this workload's +// own certificate included, and closing the connection on any error from it, +// means a rotation of this pod's own key pair that is not atomic — the +// certificate replaced, the key a moment behind — makes X509KeyPair fail and +// tears down every established platform connection, streams included, while +// every peer on them is still perfectly trusted. The SDK's reloader exists to +// swallow exactly that and keep the last good pair; building a new one per +// recheck throws the property away. +// +// It counts CONNECTIONS, not successful requests. Asserting that requests keep +// working cannot see this defect at all: the recheck closes the connection and +// net/http immediately dials another, which succeeds, so every request +// succeeds while the connection underneath is being torn down and replaced +// once a second. Two mutants lived in that gap. +func TestALeafReadFailureDoesNotCloseATrustedConnection(t *testing.T) { + c := newCell(t) + certFile, keyFile, bundleFile, _, _ := c.workload(t, testPrincipal) + host := newPlatformHost(t, c, testGatewayPrincipal) + + server := New(Manifest{ID: testSolutionID}) + server.cfg = config{identityCertFile: certFile, identityKeyFile: keyFile, trustBundleFile: bundleFile, + mintPeersFile: identitiesFile(t, testGatewayPrincipal), + gatewayPeersFile: identitiesFile(t, testGatewayPrincipal), + // The destination this test dials, so the authorization set for it is + // the mint's: the set is chosen per destination now. + mintURL: host.URL + credentialMintPath} + server.principal = testPrincipal + client, err := server.outboundClient(nil) + if err != nil { + t.Fatalf("outboundClient: %v", err) + } + + var dials atomic.Int64 + trace := &httptrace.ClientTrace{ + GotConn: func(info httptrace.GotConnInfo) { + if !info.Reused { + dials.Add(1) + } + }, + } + get := func(t *testing.T) { + t.Helper() + req, err := http.NewRequestWithContext(httptrace.WithClientTrace(context.Background(), trace), + http.MethodGet, host.URL+credentialMintPath, nil) + if err != nil { + t.Fatal(err) + } + resp, err := client.Do(req) + if err != nil { + t.Fatalf("a request failed while the peer was trusted throughout: %v", err) + } + _ = resp.Body.Close() + } + + get(t) + if dials.Load() != 1 { + t.Fatalf("the first request took %d connections, want 1", dials.Load()) + } + + // A rotation caught halfway: this workload's certificate is replaced by + // something that does not pair with the key on disk. Nothing about the + // peer has changed. + writeFile(t, certFile, "-----BEGIN CERTIFICATE-----\nhalf a rotation\n-----END CERTIFICATE-----\n") + + deadline := time.Now().Add(3 * outboundTrustRecheckInterval) + for time.Now().Before(deadline) { + get(t) + time.Sleep(outboundTrustRecheckInterval / 10) + } + if redialled := dials.Load(); redialled != 1 { + t.Errorf("the established connection was replaced %d times while its peer stayed trusted: a half-written copy of this workload's own key pair is not a judgement about the peer, and the recheck must not share a failure path with it", redialled-1) + } +} + +// TestAnEstablishedCallerDoesNotOutliveTheTrustThatAdmittedIt is the inbound +// half of the busy-connection finding, and it is the half that went unanswered. +// +// The argument — an inactivity timeout cannot bound a connection that is never +// inactive, so re-verify the established peer — was written down in this +// repository, reproduced outbound, fixed outbound, and documented as a property +// of the runtime. Inbound the listener was still `&http.Server{Handler: mux}`: +// no recheck, no IdleTimeout, no ReadHeaderTimeout. So a caller removed from +// the admitted set, or whose issuing root was pulled from the bundle, kept +// every keep-alive connection it already held — in the direction where the +// decision is whom to admit. +func TestAnEstablishedCallerDoesNotOutliveTheTrustThatAdmittedIt(t *testing.T) { + for _, tc := range []struct { + name string + // what the platform withdraws, after the caller is connected + withdraw func(t *testing.T, bundleFile, callersFile string, c *cell) + }{ + { + "its issuing root is removed from the bundle", + func(t *testing.T, bundleFile, _ string, _ *cell) { + other := newCell(t) + writeFile(t, bundleFile, string(other.anchorPEM)) + }, + }, + { + "it is removed from the admitted set", + func(t *testing.T, _, callersFile string, _ *cell) { + writeFile(t, callersFile, "spiffe://codefly.test/ns/platform/sa/somebody-else\n") + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + c := newCell(t) + certFile, keyFile, bundleFile, caller, roots := c.workload(t, testPrincipal) + callersFile := identitiesFile(t, testGatewayPrincipal) + + server := New(Manifest{ID: testSolutionID}) + server.cfg = config{port: listenOn(t, server), identityCertFile: certFile, identityKeyFile: keyFile, + trustBundleFile: bundleFile, allowedCallersFile: callersFile} + server.principal = testPrincipal + + ln, err := server.listen() + if err != nil { + t.Fatalf("listen: %v", err) + } + defer func() { _ = ln.Close() }() + watch, err := server.watchInboundTrust() + if err != nil { + t.Fatalf("watchInboundTrust: %v", err) + } + // The production server, so the hook under test is the one a + // deployment runs with rather than one this test installs. + srv := &http.Server{ + Handler: http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusNoContent) }), + ReadHeaderTimeout: inboundHandshakeTimeout, + IdleTimeout: inboundIdleTimeout, + ConnState: watch, + } + go func() { _ = srv.Serve(ln) }() + defer func() { _ = srv.Close() }() + + client := &http.Client{Transport: &http.Transport{ + TLSClientConfig: &tls.Config{RootCAs: roots, Certificates: []tls.Certificate{*caller}, + MinVersion: tls.VersionTLS13, ServerName: "localhost"}, + }} + url := "https://" + ln.Addr().String() + "/" + + // The control: an admitted caller is served, and keeps being + // served on one busy connection while trust holds. + resp, err := client.Get(url) + if err != nil { + t.Fatalf("an admitted caller was refused: %v", err) + } + _ = resp.Body.Close() + staysUsable(t, client, url, 2*inboundTrustRecheckInterval) + + tc.withdraw(t, bundleFile, callersFile, c) + + if err := busyUntilRefused(t, client, url, 4*inboundTrustRecheckInterval); err == nil { + t.Fatalf("this caller was still being served %s after the trust that admitted it was withdrawn, on the connection it already held: re-reading trust per handshake bounds nothing about a connection that has already handshaken", + 4*inboundTrustRecheckInterval) + } + }) + } +} + +// TestTheListenerBoundsAStalledPeer drives the stall through the production +// server, because the constants are not the thing under test. +// +// The previous version asserted the constants and then built an http.Server of +// its own with them set, so removing ReadHeaderTimeout from serve() — the only +// place that wires them — left it passing. A test that configures the thing it +// is checking is checking its own literals. +func TestTheListenerBoundsAStalledPeer(t *testing.T) { + // Neither of the two that would cut a conforming long-running stream. A + // declared stream may run to MaxStreamDurationLimit, so bounding the whole + // body would refuse conforming traffic to answer a question the recheck + // answers without refusing any. + if inboundHandshakeTimeout >= MaxStreamDurationLimit { + t.Fatalf("the handshake deadline %s is not shorter than the longest declared stream %s, so it is not bounding anything", inboundHandshakeTimeout, MaxStreamDurationLimit) + } + + mint := newHostMint(t, &hostMint{}) + solution := boot(t, New(Manifest{ID: testSolutionID}), mint) + + // A peer that completes the TCP connect and then sends half a ClientHello + // and nothing else. It presents no certificate and never will. + raw, err := net.Dial("tcp", strings.TrimPrefix(solution.base, "https://")) + if err != nil { + t.Fatalf("connect to the booted listener: %v", err) + } + defer func() { _ = raw.Close() }() + // A TLS record header claiming a handshake, then silence. + if _, err := raw.Write([]byte{0x16, 0x03, 0x01, 0x00, 0x40}); err != nil { + t.Fatalf("write a partial ClientHello: %v", err) + } + + // The server has to end it. Without a deadline it holds the goroutine and + // the descriptor for as long as the peer likes, and every hello costs a + // read of the trust bundle. + _ = raw.SetReadDeadline(time.Now().Add(inboundHandshakeTimeout + 5*time.Second)) + var buf [1]byte + _, err = raw.Read(buf[:]) + if err == nil { + t.Fatal("the listener answered a peer that sent half a ClientHello") + } + if errors.Is(err, os.ErrDeadlineExceeded) { + t.Fatalf("a peer that stalled mid-ClientHello was still connected after %s: with no handshake or header deadline on the booted server it holds a goroutine and a file descriptor for as long as it likes, and each attempt costs a read of the trust bundle", + inboundHandshakeTimeout) + } +} + +// TestANonAtomicRotationDoesNotFailANewDial is the other half of the +// last-good-pair property, and the half only a fresh dial can show. +// +// The SDK's reloader swallows a failed or half-written re-read and keeps +// serving the pair it already has, which is what makes a rotation invisible to +// whoever is dialling. Building a new reloader per dial throws that away, +// because the constructor loads eagerly and returns the error: with the +// certificate replaced and the key a moment behind, every new dial fails until +// both files settle. Nothing about the destination is wrong. +func TestANonAtomicRotationDoesNotFailANewDial(t *testing.T) { + c := newCell(t) + certFile, keyFile, bundleFile, _, _ := c.workload(t, testPrincipal) + host := newPlatformHost(t, c, testGatewayPrincipal) + + server := New(Manifest{ID: testSolutionID}) + server.cfg = config{identityCertFile: certFile, identityKeyFile: keyFile, trustBundleFile: bundleFile, + mintPeersFile: identitiesFile(t, testGatewayPrincipal), + gatewayPeersFile: identitiesFile(t, testGatewayPrincipal), + // The destination this test dials, so the authorization set for it is + // the mint's: the set is chosen per destination now. + mintURL: host.URL + credentialMintPath} + server.principal = testPrincipal + client, err := server.outboundClient(nil) + if err != nil { + t.Fatalf("outboundClient: %v", err) + } + transport, ok := client.Transport.(*http.Transport) + if !ok { + t.Fatalf("outbound transport is %T, want *http.Transport", client.Transport) + } + + resp, err := client.Get(host.URL + credentialMintPath) + if err != nil { + t.Fatalf("the platform was not reachable: %v", err) + } + // Drained, not just closed: an undrained body leaves the connection out of + // the pool, so CloseIdleConnections below closes nothing and the next + // request reuses it — which would make this test pass without ever + // dialling, which is the one thing it asserts. + _, _ = io.Copy(io.Discard, resp.Body) + _ = resp.Body.Close() + + // Halfway through a rotation: the certificate on disk no longer pairs with + // the key beside it. + writeFile(t, certFile, "-----BEGIN CERTIFICATE-----\nhalf a rotation\n-----END CERTIFICATE-----\n") + + // Force a genuinely new connection, so the dial path resolves the leaf + // rather than reusing one that already presented it. + transport.CloseIdleConnections() + ctx, dialled := freshDial(t) + request, err := http.NewRequestWithContext(ctx, http.MethodGet, host.URL+credentialMintPath, nil) + if err != nil { + t.Fatal(err) + } + resp, err = client.Do(request) + if err != nil { + t.Fatalf("a new dial failed while a rotation was half-written: the reloader keeps the last good pair for exactly this, and resolving a new one per dial reports the half-written state as a failure to reach the platform: %v", err) + } + _, _ = io.Copy(io.Discard, resp.Body) + _ = resp.Body.Close() + dialled() +} + +// freshDial asserts that the request made under the context it returns really +// dialled, rather than being served on a connection that was already open. +// +// Two tests here closed a response body without draining it, so the connection +// had not returned to the pool when CloseIdleConnections ran; the next request +// reused it and made no dial. One then flaked under -race (65 of 200), and the +// other passed vacuously — it asserted that a dial resolves the leaf, on a +// request that never dialled. A reused connection performs no TCP dial, which +// is what this watches for, and it works whether the request succeeds or is +// refused. +// +// Not "never handshook", which is what this comment and the failure message +// used to say: on this transport nothing observes a handshake at all (see +// below), so describing the symptom that way named a signal that is absent +// either way. +func freshDial(t *testing.T) (context.Context, func()) { + t.Helper() + var dialled atomic.Bool + // ConnectStart, not TLSHandshakeStart: this runtime's outbound transport + // sets DialTLSContext, and a custom DialTLSContext never reaches net/http's + // addTLS — which is what invokes the TLS trace hooks. So + // TLSHandshakeStart never fires on this transport at all, whether or not a + // connection was reused, and watching for it would be another signal that + // cannot distinguish the thing it was chosen to distinguish. ConnectStart + // fires on a real TCP dial, including one whose handshake is then refused, + // which is the case one of these two tests needs. + trace := &httptrace.ClientTrace{ + ConnectStart: func(string, string) { dialled.Store(true) }, + } + return httptrace.WithClientTrace(context.Background(), trace), func() { + t.Helper() + if !dialled.Load() { + t.Fatal("the request was served on a connection that was already open, so it made no TCP dial and this test asserted nothing about dialling: drain the previous response body before CloseIdleConnections, or the connection has not returned to the pool when it runs") + } + } +} + +// issueMultiURILeaf is a leaf naming two SPIFFE identities, which no SPIFFE +// certificate has. The rule it tests is not pedantry about a field: a leaf +// naming several identities could be admitted on whichever one happens to +// match, and the identity it would be held to anywhere else is unknowable from +// here. +func issueMultiURILeaf(t *testing.T, c *cell, first, second string) *tls.Certificate { + t.Helper() + public, private, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + t.Fatal(err) + } + one, err := url.Parse(first) + if err != nil { + t.Fatal(err) + } + two, err := url.Parse(second) + if err != nil { + t.Fatal(err) + } + template := &x509.Certificate{ + SerialNumber: big.NewInt(time.Now().UnixNano()), + Subject: pkix.Name{CommonName: first}, + NotBefore: time.Now().Add(-time.Minute), + NotAfter: time.Now().Add(time.Hour), + KeyUsage: x509.KeyUsageDigitalSignature, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth}, + DNSNames: []string{"localhost"}, + IPAddresses: []net.IP{net.ParseIP("127.0.0.1")}, + URIs: []*url.URL{one, two}, + } + der, err := x509.CreateCertificate(rand.Reader, template, c.anchor, public, c.anchorKey) + if err != nil { + t.Fatal(err) + } + return &tls.Certificate{Certificate: [][]byte{der}, PrivateKey: private} +} + +// TestALeafNamingSeveralIdentitiesIsRefusedInEveryDirection: a mutation that +// accepted the first URI SAN of several survived in three places, because +// every committed test used a conforming single-SAN leaf. +func TestALeafNamingSeveralIdentitiesIsRefusedInEveryDirection(t *testing.T) { + c := newCell(t) + doubled := issueMultiURILeaf(t, c, testGatewayPrincipal, rivalPrincipal) + + t.Run("an inbound caller", func(t *testing.T) { + approved := c.identity(t, testPrincipal) + server := New(Manifest{ID: testSolutionID}).Identity(staticIdentity{config: serverConfigFor(t, approved, c)}) + server.cfg = config{allowedCallersFile: identitiesFile(t, testGatewayPrincipal)} + server.principal = testPrincipal + config, err := server.serverIdentity() + if err != nil { + t.Fatalf("boot: %v", err) + } + outcome := servedTo(t, config, doubled, "") + if !outcome.refused() { + t.Fatal("a caller whose leaf names two identities was admitted: it can be accepted on whichever one matches, and the identity it is held to elsewhere is unknowable here") + } + }) + + t.Run("this workload's own leaf", func(t *testing.T) { + if err := pairPresentsThisWorkload(doubled, testGatewayPrincipal); err == nil { + t.Fatal("this workload's own leaf was accepted while naming two identities, one of which is another workload's") + } + }) + + t.Run("an outbound platform destination", func(t *testing.T) { + if _, err := oneURIIdentity([]*x509.Certificate{parsedLeaf(t, doubled)}, "platform destination"); err == nil { + t.Fatal("a platform destination naming two identities was accepted, so the one it is held to is whichever the check happened to read first") + } + }) +} + +// parsedLeaf is a pair's leaf, parsed from its own DER rather than read off the +// Leaf cache. +func parsedLeaf(t *testing.T, pair *tls.Certificate) *x509.Certificate { + t.Helper() + leaf, err := x509.ParseCertificate(pair.Certificate[0]) + if err != nil { + t.Fatal(err) + } + return leaf +} + +// serverConfigFor is a conforming listening configuration presenting pair, +// verifying callers against the cell. +func serverConfigFor(t *testing.T, pair *tls.Certificate, c *cell) *tls.Config { + t.Helper() + return &tls.Config{ + Certificates: []tls.Certificate{*pair}, + ClientAuth: tls.RequireAndVerifyClientCert, + ClientCAs: c.roots, + MinVersion: tls.VersionTLS13, + } +} + +// TestAPerConnectionAnswerIsHeldToTheAdmittedSet: holdPerConnectionPosture +// re-imposes admitOnly on the configuration a callback returns, because that +// configuration REPLACES the base one for the connection — so without it a +// source's callback serves that connection admitting every identity in the +// cell. The mutation removing it survived: every committed test drove the base +// configuration. +func TestAPerConnectionAnswerIsHeldToTheAdmittedSet(t *testing.T) { + c := newCell(t) + approved := c.identity(t, testPrincipal) + stranger := c.identity(t, rivalPrincipal) + + base := serverConfigFor(t, approved, c) + base.GetConfigForClient = func(*tls.ClientHelloInfo) (*tls.Config, error) { + // A conforming answer that simply says nothing about admission. + answer := serverConfigFor(t, approved, c) + return answer, nil + } + server := New(Manifest{ID: testSolutionID}).Identity(staticIdentity{config: base}) + server.cfg = config{allowedCallersFile: identitiesFile(t, testGatewayPrincipal)} + server.principal = testPrincipal + config, err := server.serverIdentity() + if err != nil { + t.Fatalf("boot: %v", err) + } + + // The control: the admitted caller is served over the per-connection answer. + admitted := c.identity(t, testGatewayPrincipal) + if outcome := servedTo(t, config, admitted, ""); outcome.refused() { + t.Fatalf("an admitted caller was refused over the per-connection answer, so nothing below is about admission: %s", outcome.reason()) + } + // And a caller nobody admitted is refused on that same answer. + if outcome := servedTo(t, config, stranger, ""); !outcome.refused() { + t.Fatal("a caller outside the provisioned set was served over a per-connection answer: the configuration a callback returns replaces the base one, so the admission has to be re-imposed on it or that connection admits every identity in the cell") + } +} + +// A source's own VerifyConnection used to be composed over this runtime's +// admission, and a test here asserted it still ran. That capability is gone: +// the hook handed a source pointers to the chain the rechecks read, so it is +// refused at boot (TestASourceCannotHoldTheChainTheRechecksRead). The +// composition in admitOnly is kept for the outbound direction, where the +// configuration being composed is this package's own. + +// mentionsTheCertificate reports whether a boot refusal is about the identity +// the listener would present, rather than any other way a boot can fail. +func mentionsTheCertificate(err error) bool { + // "identity source" is NOT in this list, and that is the correction. + // + // It was, and nearly every boot refusal in identity.go opens with those + // two words — including the PeerAnchorSource refusal, which is about the + // anchor and says nothing about a certificate. So the guard written to + // stop "returning on *any* boot error" was satisfied by a refusal for an + // unrelated reason, which is the same defect one layer up. + for _, named := range []string{ + "issued for another workload", + "certificate", + "principal", + } { + if strings.Contains(err.Error(), named) { + return true + } + } + return false +} + +// TestTheOutboundHandshakeIsBoundedWhateverTheCallerPassed: a custom +// DialTLSContext takes net/http out of the handshake, so +// Transport.TLSHandshakeTimeout does nothing and the only bound left is the +// caller's context. +// +// That is adequate for an ordinary request and wrong where it matters: +// Gateway.HTTPClient carries streams and therefore sets no client timeout on +// purpose, so a dial made for a stream was bounded by the method's declared +// MaxStreamDuration — up to thirty minutes — or by nothing at all. This dials +// with a context carrying no deadline at all, against a peer that accepts TCP +// and never speaks TLS. +func TestTheOutboundHandshakeIsBoundedWhateverTheCallerPassed(t *testing.T) { + c := newCell(t) + certFile, keyFile, bundleFile, _, _ := c.workload(t, testPrincipal) + + // A black hole: it accepts and says nothing, which is what a peer doing + // this deliberately looks like. + silent, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer func() { _ = silent.Close() }() + go func() { + for { + conn, err := silent.Accept() + if err != nil { + return + } + // Held open, never answered. + t.Cleanup(func() { _ = conn.Close() }) + } + }() + + server := New(Manifest{ID: testSolutionID}) + server.cfg = config{identityCertFile: certFile, identityKeyFile: keyFile, trustBundleFile: bundleFile, + mintPeersFile: identitiesFile(t, testGatewayPrincipal), + gatewayPeersFile: identitiesFile(t, testGatewayPrincipal), + // The black hole is this test's "mint", so the dial has an + // authorization set at all: the set is chosen per destination now. + mintURL: "https://" + silent.Addr().String() + credentialMintPath} + server.principal = testPrincipal + server.handshakeTimeout = 750 * time.Millisecond + client, err := server.outboundClient(nil) + if err != nil { + t.Fatalf("outboundClient: %v", err) + } + transport, ok := client.Transport.(*http.Transport) + if !ok { + t.Fatalf("outbound transport is %T, want *http.Transport", client.Transport) + } + + dialled := make(chan error, 1) + go func() { + // context.Background(): no deadline, which is what a stream's dial can + // carry. + conn, err := transport.DialTLSContext(context.Background(), "tcp", silent.Addr().String()) + if conn != nil { + _ = conn.Close() + } + dialled <- err + }() + select { + case err := <-dialled: + if err == nil { + t.Fatal("the dial succeeded against a peer that never spoke TLS") + } + case <-time.After(10 * server.handshakeTimeout): + t.Fatalf("the handshake was still running %s after a dial with no deadline on its context: the custom dialler means net/http does not bound it and TLSHandshakeTimeout does nothing, so a dial made for a stream is bounded by the stream's duration or by nothing", + 10*server.handshakeTimeout) + } +} + +// TestTheInboundWatchStartsAtAuthentication: the watch covered the connection +// from the first *request*, which is later than authentication. A caller could +// complete its handshake, have its issuing root pulled, and sit unwatched for +// as long as the header deadline allows. +// +// This caller authenticates and then sends nothing at all, which is the window +// in question. +func TestTheInboundWatchStartsAtAuthentication(t *testing.T) { + c := newCell(t) + certFile, keyFile, bundleFile, caller, roots := c.workload(t, testPrincipal) + callersFile := identitiesFile(t, testGatewayPrincipal) + + server := New(Manifest{ID: testSolutionID}) + server.cfg = config{port: listenOn(t, server), identityCertFile: certFile, identityKeyFile: keyFile, + trustBundleFile: bundleFile, allowedCallersFile: callersFile} + server.principal = testPrincipal + ln, err := server.listen() + if err != nil { + t.Fatalf("listen: %v", err) + } + defer func() { _ = ln.Close() }() + watch, err := server.watchInboundTrust() + if err != nil { + t.Fatalf("watchInboundTrust: %v", err) + } + srv := &http.Server{ + Handler: http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusNoContent) }), + ReadHeaderTimeout: inboundHandshakeTimeout, + IdleTimeout: inboundIdleTimeout, + ConnState: watch, + } + go func() { _ = srv.Serve(ln) }() + defer func() { _ = srv.Close() }() + + conn, err := tls.Dial("tcp", ln.Addr().String(), &tls.Config{ + RootCAs: roots, Certificates: []tls.Certificate{*caller}, + MinVersion: tls.VersionTLS13, ServerName: "localhost", + }) + if err != nil { + t.Fatalf("an admitted caller could not connect: %v", err) + } + defer func() { _ = conn.Close() }() + if err := conn.Handshake(); err != nil { + t.Fatalf("handshake: %v", err) + } + + // Authenticated, and not one byte of a request sent. + // + // The control runs first, and it is also what makes the withdrawal below + // unambiguous: while admission holds, this connection stays open across + // several recheck intervals. Withdrawing immediately after tls.Dial raced + // the server's own side of the handshake — the client finishes before the + // server has evaluated admission, so the server read the *new* file and + // refused the handshake, and the test passed on "bad certificate" whether + // the watch worked or not. + stillOpen := func(t *testing.T) bool { + t.Helper() + _ = conn.SetReadDeadline(time.Now().Add(inboundTrustRecheckInterval / 4)) + var buf [1]byte + _, err := conn.Read(buf[:]) + return errors.Is(err, os.ErrDeadlineExceeded) + } + settle := time.Now().Add(2 * inboundTrustRecheckInterval) + for time.Now().Before(settle) { + if !stillOpen(t) { + t.Fatal("the connection was closed while this caller was still admitted, so nothing below is about the withdrawal") + } + } + + // The platform now removes this caller. + writeFile(t, callersFile, "spiffe://codefly.test/ns/platform/sa/somebody-else\n") + + // Well inside the header deadline, which is what would otherwise be the + // only thing to end this connection. + deadline := time.Now().Add(5 * inboundTrustRecheckInterval) + if 5*inboundTrustRecheckInterval >= inboundHandshakeTimeout { + t.Fatalf("this test needs to finish inside the header deadline (%s) to mean anything", inboundHandshakeTimeout) + } + for time.Now().Before(deadline) { + if !stillOpen(t) { + return // closed, which is the point + } + } + t.Fatalf("a caller that authenticated and then sent nothing was still connected %s after its admission was withdrawn: the watch begins at the first request, so the whole pre-request window is unwatched", 5*inboundTrustRecheckInterval) +} + +// TestAResumedConnectionCannotBeAdmittedWithoutACertificate is the executed +// bypass from round five, committed. +// +// A resumed TLS connection presents no certificate: the peer is accepted on a +// ticket. So anything able to forge a ticket is admitted as whoever the ticket +// was issued to, and a denylist could not close that — it caught +// SessionTicketKey and could not catch SetSessionTicketKeys, whose keys are +// unexported and unreadable from the configuration. The review drove it end to +// end: with a known key, a client holding only that key and an admitted +// caller's ticket resumed as that caller with DidResume=true, presenting +// nothing, and the per-second recheck then re-verified the stolen certificate +// and kept the connection open. +// +// Resumption is therefore off on everything this listener serves, which is why +// this test asserts a property of the configuration rather than replaying a +// ticket: there is no ticket to replay. +func TestAResumedConnectionCannotBeAdmittedWithoutACertificate(t *testing.T) { + c := newCell(t) + certFile, keyFile, bundleFile, caller, roots := c.workload(t, testPrincipal) + + // A source that deliberately fixes its ticket key, which is the HA-replica + // shape, and reaches the bypass through the field the denylist can see. + keyed, err := projectedIdentity{certFile: certFile, keyFile: keyFile, trustBundleFile: bundleFile}.ServerTLSConfig() + if err != nil { + t.Fatal(err) + } + //nolint:staticcheck // Deliberately sets the deprecated field, to prove the refusal fires. + keyed.SessionTicketKey = [32]byte{7} + if err := usableServerIdentity(keyed); err == nil { + t.Error("a source fixing its own resumption ticket key was accepted: anyone holding that key can forge a ticket this listener resumes, and a resumed connection presents no certificate") + } + + // And the half a check cannot see: keys installed through the method. + // Nothing can read them back, so the only sound answer is that this + // listener does not resume at all. + viaMethod, err := projectedIdentity{certFile: certFile, keyFile: keyFile, trustBundleFile: bundleFile}.ServerTLSConfig() + if err != nil { + t.Fatal(err) + } + viaMethod.SetSessionTicketKeys([][32]byte{{9}}) + server := New(Manifest{ID: testSolutionID}).Identity(staticIdentity{config: viaMethod}) + server.cfg = config{allowedCallersFile: identitiesFile(t, testGatewayPrincipal)} + server.principal = testPrincipal + served, err := server.serverIdentity() + if err != nil { + t.Fatalf("boot: %v", err) + } + if !served.SessionTicketsDisabled { + t.Fatal("the configuration this listener serves permits resumption while the source controls the ticket keys: a ticket forged with those keys is admitted as whoever it was issued to, with no certificate presented and nothing in this package able to notice") + } + + // Every per-connection answer too, since that configuration replaces the + // base one for its connection. + perConnection, err := projectedIdentity{certFile: certFile, keyFile: keyFile, trustBundleFile: bundleFile}.ServerTLSConfig() + if err != nil { + t.Fatal(err) + } + if perConnection.GetConfigForClient == nil { + t.Fatal("the projected source stopped answering per connection, so this case no longer covers anything") + } + answered, err := served.GetConfigForClient(&tls.ClientHelloInfo{ServerName: "localhost"}) + if err != nil { + t.Fatalf("the per-connection answer was refused: %v", err) + } + if answered != nil && !answered.SessionTicketsDisabled { + t.Error("a per-connection answer permits resumption: the configuration a callback returns replaces the base one for that connection, so disabling it on the base alone leaves the bypass open") + } + + // The control: an admitted caller is still served, so this is not a + // listener that refuses everything. + if outcome := servedTo(t, served, caller, "localhost"); outcome.refused() { + t.Fatalf("an admitted caller was refused with resumption off: %s", outcome.reason()) + } + _ = roots +} + +// TestEachDestinationHasItsOwnAuthorizationSet: the mint and the gateway are +// two parties, and one set spanning both authorises each to stand in for the +// other at the other's address. The set already excluded every other workload +// in the cell, which was the finding it was added for, and that is a different +// question from whether "the platform" is one party. +func TestEachDestinationHasItsOwnAuthorizationSet(t *testing.T) { + c := newCell(t) + certFile, keyFile, bundleFile, _, _ := c.workload(t, testPrincipal) + const gatewayIdentity = "spiffe://codefly.test/ns/platform/sa/gateway" + const mintIdentity = "spiffe://codefly.test/ns/platform/sa/mint" + + // Two hosts: one answering as the mint, one as the gateway. + mint := newPlatformHost(t, c, mintIdentity) + gateway := newPlatformHost(t, c, gatewayIdentity) + + server := New(Manifest{ID: testSolutionID}) + server.cfg = config{identityCertFile: certFile, identityKeyFile: keyFile, trustBundleFile: bundleFile, + mintURL: mint.URL + credentialMintPath, + gatewayURL: gateway.URL, + mintPeersFile: identitiesFile(t, mintIdentity), + gatewayPeersFile: identitiesFile(t, gatewayIdentity), + } + server.principal = testPrincipal + client, err := server.outboundClient(nil) + if err != nil { + t.Fatalf("outboundClient: %v", err) + } + + t.Run("each destination admits its own identity", func(t *testing.T) { + for _, at := range []struct{ name, url string }{ + {"the mint", mint.URL + credentialMintPath}, + {"the gateway", gateway.URL + "/anything"}, + } { + resp, err := client.Get(at.url) + if err != nil { + t.Errorf("%s was refused while answering under its own provisioned identity: %v", at.name, err) + continue + } + _ = resp.Body.Close() + } + }) + + t.Run("neither stands in for the other", func(t *testing.T) { + // The gateway's identity, answering at the mint's address. Under one + // shared set this was accepted, and the projected service-account + // token went to it. + impostor := newPlatformHost(t, c, gatewayIdentity) + server.cfg.mintURL = impostor.URL + credentialMintPath + impersonating, err := server.outboundClient(nil) + if err != nil { + t.Fatalf("outboundClient: %v", err) + } + resp, err := impersonating.Get(impostor.URL + credentialMintPath) + if err == nil { + _ = resp.Body.Close() + t.Fatal("the gateway's identity was accepted at the credential mint's address: a set spanning both parties lets either present itself as the other, and the mint is the destination that receives the projected service-account token") + } + if !strings.Contains(err.Error(), gatewayIdentity) { + t.Errorf("the refusal %q does not name the identity that answered", err) + } + }) + + t.Run("a third address has no set and is refused", func(t *testing.T) { + // This runtime dials the mint and the gateway and nothing else, so + // there is no set that should admit a third destination. + server.cfg.mintURL = mint.URL + credentialMintPath + fresh, err := server.outboundClient(nil) + if err != nil { + t.Fatalf("outboundClient: %v", err) + } + stranger := newPlatformHost(t, c, mintIdentity) + resp, err := fresh.Get(stranger.URL + credentialMintPath) + if err == nil { + _ = resp.Body.Close() + t.Fatal("a destination that is neither the mint nor the gateway was dialled with this workload's credentials") + } + if !strings.Contains(err.Error(), "no authorization set") { + t.Errorf("the refusal %q does not say the address has no set, so it may be refusing for some other reason", err) + } + }) +} + +// TestOneReaderDoesNotFallBackToItsLastGoodAnswer: the admission reader must +// refuse an unusable file even after it has successfully read a usable one. +// +// Every case in TestAnAdmissionSetIsNeitherCachedNorDefaulted builds a fresh +// reader, so no good read ever precedes the bad one — and a reader that cached +// its last good answer and served it whenever a later read failed passed all of +// them. That fallback is precisely the staleness this whole mechanism exists to +// remove: an admission set that cannot be re-read has no safe direction, +// because the set on disk may be narrower than the one in memory. +func TestOneReaderDoesNotFallBackToItsLastGoodAnswer(t *testing.T) { + path := filepath.Join(t.TempDir(), "identities") + writeFile(t, path, testGatewayPrincipal+"\n") + admits := resolvedIdentities(path, "unresolved") + + // A good read first, so a cache would have something to fall back to. + got, err := admits() + if err != nil { + t.Fatalf("a usable set was refused: %v", err) + } + if len(got) != 1 || got[0] != testGatewayPrincipal { + t.Fatalf("resolved %v, want %q", got, testGatewayPrincipal) + } + + for _, tc := range []struct { + name string + then func(t *testing.T) + }{ + {"the file is emptied", func(t *testing.T) { writeFile(t, path, "\n") }}, + {"the file is removed", func(t *testing.T) { + if err := os.Remove(path); err != nil { + t.Fatal(err) + } + }}, + {"the file is replaced by a directory", func(t *testing.T) { + if err := os.Remove(path); err != nil { + t.Fatal(err) + } + if err := os.Mkdir(path, 0o755); err != nil { + t.Fatal(err) + } + }}, + } { + t.Run(tc.name, func(t *testing.T) { + // Restored for each case, so each one is a good read followed by + // this bad one on the *same* reader. + _ = os.RemoveAll(path) + writeFile(t, path, testGatewayPrincipal+"\n") + if _, err := admits(); err != nil { + t.Fatalf("the restored set was refused: %v", err) + } + tc.then(t) + if got, err := admits(); err == nil { + t.Errorf("the reader resolved %v after a good read followed by an unusable file: it is serving a remembered answer, which is the staleness this mechanism exists to remove", got) + } + }) + } +} + +// TestAFreshDialFollowsTheProvisionedPeerSet is the per-dial half, and the one +// an established connection hides. +// +// TestAnEstablishedConnectionFollowsTheProvisionedPeerSet keeps a connection +// busy, so the recheck can carry it: a runtime that snapshotted the set when +// the client was built but still rechecked established connections passed it. +// This makes a genuinely new dial after the set changes, with nothing +// established to recheck. +func TestAFreshDialFollowsTheProvisionedPeerSet(t *testing.T) { + c := newCell(t) + certFile, keyFile, bundleFile, _, _ := c.workload(t, testPrincipal) + host := newPlatformHost(t, c, testGatewayPrincipal) + provisioned := identitiesFile(t, testGatewayPrincipal) + + server := New(Manifest{ID: testSolutionID}) + server.cfg = config{identityCertFile: certFile, identityKeyFile: keyFile, trustBundleFile: bundleFile, + mintPeersFile: provisioned, + gatewayPeersFile: provisioned, + mintURL: host.URL + credentialMintPath} + server.principal = testPrincipal + client, err := server.outboundClient(nil) + if err != nil { + t.Fatalf("outboundClient: %v", err) + } + transport, ok := client.Transport.(*http.Transport) + if !ok { + t.Fatalf("outbound transport is %T, want *http.Transport", client.Transport) + } + + resp, err := client.Get(host.URL + credentialMintPath) + if err != nil { + t.Fatalf("the provisioned destination was refused: %v", err) + } + // Drained, not just closed. Undrained, the connection has not returned to + // the pool when CloseIdleConnections runs, so the second request reuses it + // and is served without a handshake — the test then fails because nothing + // refused it, which is how this flaked 65 times in 200 under -race. + _, _ = io.Copy(io.Discard, resp.Body) + _ = resp.Body.Close() + + // The destination is removed, and every connection to it is dropped, so + // the next request must dial afresh and the recheck has nothing to carry. + writeFile(t, provisioned, "spiffe://codefly.test/ns/platform/sa/somebody-else\n") + transport.CloseIdleConnections() + + ctx, dialled := freshDial(t) + request, err := http.NewRequestWithContext(ctx, http.MethodGet, host.URL+credentialMintPath, nil) + if err != nil { + t.Fatal(err) + } + resp, err = client.Do(request) + if err == nil { + _, _ = io.Copy(io.Discard, resp.Body) + _ = resp.Body.Close() + t.Fatal("a fresh dial was made to a destination removed from the provisioned set: the set has to be read per dial, not captured when the client was built") + } + dialled() + if !strings.Contains(err.Error(), testGatewayPrincipal) { + t.Errorf("the refusal %q does not name the identity that answered", err) + } +} + +// TestASourceCarryingBothAPoolAndACallbackIsRefused is N4 again, and the +// regression this round's own fix introduced. +// +// The refusal lived in peerAnchorFrom, which reached it only when the base pool +// was nil — so a source carrying BOTH a base pool and a per-handshake callback +// was accepted at boot, and then had that pool frozen for every recheck and +// every outbound dial while its callback went on resolving a fresh anchor. +// Executed: after the bundle rotated A→B, a fresh A-caller was refused and the +// established A-caller served 30 requests over 3 seconds. The doc comment and +// the README both claimed such a source was refused. The only shape that was +// refused is the one nobody writes. +func TestASourceCarryingBothAPoolAndACallbackIsRefused(t *testing.T) { + boot := newCell(t) + current := newCell(t) + approved := boot.identity(t, testPrincipal) + + withCallback := func() *tls.Config { + return &tls.Config{ + Certificates: []tls.Certificate{*approved}, + ClientAuth: tls.RequireAndVerifyClientCert, + MinVersion: tls.VersionTLS13, + // A pool from boot *and* a callback resolving a fresh one. + ClientCAs: boot.roots, + GetConfigForClient: func(*tls.ClientHelloInfo) (*tls.Config, error) { + return &tls.Config{ + Certificates: []tls.Certificate{*approved}, + ClientAuth: tls.RequireAndVerifyClientCert, + MinVersion: tls.VersionTLS13, + ClientCAs: current.roots, + }, nil + }, + } + } + + server := New(Manifest{ID: testSolutionID}).Identity(unaskableIdentity{config: withCallback()}) + server.cfg = config{allowedCallersFile: identitiesFile(t, testGatewayPrincipal)} + server.principal = testPrincipal + if _, err := server.serverIdentity(); err == nil { + t.Fatal("a source resolving its configuration per handshake, carrying a base pool as well, was accepted: that pool is one object fixed when the source built it, so every recheck and every outbound dial would be judged by the anchor this process booted with while the callback resolved a fresh one") + } else if !strings.Contains(err.Error(), "PeerAnchor") { + t.Errorf("the refusal %q does not name the interface that fixes it", err) + } + + // The control: the same shape, askable, boots. + askable := New(Manifest{ID: testSolutionID}).Identity(staticIdentity{ + config: withCallback(), + anchor: func() *x509.CertPool { return current.roots }, + }) + askable.cfg = config{allowedCallersFile: identitiesFile(t, testGatewayPrincipal)} + askable.principal = testPrincipal + if _, err := askable.serverIdentity(); err != nil { + t.Fatalf("a source that can say what its anchor is was refused: %v", err) + } +} + +// TestASharedDestinationAddressIsHeldToBothSets: the per-destination split was +// silently not in force in the only configuration that exists. +// +// admittedAt matched the mint first, so when the mint and the gateway share a +// host and port — the brokered shape the PR proposes and every fixture here +// uses — every gateway dial answered to the mint's set, and the gateway's set +// was validated at boot and then never read. Executed: a gateway set naming +// somebody else still got 200, and rewriting it cut nothing. +func TestASharedDestinationAddressIsHeldToBothSets(t *testing.T) { + c := newCell(t) + certFile, keyFile, bundleFile, _, _ := c.workload(t, testPrincipal) + const other = "spiffe://codefly.test/ns/platform/sa/somebody-else" + host := newPlatformHost(t, c, testGatewayPrincipal) + + build := func(t *testing.T, mintSet, gatewaySet []string) *http.Client { + t.Helper() + server := New(Manifest{ID: testSolutionID}) + server.cfg = config{identityCertFile: certFile, identityKeyFile: keyFile, trustBundleFile: bundleFile, + // One address for both, which is the deployed shape. + mintURL: host.URL + credentialMintPath, + gatewayURL: host.URL, + mintPeersFile: identitiesFile(t, mintSet...), + gatewayPeersFile: identitiesFile(t, gatewaySet...), + } + server.principal = testPrincipal + client, err := server.outboundClient(nil) + if err != nil { + t.Fatalf("outboundClient: %v", err) + } + return client + } + + t.Run("both sets admit it", func(t *testing.T) { + resp, err := build(t, []string{testGatewayPrincipal}, []string{testGatewayPrincipal}).Get(host.URL + credentialMintPath) + if err != nil { + t.Fatalf("an identity both sets admit was refused: %v", err) + } + _ = resp.Body.Close() + }) + + // The two halves that used to pass because only the mint's set was read. + for _, tc := range []struct { + name string + mintSet, gatewaySet []string + }{ + {"only the mint's set admits it", []string{testGatewayPrincipal}, []string{other}}, + {"only the gateway's set admits it", []string{other}, []string{testGatewayPrincipal}}, + } { + t.Run(tc.name, func(t *testing.T) { + resp, err := build(t, tc.mintSet, tc.gatewaySet).Get(host.URL + credentialMintPath) + if err == nil { + _ = resp.Body.Close() + t.Fatal("an identity only one of the two sets admits was presented this workload's credentials at an address both destinations share: a dial there cannot be attributed to one of them, so only an identity both admit is safe — and reading one set alone means the other is provisioned, validated at boot, and never consulted again") + } + }) + } + + t.Run("an empty intersection is refused at boot", func(t *testing.T) { + server := New(Manifest{ID: testSolutionID}) + server.cfg = config{port: "8080", identityCertFile: certFile, identityKeyFile: keyFile, + projectedTokenPath: "t", trustBundleFile: bundleFile, profile: localProfile, + mintURL: host.URL + credentialMintPath, + gatewayURL: host.URL, + allowedCallersFile: identitiesFile(t, testGatewayPrincipal), + mintPeersFile: identitiesFile(t, testGatewayPrincipal), + gatewayPeersFile: identitiesFile(t, other), + } + server.principal = testPrincipal + err := server.validateSources() + if err == nil { + t.Fatal("a boot accepted two sets with nothing in common at an address both destinations share: every dial there would be refused, discovered at the first mint rather than named as the provisioning gap it is") + } + if !strings.Contains(err.Error(), "both provisioned sets") { + t.Errorf("the refusal %q does not say the sets share no identity", err) + } + }) +} + +// TestACallersIdentityComesFromTheBytesItSigned: admission is decided on the +// certificate's own DER, not on the parsed object a ConnectionState hands out. +// +// Those objects are shared — tls.Conn returns the same pointers every call — so +// anything that ever holds one can change what a later read sees. The hooks +// that could hold them are refused at boot now; this is the other half, and it +// is what makes the refusal belt rather than the only line. +func TestACallersIdentityComesFromTheBytesItSigned(t *testing.T) { + c := newCell(t) + rival := c.identity(t, rivalPrincipal) + + // A parsed leaf rewritten to claim an identity the set admits, over a DER + // that says otherwise. + leaf := parsedLeaf(t, rival) + claimed, err := url.Parse(testGatewayPrincipal) + if err != nil { + t.Fatal(err) + } + leaf.URIs = []*url.URL{claimed} + state := tls.ConnectionState{PeerCertificates: []*x509.Certificate{leaf}} + + admit := verifyCaller(func() ([]string, error) { return []string{testGatewayPrincipal}, nil }) + err = admit(state) + if err == nil { + t.Fatal("a caller was admitted on a rewritten parsed leaf: the identity has to come from the bytes the peer signed, which nothing between the handshake and the check holds a reference to") + } + if !strings.Contains(err.Error(), rivalPrincipal) { + t.Errorf("the refusal %q does not name the identity the certificate was actually issued for", err) + } + + // The control: the genuinely admitted caller passes. + intact := tls.ConnectionState{PeerCertificates: []*x509.Certificate{parsedLeaf(t, c.identity(t, testGatewayPrincipal))}} + if err := admit(intact); err != nil { + t.Fatalf("an admitted caller was refused: %v", err) + } +} + +// The ordering test that stood here drove a source-supplied VerifyConnection, +// which is refused at boot now, so the scenario is unreachable. Ordering alone +// was never the fix: it corrected the admission and left the rechecks reading +// objects the source still held. See +// TestASourceCannotHoldTheChainTheRechecksRead. + +// TestASourceCannotHoldTheChainTheRechecksRead is the blocker R7-5 only half +// closed. +// +// Running this runtime's admission before a source's verifier fixes the +// admission and nothing else. A tls.ConnectionState carries *pointers* to the +// parsed chain and tls.Conn hands out the same ones on every call — so a +// source's VerifyConnection does not merely see the certificates, it holds +// them, and the per-second recheck reads those same objects minutes later. An +// established caller whose leaf was rewritten afterwards survives withdrawal, +// because the recheck re-reads the mutated object and agrees with it. +// +// Two things close it: such a source is refused at boot, and nothing this +// runtime decides is computed from the shared objects at all. +func TestASourceCannotHoldTheChainTheRechecksRead(t *testing.T) { + c := newCell(t) + approved := c.identity(t, testPrincipal) + + t.Run("a source supplying its own connection verifier is refused", func(t *testing.T) { + base := serverConfigFor(t, approved, c) + base.VerifyConnection = func(tls.ConnectionState) error { return nil } + server := New(Manifest{ID: testSolutionID}).Identity(staticIdentity{config: base}) + server.cfg = config{allowedCallersFile: identitiesFile(t, testGatewayPrincipal)} + server.principal = testPrincipal + _, err := server.serverIdentity() + if err == nil { + t.Fatal("a source supplying its own VerifyConnection was accepted: it is handed pointers to the chain this listener authenticates callers by, and the rechecks read those same objects for as long as the connection lives, so anything written to them decides who stays admitted") + } + if !strings.Contains(err.Error(), "VerifyConnection") { + t.Errorf("the refusal %q does not name the field", err) + } + }) + + t.Run("a mutated chain cannot keep a withdrawn caller admitted", func(t *testing.T) { + // The recheck's own input, mutated the way a verifier holding the + // objects would: the leaf claims an identity the set admits while its + // DER says otherwise. + admitted, rival := c.identity(t, testGatewayPrincipal), c.identity(t, rivalPrincipal) + leaf := parsedLeaf(t, rival) + claimed, err := url.Parse(testGatewayPrincipal) + if err != nil { + t.Fatal(err) + } + leaf.URIs = []*url.URL{claimed} + state := tls.ConnectionState{PeerCertificates: []*x509.Certificate{leaf}} + + anchor := func() (*x509.CertPool, error) { return c.roots, nil } + set := func() ([]string, error) { return []string{testGatewayPrincipal}, nil } + if err := callerStillAdmitted(state, anchor, set, "", nil); err == nil { + t.Fatal("the recheck admitted a caller whose parsed leaf had been rewritten to claim an admitted identity: it has to decide on the bytes the peer signed, which nothing between the handshake and the check holds a reference to") + } + + // The control: the genuinely admitted caller passes the same recheck, + // so this is not a check that refuses everything. + intact := tls.ConnectionState{PeerCertificates: []*x509.Certificate{parsedLeaf(t, admitted)}} + if err := callerStillAdmitted(intact, anchor, set, "", nil); err != nil { + t.Fatalf("an admitted caller was refused by the recheck: %v", err) + } + }) + + t.Run("the private chain does not alias the handshake's objects", func(t *testing.T) { + leaf := parsedLeaf(t, approved) + state := tls.ConnectionState{PeerCertificates: []*x509.Certificate{leaf}} + chain, err := immutableChain(state) + if err != nil { + t.Fatalf("immutableChain: %v", err) + } + if chain[0] == leaf { + t.Fatal("the chain is the handshake's own object, so anything holding it can change what a recheck reads") + } + if &chain[0].Raw[0] == &leaf.Raw[0] { + t.Fatal("the chain shares the handshake's DER bytes: copying the struct without copying the bytes leaves the parse re-derivable from memory something else can write") + } + // And it still says the same thing. + if chain[0].URIs[0].String() != testPrincipal { + t.Errorf("the private copy names %v, want %q", chain[0].URIs, testPrincipal) + } + }) +} + +// TestAnEstablishedConnectionIsHeldToTheIdentityItWasAdmittedAs carries the +// executed round's two probes: the key swap and the Raw swap. +// +// tls.ConnectionState.PeerCertificates shares its backing array with the +// connection's own slice, and TLS 1.3 runs VerifyConnection BEFORE checking +// CertificateVerify against peerCertificates[0].PublicKey. A source verifier +// that swapped the leaf pointer therefore had the handshake signature verified +// against the RIVAL's key: an attacker holding only an admitted identity's +// public certificate plus its own private key was served 16 requests in 2.5s, +// against a control of 0. A Raw swap was the quieter half — the recheck +// re-parsed the swapped DER and agreed with it, so a de-admitted caller kept +// being served. +// +// Both hooks that could do this are refused at boot, and the recheck holds a +// connection to the identity it was admitted as rather than re-asking who it +// is now. +func TestAnEstablishedConnectionIsHeldToTheIdentityItWasAdmittedAs(t *testing.T) { + c := newCell(t) + anchor := func() (*x509.CertPool, error) { return c.roots, nil } + set := func() ([]string, error) { return []string{testGatewayPrincipal}, nil } + + t.Run("the key swap is refused at boot", func(t *testing.T) { + // The shape of the attack: a source verifier that replaces the leaf + // with one carrying an admitted identity and the attacker's own key. + base := serverConfigFor(t, c.identity(t, testPrincipal), c) + base.VerifyConnection = func(state tls.ConnectionState) error { + state.PeerCertificates[0] = parsedLeaf(t, c.identity(t, testGatewayPrincipal)) + return nil + } + server := New(Manifest{ID: testSolutionID}).Identity(staticIdentity{config: base}) + server.cfg = config{allowedCallersFile: identitiesFile(t, testGatewayPrincipal)} + server.principal = testPrincipal + if _, err := server.serverIdentity(); err == nil { + t.Fatal("a source verifier able to swap the leaf was accepted: TLS 1.3 checks CertificateVerify against peerCertificates[0].PublicKey after VerifyConnection runs, so the handshake would be authenticated against the swapped key") + } + }) + + t.Run("a Raw swap cannot keep a de-admitted caller served", func(t *testing.T) { + admitted := parsedLeaf(t, c.identity(t, testGatewayPrincipal)) + pinnedID, pinnedLeaf, err := admittedIdentity(tls.ConnectionState{PeerCertificates: []*x509.Certificate{admitted}}) + if err != nil { + t.Fatalf("admittedIdentity: %v", err) + } + + // The connection's own object swapped under the recheck for another + // certificate that still verifies and still names an admitted + // identity. Re-deriving the identity each tick agrees with it; holding + // the connection to what it was admitted as does not. + reissued := parsedLeaf(t, c.identity(t, testGatewayPrincipal)) + swapped := tls.ConnectionState{PeerCertificates: []*x509.Certificate{reissued}} + if err := callerStillAdmitted(swapped, anchor, set, pinnedID, pinnedLeaf); err == nil { + t.Fatal("the recheck accepted a connection whose caller certificate is not the one the handshake authenticated: an established connection's certificate does not change, so this is not a caller to re-authorise but one whose authentication means nothing") + } + + // An identity swap, the louder half. + rival := tls.ConnectionState{PeerCertificates: []*x509.Certificate{parsedLeaf(t, c.identity(t, rivalPrincipal))}} + if err := callerStillAdmitted(rival, anchor, set, pinnedID, pinnedLeaf); err == nil { + t.Fatal("the recheck accepted a connection presenting a different identity than the one it was admitted as") + } + + // The control: the connection as it actually was stays served, so this + // is not a check that closes everything. + intact := tls.ConnectionState{PeerCertificates: []*x509.Certificate{admitted}} + if err := callerStillAdmitted(intact, anchor, set, pinnedID, pinnedLeaf); err != nil { + t.Fatalf("the connection the handshake authenticated was closed by its own recheck: %v", err) + } + }) + + t.Run("withdrawal still closes a pinned connection", func(t *testing.T) { + // Pinning says which caller this is; it must not make the connection + // immortal. The admitted set is still read every tick. + admitted := parsedLeaf(t, c.identity(t, testGatewayPrincipal)) + state := tls.ConnectionState{PeerCertificates: []*x509.Certificate{admitted}} + pinnedID, pinnedLeaf, err := admittedIdentity(state) + if err != nil { + t.Fatal(err) + } + withdrawn := func() ([]string, error) { return []string{rivalPrincipal}, nil } + if err := callerStillAdmitted(state, anchor, withdrawn, pinnedID, pinnedLeaf); err == nil { + t.Fatal("a pinned connection survived the withdrawal of its caller from the admitted set") + } + }) +} diff --git a/identity.go b/identity.go new file mode 100644 index 0000000..27f8113 --- /dev/null +++ b/identity.go @@ -0,0 +1,1286 @@ +package solution + +import ( + "bytes" + "crypto/tls" + "crypto/x509" + "fmt" + "net" + "net/http" + "net/url" + "os" + "slices" + "strings" + "sync" + "time" + + codefly "github.com/codefly-dev/sdk-go" +) + +// IdentitySource is where this workload's listening identity comes from. +// +// It is a hook, and that is the point: issuing an X.509-SVID is the platform's +// job, not this runtime's. A deployment whose identities come from a mesh CA, a +// SPIFFE workload API, a cert-manager Certificate projected into the pod, or a +// pair the local toolchain mints for a developer run differ in every detail of +// how the leaf arrives and in none of what the listener does with it. The +// default source (the platform projection named by the workload-identity +// configuration group) covers the deployed case and the local one identically, +// because both are "files at a path somebody else wrote"; a consumer whose +// platform hands out leaves another way implements this instead. +// +// A source returns a server configuration rather than a certificate so that an +// issuer with its own rotation, revocation or peer-verification rules keeps +// them: everything the handshake needs is in the value it returns. +type IdentitySource interface { + // ServerTLSConfig is the configuration this solution's listener serves + // with. It is called once, at boot, before the listener exists; a source + // that rotates material does so inside the returned configuration (through + // GetCertificate), not by being called again. + ServerTLSConfig() (*tls.Config, error) +} + +// PeerAnchorSource is an IdentitySource that can say what its current trust +// anchor is, asked directly. +// +// Optional, and the only sound way for a source that resolves its anchor per +// handshake to be re-checkable. The alternative this replaces was to call the +// source's GetConfigForClient with a *fabricated* ClientHelloInfo and read the +// pool off the answer — which is the defect this whole cutover was blocked on +// one layer along: a source keyed on the hello answers a synthetic probe with +// one thing and the real handshake with another, and that was demonstrated +// serving a neighbouring workload's certificate. The same trick is worse for an +// anchor than for a certificate. A certificate obtained through a fabricated +// hello is still held to the frozen principal at the moment it is presented, so +// a wrong answer is caught; an anchor *is* the thing that catches wrong +// answers, and nothing downstream checks it. +// +// A source that resolves its anchor per handshake and does not implement this +// is refused at boot, because there is then nothing to re-verify an established +// caller against that this runtime could obtain honestly. +type PeerAnchorSource interface { + PeerAnchor() (*x509.CertPool, error) +} + +// Identity sets where the listener's workload identity comes from, replacing +// the platform projection the workload-identity configuration group names. +// Chainable. +func (s *Server) Identity(source IdentitySource) *Server { + s.identity = source + return s +} + +// projectedIdentity is the default source: the X.509-SVID the platform projects +// for this workload, as files, and the trust anchor its peers are verified +// against. +// +// It is read through the SDK's certificate reloader, which re-reads the pair +// when the files change and keeps serving the last good pair through a +// half-written replacement. That matters more here than anywhere else in this +// runtime: workload leaves are deliberately short-lived and rotated well before +// expiry, so a process that loaded its leaf once at boot — the shape almost +// every Go server has — serves a stale leaf while a valid one sits on disk, and +// then fails every handshake, with the issuer reporting nothing wrong. +type projectedIdentity struct { + certFile, keyFile, trustBundleFile string +} + +// PeerAnchor reads the projected bundle, now. It is what makes this source +// re-checkable: the anchor it serves with is resolved per handshake, so asking +// the configuration for it would mean fabricating a handshake, and this answers +// the question directly instead. See PeerAnchorSource. +func (p projectedIdentity) PeerAnchor() (*x509.CertPool, error) { + return peerAnchor(p.trustBundleFile) +} + +func (p projectedIdentity) ServerTLSConfig() (*tls.Config, error) { + // The anchor is read here only to fail the boot on an unusable one. What + // the handshake uses is read again, per handshake, below. + if _, err := peerAnchor(p.trustBundleFile); err != nil { + return nil, err + } + // The SDK owns the floor (TLS 1.3) and the leaf's reload; this runtime owns + // which files, which anchor, and that a peer is authenticated at all. + config, err := codefly.ServerTLSConfig(p.certFile, p.keyFile, nil) + if err != nil { + return nil, fmt.Errorf("load this workload's X.509-SVID from %q/%q: %w — the listener presents it and there is no plain-HTTP listener; see %s and %s", + p.certFile, p.keyFile, err, IdentityCertFileEnvironmentVariable, IdentityKeyFileEnvironmentVariable) + } + // Peer trust is resolved per handshake rather than snapshotted into + // ClientCAs at boot. + // + // A pool read once is a pool that cannot shrink: removing a compromised + // root from the projected bundle would leave it authenticating clients for + // the life of the process, and the SDK's reloader reloads the leaf this + // listener presents, not the anchor it judges callers by. Revocation that + // takes effect only on a restart is revocation the operator has to + // remember to finish. + // + // A bundle that has become unreadable or unusable fails the handshake + // rather than falling back to the last good pool. The two failures are not + // symmetric: serving a stale *leaf* refuses callers who should be let in, + // while judging by a stale *anchor* lets in callers who should be refused. + bundle := p.trustBundleFile + config.GetConfigForClient = func(*tls.ClientHelloInfo) (*tls.Config, error) { + anchor, err := peerAnchor(bundle) + if err != nil { + return nil, err + } + fresh := config.Clone() + fresh.ClientCAs = anchor + fresh.ClientAuth = tls.RequireAndVerifyClientCert + fresh.GetConfigForClient = nil + return fresh, nil + } + config.ClientAuth = tls.RequireAndVerifyClientCert + return config, nil +} + +// peerAnchor reads the trust anchor this listener verifies its callers against. +// +// There is no "no anchor" case. A listener that verifies no peer certificate +// accepts any caller that can route to the pod, which bypasses the admission +// and exposure decisions the host makes on its own routes — so an absent anchor +// is a boot refusal naming the value, not a listener that quietly authenticates +// nobody. It was conditional once, on the grounds that the gateway presents no +// client certificate today; that is a statement about a counterpart's current +// state, and leaving a hole open for as long as the counterpart takes is the +// concession this cutover exists to stop making. +func peerAnchor(bundleFile string) (*x509.CertPool, error) { + if bundleFile == "" { + return nil, fmt.Errorf("no workload identity trust anchor resolved: this listener requires and verifies a caller's certificate, so it cannot start without the anchor those certificates are issued under. Set %s, or have the platform provision %s/%s", + IdentityTrustBundleFileEnvironmentVariable, WorkloadIdentityGroup, WorkloadIdentityTrustBundleFileKey) + } + pem, err := os.ReadFile(bundleFile) + if err != nil { + return nil, fmt.Errorf("read workload identity trust anchor %q: %w — it is what this listener verifies its callers against, so an unreadable one refuses the handshake rather than admitting everyone", + bundleFile, err) + } + anchor := x509.NewCertPool() + if !anchor.AppendCertsFromPEM(pem) { + return nil, fmt.Errorf("workload identity trust anchor %q contains no certificate: the listener would otherwise verify no peer at all", bundleFile) + } + return anchor, nil +} + +// listen binds this solution's port and wraps it in the workload's identity. +// +// There is no plain-HTTP path. The host refuses a plain-HTTP destination, so a +// listener without an identity would be a solution that boots, looks healthy, +// and is refused at the edge for a reason only the edge can see — which is the +// failure the whole delivered-presence model exists to end. A configuration +// that cannot produce an identity is refused here, naming the material that is +// missing. +func (s *Server) listen() (net.Listener, error) { + config, err := s.serverIdentity() + if err != nil { + return nil, err + } + ln := s.boundListener + if ln == nil { + var err error + if ln, err = net.Listen("tcp", ":"+s.cfg.port); err != nil { + return nil, err + } + } + // The per-connection recheck is installed on the *server*, not here: see + // watchInboundTrust. A listener that wrapped its connections would hide + // the *tls.Conn from net/http, which decides from that concrete type alone + // whether a connection is TLS at all — so r.TLS would be nil in every + // handler, ALPN and HTTP/2 would not be negotiated, and the handshake + // deadline would never be set. The recheck must not cost the listener its + // TLS-ness to get it. + return tls.NewListener(ln, config), nil +} + +// inboundHandshakeTimeout bounds the TLS handshake and the request headers of +// an accepted connection. +// +// http.Server derives its handshake deadline from the smallest of +// ReadHeaderTimeout, ReadTimeout and WriteTimeout, so this one value bounds +// both. With all three unset, as they were, any peer that completes a TCP +// connect and then sends a partial ClientHello — or a header line at a byte a +// minute — holds a goroutine and a file descriptor indefinitely, and every +// ClientHello costs a read of the trust bundle, so the cheap half of that is +// the attacker's. Neither ReadTimeout nor WriteTimeout is set, deliberately: +// they bound the whole body, and a declared long-running stream (up to +// MaxStreamDurationLimit) is a conforming request, not a slow one. +const inboundHandshakeTimeout = 10 * time.Second + +// inboundIdleTimeout bounds a keep-alive connection nobody is using. It is pool +// hygiene, not the trust bound — what bounds the trust decision is the recheck, +// because an inactivity timeout cannot bound a connection that is never +// inactive, which is the finding this runtime already answered outbound. +const inboundIdleTimeout = 30 * time.Second + +// peerTrustAnchor resolves the anchor this runtime verifies its peers against, +// per call, in whichever direction. +// +// One resolution for both directions because it is one anchor: the listener +// verifies a caller against it and the client verifies the platform against it, +// which is what makes a removed root take effect on both halves at once. +func (s *Server) peerTrustAnchor(identityConfig *tls.Config) func() (*x509.CertPool, error) { + if s.identity != nil { + source := s.identity + return func() (*x509.CertPool, error) { + anchor, err := peerAnchorFrom(source, identityConfig) + if err != nil { + return nil, fmt.Errorf("resolve the anchor this workload verifies its peers against, from the supplied identity source: %w", err) + } + return anchor, nil + } + } + return func() (*x509.CertPool, error) { + anchor, err := peerAnchor(s.cfg.trustBundleFile) + if err != nil { + return nil, fmt.Errorf("resolve the anchor this workload verifies its peers against: %w", err) + } + return anchor, nil + } +} + +// watchInboundTrust re-verifies every served connection's caller against +// current trust for as long as the connection lives, and closes it when it +// stops verifying. +// +// It is the inbound half of an argument this file already made outbound, and +// made in one direction only. Re-reading trust per *handshake* bounds nothing +// about a connection that has already had its handshake: a caller whose +// identity was removed from the admitted set, or whose issuing root was pulled +// from the bundle, kept the keep-alive connection it already had and kept being +// served on it. Inbound is where the decision is whom to *admit*, which is the +// direction where judging by something stale lets in whoever should be refused. +// +// Installed through http.Server.ConnState so the connection stays the +// *tls.Conn net/http requires it to be. StateActive is the first point at which +// there is a peer chain to re-verify — crypto/tls handshakes lazily, so at +// Accept there is nothing yet — and StateClosed/StateHijacked is where the +// watch ends. +func (s *Server) watchInboundTrust() (func(net.Conn, http.ConnState), error) { + config, err := s.serverIdentity() + if err != nil { + return nil, err + } + trust := s.peerTrustAnchor(config) + admitted := s.admittedCallers() + var watching sync.Map + return func(conn net.Conn, state http.ConnState) { + switch state { + // StateNew, not StateActive: the watch begins when the connection is + // accepted, so it covers the connection from the moment it is + // authenticated. StateActive is the first point a *request* has been + // read, which is later — a caller could complete its handshake, + // have its issuing root pulled, and sit inside the header deadline + // unwatched. The recheck loop waits for the handshake itself rather + // than for traffic. + case http.StateNew: + tlsConn, ok := conn.(*tls.Conn) + if !ok { + return + } + watched := &watchedCaller{done: make(chan struct{})} + if _, already := watching.LoadOrStore(conn, watched); already { + return + } + go recheckCaller(tlsConn, watched, trust, admitted) + case http.StateActive: + // The identity is recorded HERE, synchronously, before the handler + // runs — not on the recheck's first tick. + // + // Pinning on the first tick left a one-second window, and I wrote + // that it was sound because "no in-process hook can run inside + // it". That was false, and executed round eight proved it: a + // RequestHandler is an in-process hook, it is handed r.TLS, and + // tls.ConnectionState shares its certificates with the connection. + // A handler rewriting PeerCertificates[0].Raw inside the first + // second had the pin record the rewritten identity, and a + // de-admitted caller was served 465 requests over 10 seconds on + // one connection. + // + // net/http calls this hook after readRequest and before + // ServeHTTP (server.go, setState(StateActive, runHooks)), so this + // runs ahead of any author code on the first request and every + // one after it. It does not run for HTTP/2, where net/http passes + // skipHooks — which is why TestTheListenerDoesNotNegotiateHTTP2 + // exists: this listener advertises no ALPN protocol and is served + // with Serve rather than ServeTLS, so nothing negotiates h2 today, + // and if that changes the pin has to move rather than quietly stop + // happening. + if watched, ok := watching.Load(conn); ok { + if tlsConn, isTLS := conn.(*tls.Conn); isTLS { + watched.(*watchedCaller).pin(tlsConn.ConnectionState()) + } + } + case http.StateClosed, http.StateHijacked: + if watched, ok := watching.LoadAndDelete(conn); ok { + watched.(*watchedCaller).stop() + } + } + }, nil +} + +// watchedCaller is one watched connection's lifetime and the identity it was +// admitted as. +// +// The identity is recorded once, by whichever of the two reaches it first: the +// ConnState hook at the first request, or the recheck's own first tick for a +// connection that handshakes and then sends nothing. Both read it from a +// private parse (see immutableChain); the difference is that the hook runs +// before any handler, and the tick does not. +type watchedCaller struct { + done chan struct{} + stopOnce sync.Once + mu sync.Mutex + identity string + leaf []byte + refused error +} + +// pin records the identity this connection was authenticated as, if nothing has +// yet. +func (w *watchedCaller) pin(state tls.ConnectionState) { + if !state.HandshakeComplete { + return + } + w.mu.Lock() + defer w.mu.Unlock() + if w.identity != "" || w.refused != nil { + return + } + identity, leaf, err := admittedIdentity(state) + if err != nil { + w.refused = err + return + } + w.identity, w.leaf = identity, leaf +} + +// pinned is what was recorded, and any error that recording it produced. +func (w *watchedCaller) pinned() (string, []byte, error) { + w.mu.Lock() + defer w.mu.Unlock() + return w.identity, w.leaf, w.refused +} + +func (w *watchedCaller) stop() { + w.stopOnce.Do(func() { close(w.done) }) +} + +// recheckCaller re-verifies one established caller until the connection ends. +func recheckCaller(conn *tls.Conn, watched *watchedCaller, trust func() (*x509.CertPool, error), admitted func() ([]string, error)) { + ticker := time.NewTicker(inboundTrustRecheckInterval) + defer ticker.Stop() + for { + select { + case <-watched.done: + return + case <-ticker.C: + // Until the handshake completes there is no peer chain to judge, + // and an unhandshaken connection is not a caller who should be + // refused — it is bounded by the handshake deadline instead. This + // is why the watch can start at accept: it waits for + // authentication rather than for the first request. + state := conn.ConnectionState() + if !state.HandshakeComplete { + continue + } + // For a connection that handshook and has sent nothing, no request + // has reached StateActive, so nothing has pinned it yet and no + // handler has run either. Pinning here is then reading the chain + // as the handshake left it. + watched.pin(state) + pinned, leaf, err := watched.pinned() + if err != nil { + _ = conn.Close() + return + } + if err := callerStillAdmitted(state, trust, admitted, pinned, leaf); err != nil { + _ = conn.Close() + return + } + } + } +} + +// inboundTrustRecheckInterval bounds how long an accepted connection may +// outlive the trust that admitted it. The outbound interval's reasoning, in the +// direction where being wrong means serving a caller who should be refused. +const inboundTrustRecheckInterval = time.Second + +// callerStillAdmitted re-verifies an established caller against the current +// anchor and the current admitted set. +// +// Client auth, not server auth, and no hostname: a caller is held to the chain +// and the identity it presented, which is the same pair of questions the +// handshake answered, asked again against trust as it is now. +func callerStillAdmitted(state tls.ConnectionState, trust func() (*x509.CertPool, error), admitted func() ([]string, error), pinned string, pinnedLeaf []byte) error { + anchor, err := trust() + if err != nil { + return err + } + chain, err := immutableChain(state) + if err != nil { + return err + } + if len(chain) == 0 { + return fmt.Errorf("the established connection presents no caller certificate to re-verify") + } + intermediates := x509.NewCertPool() + for _, cert := range chain[1:] { + intermediates.AddCert(cert) + } + if _, err := chain[0].Verify(x509.VerifyOptions{ + Roots: anchor, + Intermediates: intermediates, + KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, + }); err != nil { + return fmt.Errorf("the caller this connection was authenticated against no longer verifies: %w", err) + } + allowed, err := admitted() + if err != nil { + return err + } + identity, err := oneURIIdentity(chain, "caller") + if err != nil { + return err + } + // Still the caller this connection was admitted as. A connection whose + // certificate changed under it is not a caller whose authorisation can be + // re-evaluated — it is one whose authentication no longer means anything, + // so it is closed rather than judged. + if pinned != "" && identity != pinned { + return fmt.Errorf("this connection was admitted as %q and now presents %q: the certificate authenticating an established connection does not change, so there is nothing left to re-authorise", pinned, identity) + } + if pinnedLeaf != nil && !bytes.Equal(chain[0].Raw, pinnedLeaf) { + return fmt.Errorf("this connection's caller certificate is not the one the handshake authenticated, though it still names %q", identity) + } + if !slices.Contains(allowed, identity) { + return fmt.Errorf("the established connection's caller %q is no longer one this solution admits", identity) + } + return nil +} + +// admittedIdentity is the identity and leaf an established connection was +// authenticated as, taken from a private parse so what is recorded cannot be +// changed afterwards through the objects the connection shares. +func admittedIdentity(state tls.ConnectionState) (string, []byte, error) { + chain, err := immutableChain(state) + if err != nil { + return "", nil, err + } + identity, err := oneURIIdentity(chain, "caller") + if err != nil { + return "", nil, err + } + return identity, chain[0].Raw, nil +} + +// serverIdentity resolves this workload's identity once and holds it to every +// posture rule, memoised so the boot and the listener are the same identity +// rather than two resolutions that could differ. +// +// It is called *before* the first outbound act, which is the ordering half of a +// real bug: the credential mint used to run before listen(), so a leaf issued +// for a neighbouring workload had already been presented to the host — along +// with this workload's projected token — by the time the principal check +// refused the listener. The check that says "this is the workload the platform +// provisioned" is worth nothing if the mint has already happened under the +// wrong one. +func (s *Server) serverIdentity() (*tls.Config, error) { + if s.identityConfig != nil { + return s.identityConfig, nil + } + source := s.identity + if source == nil { + source = projectedIdentity{ + certFile: s.cfg.identityCertFile, + keyFile: s.cfg.identityKeyFile, + trustBundleFile: s.cfg.trustBundleFile, + } + } + answered, err := source.ServerTLSConfig() + if err != nil { + return nil, err + } + if err := usableServerIdentity(answered); err != nil { + return nil, err + } + // A source that resolves anything per handshake has to be askable, and + // carrying a base pool as well does not make it so. + // + // The refusal used to live in peerAnchorFrom, which reached it only when + // the base pool was nil — so a source with BOTH a base pool and a + // per-handshake callback was accepted, and then had the base pool frozen + // for every recheck and every outbound dial while its callback went on + // resolving a fresh one. Executed: after the bundle rotated A→B a fresh + // A-caller was refused and the ESTABLISHED A-caller served 30 requests + // over 3 seconds. This file and the README both said that source was + // refused at boot. It was not, and the half that was true was the half + // nobody writes. + if answered.GetConfigForClient != nil { + if _, askable := source.(PeerAnchorSource); !askable { + return nil, fmt.Errorf("the identity source resolves its configuration per handshake (GetConfigForClient is set) and does not implement PeerAnchorSource, so this runtime cannot obtain the anchor it re-verifies established peers against — or verifies the platform it dials against — except through a handshake nobody made. Carrying a pool on the base configuration does not answer it: that pool is one object fixed when the source built it, so it would freeze the anchor this process judges by. Implement PeerAnchor() (*x509.CertPool, error)") + } + } + // On a clone, because everything below writes to it: the certificate hold + // wraps GetCertificate, admitOnly composes over VerifyConnection, and the + // per-connection hold wraps GetConfigForClient. A source is under no + // obligation to hand back a fresh configuration — a consumer may return one + // it keeps and reuses, which is the ordinary way to write one — so mutating + // the answer is a write to a value another handshake may be reading, and a + // second call would wrap the wrapper. + // + // holdPerConnectionPosture already says this about a *per-connection* + // answer, in those words, and this function then did the thing that comment + // forbids one level up. + config := answered.Clone() + refuseResumption(config) + if err := holdServedCertificate(config, s.principal); err != nil { + return nil, err + } + admitOnly(config, s.admittedCallers()) + holdPerConnectionPosture(config, s.principal, s.admittedCallers()) + s.identityConfig = config + return config, nil +} + +// admitOnly decides which authenticated callers this listener actually serves. +// +// Verifying a caller's certificate against the cell's anchor answers "does this +// caller hold an identity the platform issued". It does not answer "may this +// caller call me", and in a cell those are very different sets: every workload +// in the trust domain holds such a certificate, including the modules this +// solution consumes. Without this, any of them could reach a handler or a +// passthrough route directly, bypassing the admission the host decides on its +// own routes — and because gatewayFor reads x-org-id and x-session-id as +// headers the gateway stamped from a verified bearer, such a caller could set +// them itself and have this runtime mint capabilities, under this workload's +// attestation, for an organization and session nobody authenticated. A confused +// deputy, where the deputy is the one process the issuer trusts to say which +// module is asking. +// +// So the admitted set is declared, provisioned, and refused at boot when +// absent (validateSources) — not derived from the anchor, which is exactly the +// set that is too wide. The comparison is the SPIFFE ID in the leaf's URI SAN, +// the same thing the listener's own identity is held to. +func admitOnly(config *tls.Config, allowed func() ([]string, error)) { + // Assigned, not composed, and that is a consequence rather than a + // preference. + // + // This composed over a source's own VerifyConnection, so that a consumer's + // check was not silently deleted by adding one. Then the order had to + // change — tls.ConnectionState carries pointers to the parsed chain, so a + // source's verifier running first could edit the certificate this + // admission reads, and rewriting the leaf's URI SANs was demonstrated + // getting a rival caller in. Then the hook was refused outright, because + // holding those pointers also let it change what the per-second rechecks + // read minutes later, and there is no safe way to lend a source the + // objects this listener authenticates by. + // + // So the composition had nothing left to compose: usableServerIdentity + // refuses a source that sets VerifyConnection, on the base configuration + // and on every per-connection answer, and clientTLSFrom builds the + // outbound configuration from scratch rather than copying one. The branch + // was kept here with a comment claiming it served the outbound direction. + // It did not, and a mutation that removed it survived the whole suite. + // Legacy means delete. + config.VerifyConnection = verifyCaller(allowed) +} + +// verifyCaller is the check itself, so it can be applied to a per-connection +// configuration as well as the base one. The set is resolved per handshake, not +// captured: see config.resolveAllowedCallers. +func verifyCaller(allowed func() ([]string, error)) func(tls.ConnectionState) error { + return func(state tls.ConnectionState) error { + admitted, err := allowed() + if err != nil { + return err + } + chain, err := immutableChain(state) + if err != nil { + return err + } + identity, err := oneURIIdentity(chain, "caller") + if err != nil { + return err + } + if slices.Contains(admitted, identity) { + return nil + } + // The refusal names what arrived, not the allowed set: a caller is not + // told who else may call. + return fmt.Errorf("refusing a caller whose identity %q is not one this solution admits: it holds a certificate from this cell's anchor, which is not the same as being a caller this solution serves — see %s and %s/%s", + identity, IdentityAllowedCallersFileEnvironmentVariable, WorkloadIdentityGroup, WorkloadIdentityAllowedCallersFileKey) + } +} + +// admitOnlyPlatform is admitOnly for the destinations this runtime dials: the +// mint and the gateway. +// +// Verifying the chain and the hostname answers "did this cell issue the +// certificate at this address", which every workload in the cell satisfies — +// and a workload holding a certificate valid for the gateway's hostname +// completed this handshake and was handed the projected service-account token, +// the viewer's bearer and this workload's own credential. Authentication is not +// authorisation outbound either, and outbound is the direction where being +// wrong means handing the credentials over rather than accepting a call. +// +// Composed over whatever the configuration already verifies, never assigned +// over it, for the reason admitOnly records. +func admitOnlyPlatform(config *tls.Config, expected func() ([]string, error)) { + // Assigned too, for the same reason: clientTLSFrom builds this + // configuration field by field and never carries a source's + // VerifyConnection onto it, so there was never anything here to compose + // with either. + config.VerifyConnection = func(state tls.ConnectionState) error { + peers, err := expected() + if err != nil { + return err + } + chain, err := immutableChain(state) + if err != nil { + return err + } + identity, err := oneURIIdentity(chain, "platform destination") + if err != nil { + return err + } + if slices.Contains(peers, identity) { + return nil + } + return fmt.Errorf("refusing to present this workload's credentials to %q: it holds a certificate from this cell's anchor for the address this runtime dialled, which is not the same as being the party that answers there — each destination has its own provisioned set under %s, so neither the mint nor the gateway can stand in for the other", + identity, WorkloadIdentityGroup) + } +} + +// immutableChain is a private copy of the chain a handshake authenticated, +// parsed from copied bytes. +// +// Nothing this runtime decides is computed from the *x509.Certificate objects +// crypto/tls put in the ConnectionState. Those are shared: tls.Conn returns the +// same pointers on every call, so anything that was ever handed them can change +// what a later read sees — and the rechecks read minutes after the handshake. +// The source-supplied verifiers that could hold them are refused outright, and +// this is the other half: even if something acquires them, the identity and the +// re-verification are computed from bytes nothing else has a reference to. +// +// The copy is of the DER, not of the parsed struct: copying the struct would +// copy the pointers inside it. +func immutableChain(state tls.ConnectionState) ([]*x509.Certificate, error) { + chain := make([]*x509.Certificate, 0, len(state.PeerCertificates)) + for i, presented := range state.PeerCertificates { + der := make([]byte, len(presented.Raw)) + copy(der, presented.Raw) + parsed, err := x509.ParseCertificate(der) + if err != nil { + return nil, fmt.Errorf("the certificate at position %d of this peer's chain cannot be parsed from the bytes it presented: %w", i, err) + } + chain = append(chain, parsed) + } + return chain, nil +} + +// oneURIIdentity is the SPIFFE ID a verified peer presented. +// +// Exactly one URI SAN, because that is what a SPIFFE certificate has. Accepting +// "any SAN that matches" lets a leaf naming several identities be admitted on +// whichever one is convenient, and a leaf naming several is not a SPIFFE +// identity at all — the one it would be held to elsewhere is unknowable from +// here. +func oneURIIdentity(chain []*x509.Certificate, side string) (string, error) { + if len(chain) == 0 { + return "", fmt.Errorf("refusing a %s that presented no certificate", side) + } + leaf := chain[0] + if len(leaf.URIs) != 1 { + return "", fmt.Errorf("refusing a %s whose certificate names %d URI identities: a SPIFFE certificate names exactly one, and a leaf naming several could be accepted on whichever happens to match", + side, len(leaf.URIs)) + } + return leaf.URIs[0].String(), nil +} + +// uriStrings renders a leaf's URI SANs for a refusal. +func uriStrings(uris []*url.URL) []string { + rendered := make([]string, 0, len(uris)) + for _, uri := range uris { + rendered = append(rendered, uri.String()) + } + return rendered +} + +// peerAnchorFrom is the pool this runtime verifies its peers against, for a +// consumer-supplied source, resolved without fabricating a handshake. +// +// The order is: ask the source (PeerAnchorSource), then the anchor it carried +// on the configuration it returned. There is no third option, and in +// particular there is no asking GetConfigForClient with a hello nobody sent — +// see PeerAnchorSource for why that is unsound for an anchor specifically. +// +// Preferring the hook over the base pool is what keeps a rotating source's +// anchor live: a *x509.CertPool on the base configuration is one object fixed +// when the source built it, so a source that rotates its anchor and carries a +// base pool had every peer judged by the pool this process booted with. +func peerAnchorFrom(source IdentitySource, config *tls.Config) (*x509.CertPool, error) { + if asked, ok := source.(PeerAnchorSource); ok { + anchor, err := asked.PeerAnchor() + if err != nil { + return nil, fmt.Errorf("the identity source could not say what trust anchor its peers are verified against: %w", err) + } + if anchor == nil { + return nil, fmt.Errorf("the identity source answered PeerAnchor with no pool, so there is nothing to verify a peer against") + } + return anchor, nil + } + if config.ClientCAs != nil { + return config.ClientCAs, nil + } + return nil, fmt.Errorf("the identity source resolves its trust anchor per handshake and does not implement PeerAnchorSource, so this runtime cannot obtain the anchor to re-verify an established caller — or to verify the platform it dials — without asking the source through a handshake nobody made. Carry the anchor on the configuration ServerTLSConfig returns, or implement PeerAnchor() (*x509.CertPool, error)") +} + +// clientTLSFrom derives the configuration this runtime dials the platform with +// from the one its listener serves: the same certificate, the same anchor. +// +// The certificate is taken through the source's own callback rather than copied, +// so a rotated leaf is presented outbound as well — snapshotting it here would +// mean the listener served a fresh leaf while the mint presented an expired one, +// and the failure would name neither. +func clientTLSFrom(source IdentitySource, config *tls.Config) (*tls.Config, error) { + anchor, err := peerAnchorFrom(source, config) + if err != nil { + return nil, err + } + client := &tls.Config{RootCAs: anchor, MinVersion: tls.VersionTLS13} + switch { + case config.GetCertificate != nil: + get := config.GetCertificate + client.GetClientCertificate = func(*tls.CertificateRequestInfo) (*tls.Certificate, error) { + // An empty hello. There is no real one to pass: this is the + // *client* side, where no ClientHelloInfo exists at all, and the + // source's only certificate callback is the server-side one. + // + // Sound here, unlike for an anchor, and the difference is worth + // stating because the same fabrication is a blocker one field + // along: whatever comes back is held to the frozen principal by + // holdPresentedCertificate, at the moment it is presented, so a + // source that answers this differently from the real handshake is + // caught by that hold. An anchor has no such check behind it — + // it *is* the check — which is why peerAnchorFrom refuses to + // obtain one this way and asks PeerAnchorSource instead. + return get(&tls.ClientHelloInfo{}) + } + case len(config.Certificates) > 0: + client.Certificates = config.Certificates + default: + return nil, fmt.Errorf("the identity source produces no certificate this runtime can present to the platform: it would dial the mint and the gateway as an anonymous client, which cannot be held to this installation") + } + return client, nil +} + +// usableServerIdentity refuses a configuration this listener must not serve +// under. An IdentitySource is consumer code — a mesh CA, a SPIFFE workload API, +// a pair the local toolchain minted — and every one of them has to arrive at +// the same posture, not just the one this package writes itself. +// +// Three things are checked, and each of them was a way in: +// +// - no certificate at all: a source that returns a bare &tls.Config{}, or one +// whose material it silently failed to load, produces a listener that +// completes no handshake and reports a TLS error naming nothing to every +// caller; +// - a floor below TLS 1.3: the default source gets its floor from the SDK, so +// checking only the certificate left a consumer's own config free to accept +// 1.2 — the strongest guarantee in this file silently optional for exactly +// the callers who wrote their own source; +// - a peer it does not authenticate: see peerAnchor. +func usableServerIdentity(config *tls.Config) error { + if config == nil { + return fmt.Errorf("the identity source returned no TLS configuration: this listener presents this workload's X.509-SVID and there is no plain-HTTP listener to fall back to") + } + if config.GetCertificate == nil && config.GetConfigForClient == nil && len(config.Certificates) == 0 { + return fmt.Errorf("the identity source returned a TLS configuration with no certificate: a handshake would be answered with none, so the listener is refused rather than started") + } + if config.MinVersion < tls.VersionTLS13 { + return fmt.Errorf("the identity source returned a TLS configuration whose floor is below TLS 1.3 (MinVersion 0x%04x): set MinVersion to tls.VersionTLS13, which is what every destination in this model is held to", + config.MinVersion) + } + if config.ClientAuth < tls.RequireAndVerifyClientCert { + return fmt.Errorf("the identity source returned a TLS configuration that does not require and verify a caller's certificate (ClientAuth %v): a listener that authenticates no peer accepts anything that can route to it, bypassing the admission its host decides on its routes", + config.ClientAuth) + } + // And it has to say *which* anchor a caller is verified against. This is + // not pedantry about a field: with RequireAndVerifyClientCert and a nil + // ClientCAs, Go verifies client certificates against the host's system + // roots, so the listener demands a certificate and then accepts one from + // any public CA — a posture that reads as mutual TLS in every log line and + // admits the internet. Requiring a certificate and not saying whose is + // strictly worse than requiring none, because it looks like the strong + // thing. + // + // A source may resolve the anchor per handshake instead, which is what the + // projected one does so that a removed root takes effect without a restart; + // then the callback's answer is where the pool must appear, and + // holdPerConnectionPosture checks it there. + if config.ClientCAs == nil && config.GetConfigForClient == nil { + return fmt.Errorf("the identity source returned a TLS configuration that requires a caller's certificate but names no trust anchor to verify it against (ClientCAs is nil): Go would fall back to this host's system roots, so the listener would accept a client certificate from any public CA. Set ClientCAs, or resolve it per handshake in GetConfigForClient") + } + return unsubvertedPosture(config) +} + +// refuseResumption turns session resumption off on the configuration this +// listener actually serves. +// +// Taken, not filtered, and this is the one place in this file where that +// distinction was load-bearing. A resumed TLS connection presents **no +// certificate**: the peer is accepted on the strength of a ticket. This +// listener's whole model is that a caller's certificate names them and the +// admitted set decides whether that name may call, so anything able to forge a +// ticket is able to be admitted as whoever the ticket was issued to, without a +// certificate ever being presented. +// +// A denylist could not close that. It caught `SessionTicketKey`, and it could +// not catch `SetSessionTicketKeys`, which installs unexported keys that cannot +// be read back off the configuration — and a review demonstrated the gap end to +// end: with a known key, a client holding only that key and an admitted +// caller's ticket resumed as that caller, `DidResume=true`, peer identity +// intact, presenting nothing. The per-connection recheck then re-verified the +// stolen certificate from the resumed state and kept the connection open, which +// is the recheck working correctly on a premise that was already false. +// +// So resumption is off, on the held clone and on every per-connection clone. +// What it costs is one round trip on a reconnect, for a service that re-reads +// its admission set every second anyway; what it buys is that every connection +// this listener serves had a certificate presented on it. The denylist entry +// stays, because a source that sets the field deserves to be told at boot +// rather than silently overridden. +func refuseResumption(config *tls.Config) { + config.SessionTicketsDisabled = true +} + +// unsubvertedPosture refuses the fields that leave every check above passing +// and the posture gone. +// +// It is a denylist over a struct this package does not own, and the claim that +// it was "complete for the fields that exist in the Go version in go.mod" was +// false twice: first for the session-ticket keys, then for +// VerifyPeerCertificate. A Clone() copies 29 of 32 fields unchanged, so what +// this does not name, a source keeps. Where a property matters it is taken +// rather than checked — resumption is disabled outright, and the caller's +// identity is read from a fresh parse of the raw DER — and this list is the +// part that only *tells* a source it has gone wrong. +// +// The checks above enumerate what a conforming listener must *have*, which +// answers the wrong question on its own: a source can satisfy all of them and +// still hand back a configuration whose certificate validity is judged against +// a clock it chose, whose session secrets it writes to a file, whose key +// material comes from a reader it supplies, or whose resumption it encodes and +// decodes itself — and resumption is the sharpest of them, because a resumed +// connection does not re-present a certificate, so a source that controls +// ticket encoding controls who gets in without the rest of this file ever +// seeing a handshake. +// +// Each of these has a safe zero value that means "use Go's own", so refusing a +// non-zero one is exact rather than a guess about intent. That is also the +// limit worth stating: this is a denylist over a struct this package does not +// own, so it is complete for the fields that exist in the Go version in go.mod +// and not by construction. A field added later that lowers posture would pass +// until it is named here, which is why the per-connection answer is checked by +// the same function rather than by a copy of its reasoning. +func unsubvertedPosture(config *tls.Config) error { + for _, field := range []struct { + name, why string + set bool + }{ + {"Time", "certificate expiry and validity are judged against this clock, so a source supplying one decides that an expired certificate is current", config.Time != nil}, + //nolint:staticcheck // Reading a deprecated field is the point: this refuses a source that set it. Naming it keeps the reference compile-checked, which reflection by field name would not — and a security check that stops working silently when a field is renamed is the one outcome worse than the warning. + {"Rand", "the handshake's key material comes from this reader, so a source supplying one decides how guessable this connection's secrets are", config.Rand != nil}, + {"KeyLogWriter", "the session secrets of every connection are written here in the clear, which is a decryption key for the traffic this listener exists to protect", config.KeyLogWriter != nil}, + {"WrapSession", "resumption tickets are encoded by this, and a resumed connection presents no certificate, so a source encoding its own tickets admits callers without a handshake this package sees", config.WrapSession != nil}, + {"UnwrapSession", "resumption tickets are decoded by this, with the same consequence as WrapSession", config.UnwrapSession != nil}, + {"InsecureSkipVerify", "the peer's certificate chain is not verified at all", config.InsecureSkipVerify}, + // VerifyPeerCertificate is handed the parsed chain this runtime then + // reads the caller's identity from, and it runs first. A source that + // rewrites leaf.URIs there had a rival caller admitted — and the + // per-second recheck read the same mutated leaf and agreed with it. + // The identity check now re-parses the raw DER, so the mutation no + // longer decides anything; this refusal is the other half, because a + // source has no business holding a pointer to the chain this + // listener's admission is computed from. + {"VerifyPeerCertificate", "it is handed the parsed certificate chain this listener authenticates the caller by, and it runs before the signature check — so a source can change the public key the handshake is verified against, as well as the identity the admission reads", config.VerifyPeerCertificate != nil}, + // And VerifyConnection, for the half that ordering could not close. + // + // A tls.ConnectionState carries *pointers* to the parsed chain, and + // tls.Conn hands out the same ones on every ConnectionState() call — + // so a source's verifier does not merely see the certificates, it + // holds them, and the per-second rechecks read them minutes later. + // Running this runtime's admission first fixes the admission and + // nothing else: an established caller whose identity was rewritten + // afterwards survives withdrawal, because the recheck re-reads the + // mutated object and agrees with it. + // + // There is no safe way to lend a source these objects, so it is not + // lent them. A consumer that needs its own connection check is a + // follow-up for a hook that passes an immutable copy; the composition + // below is kept for the outbound direction, where the configuration is + // this package's own. + {"VerifyConnection", "it is handed pointers to the certificate chain this listener authenticates its callers by, and the per-connection rechecks read those same objects for as long as the connection lives — so anything written to them there decides who stays admitted", config.VerifyConnection != nil}, + // The ticket *keys*, not just the ticket callbacks. Resumption is the + // sharpest of these because a resumed connection presents no + // certificate, and a source can reach it two ways: by encoding the + // tickets itself (WrapSession/UnwrapSession, above) or by choosing the + // key they are sealed with. A chosen key is the same capability with + // fewer lines — anyone holding it can forge a ticket this listener + // accepts — and the denylist named only the callbacks. + //nolint:staticcheck // As above: deprecated is why it is refused, and the reference stays compile-checked. + {"SessionTicketKey", "resumption tickets are sealed with a key the source chose, and anyone holding it can forge a ticket this listener resumes without any certificate being presented", config.SessionTicketKey != [32]byte{}}, + // SetSessionTicketKeys installs keys that cannot be read back off the + // configuration, so no entry here can catch it. That hole is closed by + // refuseResumption instead, which is why this list is advice to a + // source and not the control. + } { + if field.set { + return fmt.Errorf("the identity source returned a TLS configuration that sets %s: %s. Leave it unset and Go's own is used — this listener's posture is not a thing a source may replace, only a thing it may satisfy", field.name, field.why) + } + } + return nil +} + +// holdPerConnectionPosture holds a source's per-connection callback to the +// posture its base configuration was checked against. +// +// The checks above read the configuration a source returns *at boot*. Go hands +// each handshake to GetConfigForClient when one is set, and the configuration +// that callback returns replaces the base one for that connection — floor, peer +// requirement and certificate included. So without this, a source could conform +// at boot, pass every check in this file, and then answer each individual +// handshake with TLS 1.2, no peer authentication, or a neighbouring workload's +// leaf. +// +// That is not a theoretical shape: the projected source uses that very callback +// to re-read peer trust, so "a source with a per-connection callback" is the +// documented pattern here rather than an odd one — and a consumer following it +// is one line away from lowering the floor for every connection while the boot +// still reports a conforming listener. +// +// It is the same argument as the rest of this cutover, one layer along: +// boot-time approval is not authorization at use. A callback returning nil is +// Go's "serve the base configuration", which was already checked, so it passes +// through untouched; anything else is checked again, and a configuration below +// the posture fails that handshake rather than serving it weakened — the caller +// that happens to arrive is not what is in question, the configuration the +// listener would answer anyone with is. +func holdPerConnectionPosture(config *tls.Config, principal string, allowed func() ([]string, error)) { + // The base configuration's own certificate is already held at use by + // holdServedCertificate, which serverIdentity calls first — a rotation + // landing a certificate for another workload is refused as it is served, + // not as it was at boot. This function used to wrap GetCertificate a second + // time for that, and the duplicate quietly made the real wrapper + // untestable: a mutation that disabled it left the copy here passing every + // certificate-selection test, so the check this file argues for could have + // been broken with nothing to notice. + inner := config.GetConfigForClient + if inner == nil { + return + } + // Whether the base configuration stands on its own, decided here rather + // than at the handshake: a callback is allowed to answer nil, which is + // Go's "serve the base configuration", and the base was only admitted at + // boot *because* a callback would resolve the anchor per handshake. A + // source that answered nil for the hello that actually arrived therefore + // served a configuration requiring a caller's certificate against no + // anchor at all — system roots — while every check in this file had + // passed. A conditional nil answer was demonstrated doing exactly that. + baseStandsAlone := config.ClientCAs != nil + config.GetConfigForClient = func(hello *tls.ClientHelloInfo) (*tls.Config, error) { + answer, err := inner(hello) + if err != nil { + return nil, err + } + if answer == nil { + if !baseStandsAlone { + return nil, fmt.Errorf("the identity source answered this handshake with nothing, which serves its base configuration — and that one names no trust anchor to verify a caller against (ClientCAs is nil), so Go would accept a client certificate from any public CA. A source that resolves the anchor per handshake has to resolve it for every handshake, or carry a usable one on the configuration it returned at boot") + } + return nil, nil + } + if err := usableServerIdentity(answer); err != nil { + return nil, fmt.Errorf("the identity source answered this handshake with a configuration below the posture its boot configuration was held to: %w", err) + } + // On a per-connection answer the pool is not optional: this *is* the + // configuration the handshake runs under, so "resolved per handshake" + // has to mean resolved, not deferred again to system roots. + if answer.ClientCAs == nil { + return nil, fmt.Errorf("the identity source answered this handshake with a configuration that requires a caller's certificate but names no trust anchor (ClientCAs is nil): Go would verify it against this host's system roots, accepting a client certificate from any public CA") + } + // Who may call is re-imposed too: a source's callback that returned a + // configuration without it would serve that connection admitting every + // identity in the cell. + // + // On a clone, never on what the callback handed back. crypto/tls does + // not promise a fresh configuration per call — a source may return one + // it keeps, and the projected source here returns a clone only because + // it happens to — so mutating the answer is a write to a value another + // handshake may be reading, which is both a data race and a violation + // of the rule that a configuration in use is not modified. + admitted := answer.Clone() + refuseResumption(admitted) + // Held to the frozen principal here, on the clone, and not by probing + // the answer: a per-connection answer selects its own certificate, and + // an answer whose GetCertificate returned the approved leaf for a + // synthetic probe and another workload's for the hello that actually + // arrived was demonstrated completing the handshake as the other + // workload. holdServedCertificate wraps that selection instead. + if err := holdServedCertificate(admitted, principal); err != nil { + return nil, fmt.Errorf("the identity source answered this handshake with a configuration whose certificate is not this workload's: %w", err) + } + admitOnly(admitted, allowed) + return admitted, nil + } +} + +// holdServedCertificate rewrites config so that whichever certificate a +// handshake actually selects is this workload's. +// +// The check this replaces read one certificate and approved a configuration: +// it called GetCertificate with a synthetic hello naming "localhost", parsed +// what came back, compared it to the frozen principal, and left the +// configuration alone. Every part of that is a different certificate from the +// one a caller is served, and four separate shapes were demonstrated serving a +// neighbouring workload's leaf through a configuration that had passed: +// +// - a source keyed on SNI answered the synthetic "localhost" probe with the +// approved leaf and the real (no-SNI, IP-addressed) hello with another — +// the probe is a name nothing in a cell dials by; +// - NameToCertificate selected a second certificate by name; +// - the base configuration carried GetCertificate returning the approved leaf +// *and* a static Certificates[0] that was not, and Go consults +// GetCertificate only when there is a certificate list to fall back from or +// an SNI name to key on (crypto/tls, (*Config).getCertificate) — so a +// caller addressing this pod by IP was served the static one, unprobed. The +// SDK's own ServerTLSConfig documents that trap and sets GetCertificate +// alone for exactly this reason; a consumer's source need not; +// - tls.Certificate.Leaf described the approved identity while the DER beside +// it described another. Leaf is a parse cache the holder fills in, not the +// bytes the handshake sends. +// +// So nothing here trusts a probe. The selection surface is narrowed to one +// certificate, a static one is checked as itself, and the callback is wrapped +// so the pair *it returns for this handshake* is what gets compared — the +// certificate is validated at use and the validated pair is what is served. +// +// The principal is an authority-bearing value the platform provisioned and this +// process read once; the leaf is what callers actually see. Nothing else in the +// boot compares them, so a pair projected for another workload — the wrong +// Secret mounted, a Certificate issued for a neighbouring service — would be +// served happily, and the mismatch would surface at whatever verifies the +// destination's identity, as a refusal naming neither file. +func holdServedCertificate(config *tls.Config, principal string) error { + if principal == "" { + // Nothing to hold the leaf to. openAuthority runs before listen on + // every path, so this is a programming error rather than a + // deployment's, and it fails rather than skipping the check. + return fmt.Errorf("this listener has no frozen principal to hold its own certificate to: the authority-bearing values must be read before the listener is built") + } + if err := oneCertificateToChooseFrom(config, "this listener"); err != nil { + return err + } + // A static certificate is reachable whatever else is set, so it is checked + // as itself rather than through a callback that may not be consulted. + if len(config.Certificates) == 1 { + if err := pairPresentsThisWorkload(&config.Certificates[0], principal); err != nil { + return err + } + } + if serve := config.GetCertificate; serve != nil { + config.GetCertificate = func(hello *tls.ClientHelloInfo) (*tls.Certificate, error) { + pair, err := serve(hello) + if err != nil { + return nil, err + } + if pair == nil { + // Go's documented "nothing for this hello, use the list", + // which was checked above or does not exist. + return nil, nil + } + if err := pairPresentsThisWorkload(pair, principal); err != nil { + return nil, err + } + return pair, nil + } + // And asked once now, so a misprovisioned boot is refused at boot + // rather than at every handshake — which also keeps the ordering this + // check exists for: serverIdentity runs before the first outbound act, + // so a pair issued for a neighbouring workload is caught before the + // projected token has been presented to anyone under it. + // + // The hello is empty, which is not a synthetic name but the hello a + // peer addressing this pod by IP actually sends — the previous check + // made one up ("localhost") and a source keyed on it answered the probe + // and the real handshake differently. And it can only *refuse* a boot, + // never approve one: a source that cannot answer an empty hello is left + // to the wrapper above, which is where the guarantee is. + if pair, err := serve(&tls.ClientHelloInfo{}); err == nil && pair != nil { + if err := pairPresentsThisWorkload(pair, principal); err != nil { + return err + } + } + return nil + } + if len(config.Certificates) == 1 { + return nil + } + return fmt.Errorf("the identity source produces its certificate per connection (GetConfigForClient only), so the identity this listener presents cannot be held to this workload's: return a configuration with GetCertificate or Certificates as well") +} + +// oneCertificateToChooseFrom refuses a configuration that selects between +// several certificates. +// +// A workload has one X.509-SVID. A configuration offering a choice is one whose +// served identity is decided at handshake time — by the name a caller sends, by +// which chain happens to be compatible with their signature algorithms — and +// "the one that was checked" is then a property of the caller rather than of +// this configuration. Narrowing the surface is what makes validating at use +// complete instead of validating whichever branch a test happened to take. +func oneCertificateToChooseFrom(config *tls.Config, side string) error { + //nolint:staticcheck // Deprecated and still consulted by crypto/tls whenever more than one certificate is configured, which is exactly the case this refuses. Not reading it would reinstate the bug. + if config.NameToCertificate != nil { + return fmt.Errorf("the identity source selects %s's certificate by server name (NameToCertificate): this workload has exactly one identity, and a configuration that chooses between several serves whichever the caller's name selects, not the one held to the frozen principal", side) + } + if len(config.Certificates) > 1 { + return fmt.Errorf("the identity source offers %s %d certificates to choose from: a workload has one X.509-SVID, and with several Go selects one at handshake time by name and by what the caller supports, so the certificate checked is not the certificate served", + side, len(config.Certificates)) + } + return nil +} + +// pairPresentsThisWorkload holds one certificate — the exact pair a handshake +// selected — to the frozen principal. +func pairPresentsThisWorkload(pair *tls.Certificate, principal string) error { + if pair == nil || len(pair.Certificate) == 0 { + return fmt.Errorf("the identity source produced an empty certificate chain for this handshake") + } + // The DER that will be sent, never pair.Leaf. + // + // Leaf is a parse cache its holder fills in and nothing keeps it honest: a + // pair carrying the approved identity in Leaf and another workload's + // certificate in Certificate[0] was demonstrated passing this check and + // completing a handshake as the other workload. The bytes on the wire are + // the identity; a struct field describing them is a claim. Parsing costs a + // few microseconds per handshake, which is the same trade this file already + // makes to re-read the trust anchor. + leaf, err := x509.ParseCertificate(pair.Certificate[0]) + if err != nil { + return fmt.Errorf("the certificate this handshake would be answered with cannot be parsed: %w", err) + } + return leafPresentsThisWorkload(leaf, principal) +} + +// leafPresentsThisWorkload is the comparison itself. +// +// The comparison is the SPIFFE ID in a URI SAN, which is what names a workload +// in this model (a presence document's expected identity is one). A leaf +// carrying none, or none that matches, is refused. +func leafPresentsThisWorkload(leaf *x509.Certificate, principal string) error { + // Exactly one, for the same reason a caller's is: a leaf naming several + // identities is not a SPIFFE identity, and holding it to "one of them + // matches" means this workload is whichever of them the far end chooses. + if len(leaf.URIs) == 1 && leaf.URIs[0].String() == principal { + return nil + } + if len(leaf.URIs) > 1 { + return fmt.Errorf("the certificate this workload would present names %d URI identities (%s): a SPIFFE certificate names exactly one, and this workload cannot be held to a leaf that claims several", + len(leaf.URIs), strings.Join(uriStrings(leaf.URIs), ", ")) + } + return fmt.Errorf("the certificate this workload would present names %s, and its frozen principal is %q (%s/%s): a pair projected for another workload would otherwise be served, and the mismatch would surface at whatever verifies this destination rather than here", + presentedIdentities(uriStrings(leaf.URIs)), principal, AuthorityGroup, AuthorityPrincipalKey) +} + +// presentedIdentities renders what a leaf does name, so a refusal distinguishes +// "the wrong identity" from "no identity at all". +func presentedIdentities(uris []string) string { + if len(uris) == 0 { + return "no URI identity at all" + } + return "the identities [" + strings.Join(uris, ", ") + "]" +} + +// heldToTheFrozenPrincipal wraps ONE certificate reader so the pair it hands +// back is checked against the frozen principal every time it answers. +// +// This exists as a function rather than only inside holdPresentedCertificate +// because there are now two outbound readers and only one of them was a +// *tls.Config. The mint's transport belongs to the SDK and takes a reader +// directly (MintOptions.ClientCertificate), so the runtime passed +// outboundLeaf — the raw projected-file reloader — straight to it, and the hold +// that the gateway's configuration gets was simply absent on the hop that +// carries the projected service-account token. After a same-CA rotation to a +// neighbouring workload's identity, the mint presented that identity while the +// frozen principal was still this one; the SDK checks that a certificate is +// non-empty and has a key, which is the right division of labour and not a +// check of WHOSE it is. +// +// Both callers run through this, rather than one of them running through a copy +// of its reasoning, because that is how the half-covered check above happened +// in the first place. +func heldToTheFrozenPrincipal( + read func(*tls.CertificateRequestInfo) (*tls.Certificate, error), + principal, destination string, +) func(*tls.CertificateRequestInfo) (*tls.Certificate, error) { + return func(info *tls.CertificateRequestInfo) (*tls.Certificate, error) { + pair, err := read(info) + if err != nil { + return nil, fmt.Errorf("the identity source could not produce the certificate this runtime presents to %s: %w", destination, err) + } + if principal == "" { + return nil, fmt.Errorf("this workload has no frozen principal to hold the certificate it presents to %s against: the authority-bearing values must be read before any outbound act", destination) + } + if err := pairPresentsThisWorkload(pair, principal); err != nil { + return nil, err + } + return pair, nil + } +} + +// holdPresentedCertificate holds an *outbound* configuration's certificate to +// the frozen principal, at the moment it is presented. +// +// The listener's leaf was checked and the client's was not, on the default path +// where they come from two separate reloaders over the same files. So a pair +// that rotated to a neighbouring workload's identity was presented to the mint +// and the gateway by this client while the listener refused to serve it — the +// check existed, ran on one of the two things it had to cover, and the half it +// missed is the half that talks to the party holding the projected token. +// +// Then the check it grew was a *preflight*: it called GetClientCertificate +// once, compared what came back, and left the source's own callback installed +// for the handshake. Rotating the files between the two was demonstrated +// presenting the other workload's leaf successfully — the same check/use gap as +// the inbound side, and the reason this wraps the callback rather than calling +// it. What a dial presents is what gets compared. +func holdPresentedCertificate(config *tls.Config, principal string) error { + if principal == "" { + return fmt.Errorf("this workload's outbound client has no frozen principal to hold its own certificate to: the authority-bearing values must be read before any outbound act") + } + if err := oneCertificateToChooseFrom(config, "this workload's outbound client"); err != nil { + return err + } + if len(config.Certificates) == 1 { + if err := pairPresentsThisWorkload(&config.Certificates[0], principal); err != nil { + return err + } + } + if get := config.GetClientCertificate; get != nil { + config.GetClientCertificate = heldToTheFrozenPrincipal(get, principal, "the platform") + // Refused at boot as well, for the reason holdServedCertificate + // records: a mismatch found at the first dial is a mismatch found after + // the projected token has gone out under it. + if pair, err := get(&tls.CertificateRequestInfo{}); err == nil && pair != nil { + if err := pairPresentsThisWorkload(pair, principal); err != nil { + return err + } + } + return nil + } + if len(config.Certificates) == 1 { + return nil + } + return fmt.Errorf("this workload's outbound client would present no certificate, so the platform cannot tell this workload from anything else that reached it") +} diff --git a/identity_test.go b/identity_test.go new file mode 100644 index 0000000..7c1ce25 --- /dev/null +++ b/identity_test.go @@ -0,0 +1,1227 @@ +package solution + +import ( + "context" + "crypto/ed25519" + "crypto/rand" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "fmt" + "github.com/codefly-dev/sdk-go/workcontext" + "io" + "math/big" + "net" + "net/http" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +// TestABootWithoutAWorkloadIdentityIsRefusedByName is the acceptance criterion +// for removing plain HTTP: the runtime refuses a listener configuration it +// cannot present an identity for, and names the material that is missing. A +// solution that came up on plain HTTP would be refused at the edge instead, +// for a reason only the edge can see. +func TestABootWithoutAWorkloadIdentityIsRefusedByName(t *testing.T) { + certFile, keyFile, bundleFile, _, _ := workloadIdentity(t, testPrincipal) + for _, tc := range []struct { + name string + cert string + key string + token string + bundle string + callers string + peers string + names string + variable string + }{ + {name: "no certificate", peers: testGatewayPrincipal, key: keyFile, token: "t", bundle: bundleFile, callers: testGatewayPrincipal, names: "certificate", variable: IdentityCertFileEnvironmentVariable}, + {name: "no private key", peers: testGatewayPrincipal, cert: certFile, token: "t", bundle: bundleFile, callers: testGatewayPrincipal, names: "private key", variable: IdentityKeyFileEnvironmentVariable}, + {name: "no trust anchor", peers: testGatewayPrincipal, cert: certFile, key: keyFile, token: "t", callers: testGatewayPrincipal, names: "trust anchor", variable: IdentityTrustBundleFileEnvironmentVariable}, + {name: "no projected token", peers: testGatewayPrincipal, cert: certFile, key: keyFile, bundle: bundleFile, callers: testGatewayPrincipal, names: "projected service-account token", variable: ProjectedTokenFileEnvironmentVariable}, + // Not material this workload presents, but the other half of the + // posture: a listener that verifies every caller in the cell and + // admits all of them is a listener with no admission at all, so the + // admitted set is provisioned and named when it is absent. + {name: "no allowed callers", cert: certFile, key: keyFile, token: "t", bundle: bundleFile, peers: testGatewayPrincipal, names: "allowed caller identities", variable: IdentityAllowedCallersFileEnvironmentVariable}, + // And the same question outbound. Every workload in the cell holds a + // certificate from the same anchor, so a chain and a hostname do not + // say a destination is the platform — and outbound is the direction + // where being wrong hands over the projected token rather than + // accepting a call. + {name: "no platform peers", cert: certFile, key: keyFile, token: "t", bundle: bundleFile, callers: testGatewayPrincipal, names: "credential mint peer identity", variable: IdentityMintPeersFileEnvironmentVariable}, + } { + t.Run(tc.name, func(t *testing.T) { + server := New(Manifest{ID: testSolutionID}) + server.cfg = config{ + port: "8080", + gatewayURL: "https://gateway:42152", + mintURL: "https://gateway:42152" + credentialMintPath, + identityCertFile: tc.cert, + identityKeyFile: tc.key, + projectedTokenPath: tc.token, + trustBundleFile: tc.bundle, + allowedCallersFile: optionalIdentitiesFile(t, tc.callers), + mintPeersFile: optionalIdentitiesFile(t, tc.peers), + gatewayPeersFile: optionalIdentitiesFile(t, tc.peers), + profile: localProfile, + } + if err := server.cfg.validate(); err != nil { + t.Fatalf("the resolved configuration itself is unusable: %v", err) + } + err := server.validateSources() + if err == nil { + t.Fatal("the boot accepted a configuration with no workload identity") + } + if !strings.Contains(err.Error(), tc.names) { + t.Errorf("refusal %q does not name the missing %s", err, tc.names) + } + if !strings.Contains(err.Error(), tc.variable) { + t.Errorf("refusal %q does not name %s, the override that sets it", err, tc.variable) + } + if !strings.Contains(err.Error(), WorkloadIdentityGroup) { + t.Errorf("refusal %q does not name the %q group the platform provisions", err, WorkloadIdentityGroup) + } + }) + } +} + +// TestACredentialBearingDestinationMustBeHTTPS: a TLS listener protects what +// callers send to this process and nothing it sends out. Over http:// the mint +// hands this workload's projected token to anything on the path, and a viewer's +// mint hands over their bearer and the capability minted for them. +func TestACredentialBearingDestinationMustBeHTTPS(t *testing.T) { + for _, tc := range []struct { + name string + gateway string + mint string + names string + variable string + }{ + {"a plaintext gateway", "http://gateway:42152", "https://gateway:42152" + credentialMintPath, "viewer's bearer", "GATEWAY_URL"}, + {"a plaintext mint", "https://gateway:42152", "http://gateway:42152" + credentialMintPath, "service-account token", CredentialMintURLEnvironmentVariable}, + } { + t.Run(tc.name, func(t *testing.T) { + cfg := config{port: "8080", gatewayURL: tc.gateway, mintURL: tc.mint, profile: localProfile} + err := cfg.validate() + if err == nil { + t.Fatal("validate accepted a plaintext credential-bearing destination") + } + if !strings.Contains(err.Error(), tc.names) { + t.Errorf("refusal %q does not say what a plaintext hop would disclose (%s)", err, tc.names) + } + if !strings.Contains(err.Error(), tc.variable) { + t.Errorf("refusal %q does not name %s", err, tc.variable) + } + }) + } +} + +// TestASuppliedSourceNeedsNoProvisioningItDoesNotRead: a consumer that supplied +// an identity source has said where its identity comes from, so requiring the +// projected file paths as well would be requiring provisioning nothing reads. +// What no source changes is the authority identity. +func TestASuppliedSourceNeedsNoProvisioningItDoesNotRead(t *testing.T) { + base := config{ + port: "8080", + gatewayURL: "https://gateway:42152", + mintURL: "https://gateway:42152" + credentialMintPath, + profile: localProfile, + // Required whatever the identity source is: the source says who this + // workload is, this says which callers it serves — and which + // destinations are the platform. + allowedCallersFile: identitiesFile(t, testGatewayPrincipal), + mintPeersFile: identitiesFile(t, testGatewayPrincipal), + gatewayPeersFile: identitiesFile(t, testGatewayPrincipal), + } + t.Run("an identity source replaces the projected pair", func(t *testing.T) { + server := New(Manifest{ID: testSolutionID}).Identity(staticIdentity{}) + server.cfg = base + server.cfg.projectedTokenPath = "/var/run/token" + if err := server.validateSources(); err != nil { + t.Errorf("the boot required material the selected source does not read: %v", err) + } + }) + t.Run("a credential source replaces the projected token", func(t *testing.T) { + certFile, keyFile, bundleFile, _, _ := workloadIdentity(t, testPrincipal) + server := New(Manifest{ID: testSolutionID}).Credential(stubCredentialSource{}) + server.cfg = base + server.cfg.identityCertFile, server.cfg.identityKeyFile, server.cfg.trustBundleFile = certFile, keyFile, bundleFile + if err := server.validateSources(); err != nil { + t.Errorf("the boot required material the selected source does not read: %v", err) + } + }) +} + +// TestAnIdentitySourceReachesTheSamePostureAsTheProjectedOne: a source is +// consumer code — a mesh CA, a SPIFFE workload API, a pair the local toolchain +// minted — and every one of them has to arrive at the same posture. Checking +// only that a certificate was returned left the TLS floor and peer +// authentication silently optional for exactly the callers who wrote their own +// source. +func TestAnIdentitySourceReachesTheSamePostureAsTheProjectedOne(t *testing.T) { + certFile, keyFile, bundleFile, _, _ := workloadIdentity(t, testPrincipal) + usable, err := projectedIdentity{certFile: certFile, keyFile: keyFile, trustBundleFile: bundleFile}.ServerTLSConfig() + if err != nil { + t.Fatal(err) + } + for _, tc := range []struct { + name string + config *tls.Config + says string + }{ + {"no configuration at all", nil, "plain-HTTP"}, + {"no certificate", &tls.Config{MinVersion: tls.VersionTLS13, ClientAuth: tls.RequireAndVerifyClientCert}, "no certificate"}, + {"a floor below TLS 1.3", withFloor(usable, tls.VersionTLS12), "TLS 1.3"}, + {"no peer authentication", withClientAuth(usable, tls.NoClientCert), "authenticates no peer"}, + {"a peer it verifies but does not require", withClientAuth(usable, tls.VerifyClientCertIfGiven), "authenticates no peer"}, + // Requiring a certificate and naming no anchor is strictly worse than + // requiring none, because it reads as mutual TLS in every log line: + // with ClientCAs nil, Go verifies the caller against this host's + // system roots, so the listener demands a certificate and then accepts + // one from any public CA. + {"a required certificate with no anchor to verify it against", withoutAnchor(usable), "system roots"}, + } { + t.Run(tc.name, func(t *testing.T) { + server := New(Manifest{ID: testSolutionID}).Identity(staticIdentity{config: tc.config}) + server.cfg = config{port: listenOn(t, server)} + server.principal = testPrincipal + ln, err := server.listen() + if ln != nil { + _ = ln.Close() + } + if err == nil { + t.Fatal("listen came up on a configuration below this model's posture") + } + if !strings.Contains(err.Error(), tc.says) { + t.Errorf("refusal %q does not say %q", err, tc.says) + } + }) + } + + t.Run("a source that reaches the posture is accepted", func(t *testing.T) { + server := New(Manifest{ID: testSolutionID}).Identity(staticIdentity{config: usable}) + server.cfg = config{port: listenOn(t, server)} + server.principal = testPrincipal + ln, err := server.listen() + if err != nil { + t.Fatalf("listen refused a conforming source: %v", err) + } + _ = ln.Close() + }) +} + +// TestAListenerIsHeldToItsOwnFrozenPrincipal: the pair a platform projects and +// the principal it provisioned are two facts nothing else in the boot compares. +// A Secret mounted for a neighbouring workload would otherwise be served +// happily, and the mismatch would surface at whatever verifies this +// destination, as a refusal naming neither file. +func TestAListenerIsHeldToItsOwnFrozenPrincipal(t *testing.T) { + certFile, keyFile, bundleFile, _, _ := workloadIdentity(t, "spiffe://codefly.test/ns/solutions/sa/another-workload") + server := New(Manifest{ID: testSolutionID}) + server.cfg = config{port: listenOn(t, server), identityCertFile: certFile, identityKeyFile: keyFile, trustBundleFile: bundleFile} + server.principal = testPrincipal + ln, err := server.listen() + if ln != nil { + _ = ln.Close() + } + if err == nil { + t.Fatal("listen served a certificate issued for another workload") + } + for _, want := range []string{"another-workload", testPrincipal, AuthorityGroup} { + if !strings.Contains(err.Error(), want) { + t.Errorf("refusal %q does not name %q", err, want) + } + } + + t.Run("a leaf with no URI identity at all", func(t *testing.T) { + config, err := projectedIdentity{certFile: certFile, keyFile: keyFile, trustBundleFile: bundleFile}.ServerTLSConfig() + if err != nil { + t.Fatal(err) + } + stripped := config.Clone() + stripped.GetCertificate = nil + stripped.GetConfigForClient = nil + stripped.Certificates = []tls.Certificate{{Certificate: [][]byte{selfSignedWithoutURI(t)}}} + if err := holdServedCertificate(stripped, testPrincipal); err == nil || !strings.Contains(err.Error(), "no URI identity at all") { + t.Errorf("holdServedCertificate = %v, want a refusal distinguishing no identity from the wrong one", err) + } + }) +} + +// TestPeerTrustIsReadAtEveryHandshake: a pool read once cannot shrink. Removing +// a compromised root from the projected bundle has to stop it authenticating +// callers without a restart, because revocation that takes effect only on a +// restart is revocation an operator has to remember to finish. +func TestPeerTrustIsReadAtEveryHandshake(t *testing.T) { + certFile, keyFile, bundleFile, _, _ := workloadIdentity(t, testPrincipal) + config, err := projectedIdentity{certFile: certFile, keyFile: keyFile, trustBundleFile: bundleFile}.ServerTLSConfig() + if err != nil { + t.Fatal(err) + } + first, err := config.GetConfigForClient(&tls.ClientHelloInfo{}) + if err != nil { + t.Fatalf("first handshake: %v", err) + } + if first.ClientAuth != tls.RequireAndVerifyClientCert { + t.Errorf("ClientAuth = %v, want RequireAndVerifyClientCert", first.ClientAuth) + } + if first.ClientCAs == nil { + t.Fatal("the handshake was offered no peer anchor") + } + + // The anchor is replaced with a different one: a caller issued under the + // old root must no longer authenticate, and the pool the next handshake + // uses must be the new one. + replacement, _, _ := issueAnchor(t) + writeFile(t, bundleFile, string(replacement)) + second, err := config.GetConfigForClient(&tls.ClientHelloInfo{}) + if err != nil { + t.Fatalf("handshake after the anchor was replaced: %v", err) + } + if second.ClientCAs.Equal(first.ClientCAs) { + t.Error("the handshake still judges callers by the anchor that was removed: peer trust was snapshotted at boot") + } + + // An anchor that has become unreadable fails the handshake rather than + // falling back to the last good pool: judging by a stale anchor lets in + // callers who should be refused, which is not symmetric with serving a + // stale leaf. + if err := os.Remove(bundleFile); err != nil { + t.Fatal(err) + } + if _, err := config.GetConfigForClient(&tls.ClientHelloInfo{}); err == nil { + t.Error("a handshake was answered after the trust anchor became unreadable") + } +} + +// TestAnUnusableTrustAnchorIsRefusedNotDropped: dropped, the listener comes up +// verifying no peer at all — exactly what configuring an anchor was meant to +// prevent — and nothing says so. +func TestAnUnusableTrustAnchorIsRefusedNotDropped(t *testing.T) { + certFile, keyFile, _, _, _ := workloadIdentity(t, testPrincipal) + bundle := filepath.Join(t.TempDir(), "ca.crt") + writeFile(t, bundle, "not a certificate") + _, err := projectedIdentity{certFile: certFile, keyFile: keyFile, trustBundleFile: bundle}.ServerTLSConfig() + if err == nil || !strings.Contains(err.Error(), "no certificate") { + t.Errorf("an unparseable anchor was reported as %v, want a refusal saying it carries no certificate", err) + } + + _, err = projectedIdentity{certFile: certFile, keyFile: keyFile}.ServerTLSConfig() + if err == nil || !strings.Contains(err.Error(), IdentityTrustBundleFileEnvironmentVariable) { + t.Errorf("an absent anchor was reported as %v, want a refusal naming %s", err, IdentityTrustBundleFileEnvironmentVariable) + } +} + +// TestTheListenerServesTheRotatedLeaf: workload leaves are short-lived and +// rotated well before expiry, so a listener that loaded its pair once at boot +// serves a stale leaf while a valid one sits on disk, and then fails every +// handshake with the issuer reporting nothing wrong. +func TestTheListenerServesTheRotatedLeaf(t *testing.T) { + certFile, keyFile, bundleFile, _, _ := workloadIdentity(t, testPrincipal) + config, err := projectedIdentity{certFile: certFile, keyFile: keyFile, trustBundleFile: bundleFile}.ServerTLSConfig() + if err != nil { + t.Fatal(err) + } + first, err := config.GetCertificate(&tls.ClientHelloInfo{ServerName: "localhost"}) + if err != nil { + t.Fatal(err) + } + rotatedCert, rotatedKey, _, _, _ := workloadIdentity(t, testPrincipal) + writeFile(t, certFile, readFile(t, rotatedCert)) + writeFile(t, keyFile, readFile(t, rotatedKey)) + second, err := config.GetCertificate(&tls.ClientHelloInfo{ServerName: "localhost"}) + if err != nil { + t.Fatal(err) + } + if string(first.Certificate[0]) == string(second.Certificate[0]) { + t.Error("the listener served the boot-time leaf after the projection rotated") + } +} + +// staticIdentity is a consumer-supplied identity source, including the +// degenerate ones a consumer can write by mistake. +type staticIdentity struct { + config *tls.Config + // anchor, when set, is what this source says its current anchor is. Unset, + // it answers with the pool on the configuration — truthful for a source + // whose anchor really is static, which is every source in these tests. + anchor func() *x509.CertPool +} + +func (s staticIdentity) ServerTLSConfig() (*tls.Config, error) { return s.config, nil } + +// PeerAnchor makes this source askable, which a source setting +// GetConfigForClient now has to be: the runtime re-verifies established peers +// and verifies the platform it dials, and neither can be answered by probing a +// per-handshake callback with a hello nobody sent. +func (s staticIdentity) PeerAnchor() (*x509.CertPool, error) { + if s.anchor != nil { + return s.anchor(), nil + } + if s.config.ClientCAs != nil { + return s.config.ClientCAs, nil + } + // Mirrors what the projected source does: resolve it now rather than hold + // a pool from boot. + if s.config.GetConfigForClient != nil { + answer, err := s.config.GetConfigForClient(&tls.ClientHelloInfo{}) + if err == nil && answer != nil && answer.ClientCAs != nil { + return answer.ClientCAs, nil + } + } + return nil, fmt.Errorf("this test source names no anchor") +} + +// withoutAnchor is a source that requires a caller's certificate and names +// nothing to verify it against, with no per-connection callback to resolve one +// later either. +func withoutAnchor(base *tls.Config) *tls.Config { + changed := base.Clone() + changed.ClientCAs = nil + changed.GetConfigForClient = nil + return changed +} + +func withFloor(base *tls.Config, floor uint16) *tls.Config { + changed := base.Clone() + changed.MinVersion = floor + return changed +} + +func withClientAuth(base *tls.Config, auth tls.ClientAuthType) *tls.Config { + changed := base.Clone() + changed.ClientAuth = auth + return changed +} + +// selfSignedWithoutURI is a leaf whose identity is a DNS name and nothing else: +// the shape a pair projected by something that does not know about SPIFFE has. +func selfSignedWithoutURI(t *testing.T) []byte { + t.Helper() + _, anchor, anchorKey := issueAnchor(t) + public, _, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + t.Fatal(err) + } + der, err := x509.CreateCertificate(rand.Reader, &x509.Certificate{ + SerialNumber: big.NewInt(99), + Subject: pkix.Name{CommonName: "no-uri"}, + NotBefore: time.Now().Add(-time.Minute), + NotAfter: time.Now().Add(time.Hour), + KeyUsage: x509.KeyUsageDigitalSignature, + DNSNames: []string{"localhost"}, + }, anchor, public, anchorKey) + if err != nil { + t.Fatal(err) + } + return der +} + +func readFile(t *testing.T, path string) string { + t.Helper() + raw, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + return string(raw) +} + +// TestASourcesPerConnectionCallbackCannotLowerThePosture closes the last way +// into this file, and it is the same argument the rest of the cutover is built +// on: boot-time approval is not authorization at use. +// +// usableServerIdentity and holdServedCertificate read the configuration a source +// returns at boot. Go hands each handshake to GetConfigForClient when one is +// set, and the configuration that callback returns replaces the base one for +// that connection. So a source could conform at boot, pass every other check +// here, and serve each individual handshake under TLS 1.2, with no peer +// authentication, or holding a neighbouring workload's leaf. +// +// The projected source uses that callback to re-read peer trust, so this is the +// documented pattern in this package, not an odd one — a consumer following it +// is one line from lowering the floor on every connection while the boot still +// reports a conforming listener. +// +// A configuration below the posture fails the handshake outright rather than +// serving it in a weakened form: the caller that happens to arrive is not the +// thing in question, the configuration the listener would answer anyone with +// is. +func TestASourcesPerConnectionCallbackCannotLowerThePosture(t *testing.T) { + // One anchor for the whole test, so a rival leaf is one a caller would + // actually verify — issued by the same CA, naming another workload. + c := newCell(t) + certFile, keyFile, bundleFile, caller, roots := c.workload(t, testPrincipal) + rival := c.identity(t, "spiffe://codefly.test/ns/solutions/sa/another-workload") + conforming, err := projectedIdentity{certFile: certFile, keyFile: keyFile, trustBundleFile: bundleFile}.ServerTLSConfig() + if err != nil { + t.Fatal(err) + } + authenticated := func() *tls.Config { + return &tls.Config{RootCAs: roots, Certificates: []tls.Certificate{*caller}, MinVersion: tls.VersionTLS13, ServerName: "localhost"} + } + + // A source conforming at boot — certificate, TLS 1.3, + // RequireAndVerifyClientCert — that downgrades per connection. + downgrading := func(change func(*tls.Config)) *tls.Config { + base := conforming.Clone() + base.GetConfigForClient = func(*tls.ClientHelloInfo) (*tls.Config, error) { + answer := conforming.Clone() + answer.GetConfigForClient = nil + anchor, err := peerAnchor(bundleFile) + if err != nil { + return nil, err + } + answer.ClientCAs = anchor + answer.ClientAuth = tls.RequireAndVerifyClientCert + change(answer) + return answer, nil + } + return base + } + + for _, tc := range []struct { + name string + // what the callback does to each connection's configuration + change func(*tls.Config) + // a caller the lowered posture would have admitted, and that must not + // be, alongside the authenticated one which is refused with it + exploits func() *tls.Config + }{ + { + "a floor below TLS 1.3 per connection", + func(cfg *tls.Config) { cfg.MinVersion = tls.VersionTLS12 }, + func() *tls.Config { + return &tls.Config{RootCAs: roots, Certificates: []tls.Certificate{*caller}, MinVersion: tls.VersionTLS12, MaxVersion: tls.VersionTLS12, ServerName: "localhost"} + }, + }, + { + "no peer authentication per connection", + func(cfg *tls.Config) { cfg.ClientAuth = tls.NoClientCert }, + func() *tls.Config { + return &tls.Config{RootCAs: roots, MinVersion: tls.VersionTLS13, ServerName: "localhost"} + }, + }, + { + "a peer verified but not required per connection", + func(cfg *tls.Config) { cfg.ClientAuth = tls.VerifyClientCertIfGiven }, + func() *tls.Config { + return &tls.Config{RootCAs: roots, MinVersion: tls.VersionTLS13, ServerName: "localhost"} + }, + }, + { + // Resolving the anchor per handshake has to mean resolved: a + // callback that answers with no pool defers to the system roots + // for that connection, which is the same hole one layer along. + "no trust anchor per connection", + func(cfg *tls.Config) { cfg.ClientCAs = nil }, + func() *tls.Config { + return &tls.Config{RootCAs: roots, MinVersion: tls.VersionTLS13, ServerName: "localhost"} + }, + }, + { + // And the same with the answer carrying a further callback. Go + // calls GetConfigForClient once, on the base configuration, so a + // callback *on the answer* is never called and cannot be where the + // pool arrives — the connection runs on this configuration, with + // system roots. It is the one shape the boot-time check reads as + // "resolved later". + "no trust anchor per connection, deferred to a callback Go never calls", + func(cfg *tls.Config) { + cfg.ClientCAs = nil + cfg.GetConfigForClient = func(*tls.ClientHelloInfo) (*tls.Config, error) { return nil, nil } + }, + func() *tls.Config { + return &tls.Config{RootCAs: roots, MinVersion: tls.VersionTLS13, ServerName: "localhost"} + }, + }, + { + // Nothing extra is admitted here; what changes is whose identity + // the listener presents, so the authenticated caller is the one + // that reads the defect. + "another workload's leaf per connection", + func(cfg *tls.Config) { cfg.Certificates, cfg.GetCertificate = []tls.Certificate{*rival}, nil }, + nil, + }, + } { + t.Run(tc.name, func(t *testing.T) { + server := New(Manifest{ID: testSolutionID}).Identity(staticIdentity{config: downgrading(tc.change)}) + server.cfg = config{port: listenOn(t, server), allowedCallersFile: identitiesFile(t, testGatewayPrincipal)} + server.principal = testPrincipal + // The two failures listen() returns as one are separated here. + // Refusing the *identity* at boot is an acceptable answer — what + // must not happen is a handshake served under the lowered posture — + // but failing to *bind* is this test not running, and returning on + // it made every subtest below pass for a reason that has nothing to + // do with the posture. + config, refused := server.serverIdentity() + if refused != nil { + // A boot refusal is an acceptable answer, but only the right + // one. Returning on *any* error made every subtest below pass + // the moment serverIdentity failed for an unrelated reason — + // an unresolved path, a missing admission file, a mutated + // helper — which is the shape that makes a negative test + // report success for a cause it never exercised. + if !strings.Contains(refused.Error(), "identity source") { + t.Fatalf("the boot refused the identity for something other than its posture, so this case never ran: %v", refused) + } + return + } + // The listener this server was handed, not a second bind of the + // same port. The distinction the comment above cares about — a + // boot that refuses the identity versus a test that could not + // bind — is kept and sharpened: the bind happened in boundPort, + // which fails the test outright, so reaching here means a + // listener exists and only the posture is in question. + ln := tls.NewListener(server.boundListener, config) + defer func() { _ = ln.Close() }() + serve(t, ln) + + if tc.exploits != nil { + if conn, err := tls.Dial("tcp", ln.Addr().String(), tc.exploits()); err == nil { + state := conn.ConnectionState() + _ = conn.Close() + t.Errorf("a caller the boot configuration would have refused was served over TLS 0x%04x with %d peer certificate(s): the per-connection configuration is not held to the posture the boot checked", + state.Version, len(state.PeerCertificates)) + } + } + // And the lowered configuration is refused for everyone, not + // weakened for whoever asks: the configuration is what is wrong. + conn, err := tls.Dial("tcp", ln.Addr().String(), authenticated()) + if err == nil { + state := conn.ConnectionState() + served := "none" + if len(state.PeerCertificates) > 0 && len(state.PeerCertificates[0].URIs) > 0 { + served = state.PeerCertificates[0].URIs[0].String() + } + _ = conn.Close() + t.Errorf("the listener answered a handshake under a configuration below its boot posture (TLS 0x%04x, presenting %s)", state.Version, served) + } + }) + } + + // The other direction, or the check above is just a broken listener: a + // per-connection callback that keeps the posture is served normally. This + // is the projected source's own shape — it uses the callback to re-read + // peer trust on every handshake. + t.Run("a conforming callback is served", func(t *testing.T) { + server := New(Manifest{ID: testSolutionID}).Identity(staticIdentity{config: conforming}) + // The admitted set is provisioned, because listen() is the production + // path and a deployment reaching it has been through validateSources. + // Without it this subtest dialled a listener that refused the caller + // and read as a success anyway: in TLS 1.3 the client finishes before + // it learns whether the server accepted its certificate, so a + // server-side refusal is invisible to tls.Dial. + server.cfg = config{port: listenOn(t, server), allowedCallersFile: identitiesFile(t, testGatewayPrincipal)} + server.principal = testPrincipal + ln, err := server.listen() + if err != nil { + t.Fatalf("listen refused the projected source's own per-connection shape: %v", err) + } + // This one may fail for a bind, and says so rather than skipping. + defer func() { _ = ln.Close() }() + serve(t, ln) + + good, err := tls.Dial("tcp", ln.Addr().String(), authenticated()) + if err != nil { + t.Fatalf("a conforming caller was refused, so the per-connection check broke the listener: %v", err) + } + state := good.ConnectionState() + _ = good.Close() + if state.Version < tls.VersionTLS13 { + t.Errorf("the connection was served TLS 0x%04x, want a 1.3 floor", state.Version) + } + leaf := state.PeerCertificates[0] + if len(leaf.URIs) == 0 || leaf.URIs[0].String() != testPrincipal { + t.Errorf("the connection was served %v, want this workload's frozen principal %q", leaf.URIs, testPrincipal) + } + }) +} + +// serve accepts and handshakes until the listener closes, so a refused +// handshake is refused by the listener's own configuration rather than by +// nobody being there. +func serve(t *testing.T, ln net.Listener) { + t.Helper() + done := make(chan struct{}) + go func() { + defer close(done) + for { + conn, err := ln.Accept() + if err != nil { + return + } + go func() { + if tlsConn, ok := conn.(*tls.Conn); ok { + _ = tlsConn.Handshake() + } + _ = conn.Close() + }() + } + }() + t.Cleanup(func() { <-done }) +} + +// suppliedIdentity is a consumer's IdentitySource that happens to produce a +// conforming configuration — the shape the README's example implies, built over +// the same anchor the fake host is served under so the boot fails on the +// property under test and not on verifying the host. +type suppliedIdentity struct{ certFile, keyFile, bundleFile string } + +func (c suppliedIdentity) inner() projectedIdentity { + return projectedIdentity{certFile: c.certFile, keyFile: c.keyFile, trustBundleFile: c.bundleFile} +} + +func (c suppliedIdentity) ServerTLSConfig() (*tls.Config, error) { + return c.inner().ServerTLSConfig() +} + +// PeerAnchor is delegated too, and a consumer wrapping a source has to do the +// same: this source resolves its anchor per handshake, which is the right +// shape, and that is exactly the shape that has to be able to answer what the +// anchor currently is rather than be probed with a handshake nobody made. +func (c suppliedIdentity) PeerAnchor() (*x509.CertPool, error) { + return c.inner().PeerAnchor() +} + +// TestASuppliedSourceBootsAndStaysAuthenticatedOutbound drives the two bugs a +// second reviewer found behind the "custom sources work through the boot +// contract" answer. The test that answered it before stopped at +// validateSources, so neither was reachable by it. +// +// The outbound client was built `if s.credential == nil || s.identity == nil`: +// +// - supply **both** sources and it was never built, so s.outbound stayed nil +// and every gateway call fell through to the unauthenticated transport — +// system roots, no client certificate. The one posture this cutover exists +// to remove, reached by configuring more rather than less. +// - supply **only Identity** and the condition was true, so the client read +// the projected files validateSources had just declared unnecessary for +// that boot. The README's own example could not start. +func TestASuppliedSourceBootsAndStaysAuthenticatedOutbound(t *testing.T) { + // No projected identity files at all, so a boot that reads them fails and a + // boot that honours the supplied source does not. + withoutProjectedIdentity := func(t *testing.T) { + t.Helper() + t.Setenv(IdentityCertFileEnvironmentVariable, "") + t.Setenv(IdentityKeyFileEnvironmentVariable, "") + t.Setenv(IdentityTrustBundleFileEnvironmentVariable, "") + } + + t.Run("identity only, the README's own example", func(t *testing.T) { + mint := newHostMint(t, &hostMint{}) + certFile, keyFile, bundleFile, _, _ := bootIdentity(t, mint, testPrincipal) + withoutProjectedIdentity(t) + + server := New(Manifest{ID: testSolutionID}). + Identity(suppliedIdentity{certFile: certFile, keyFile: keyFile, bundleFile: bundleFile}) + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + ln, err := takeListener(server).start(ctx) + if err != nil { + t.Fatalf("a boot supplying only Identity(...) was refused: %v\nthe supplied source says where the identity comes from, so the projected files are provisioning this boot does not read", err) + } + defer func() { _ = ln.Close() }() + if server.outbound == nil { + t.Fatal("the boot came up with no outbound client at all") + } + if got := mint.count(); got != 1 { + t.Errorf("the host was asked %d times, want the one mint: the supplied-identity boot must still obtain its credential", got) + } + }) + + t.Run("both sources supplied: outbound still presents this workload", func(t *testing.T) { + mint := newHostMint(t, &hostMint{}) + certFile, keyFile, bundleFile, caller, roots := bootIdentity(t, mint, testPrincipal) + tokenFile := filepath.Join(t.TempDir(), "token") + writeFile(t, tokenFile, "projected-token") + withoutProjectedIdentity(t) + + // The fake host is served over TLS under the cell's anchor, and a + // supplied credential source brings its own transport — the runtime + // hands it none, which is the point of supplying one. + // + // It no longer brings a transport: the SDK's mint client builds and + // owns that, and takes only the roots. A consumer supplying a + // credential source therefore supplies the anchor the mint endpoint is + // verified against and nothing else about how it is dialled. + _, _ = caller, roots + server := New(Manifest{ID: testSolutionID}). + Identity(suppliedIdentity{certFile: certFile, keyFile: keyFile, bundleFile: bundleFile}). + Credential(mintClientFor(t, mint, tokenFile)) + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + ln, err := takeListener(server).start(ctx) + if err != nil { + t.Fatalf("a boot supplying both sources was refused: %v", err) + } + defer func() { _ = ln.Close() }() + + if server.outbound == nil { + t.Fatal("s.outbound is nil with both sources supplied, so every gateway call falls through to the unauthenticated transport: system roots, no client certificate") + } + transport, ok := server.outbound.Transport.(*http.Transport) + if !ok { + t.Fatalf("outbound transport is %T, want *http.Transport", server.outbound.Transport) + } + if transport.DialTLSContext == nil { + t.Fatal("the outbound client has no per-connection dialler, so its trust is snapshotted") + } + // What that dialler builds, checked directly: identity and anchor from + // the supplied source, floor at 1.3. + built, err := clientTLSFrom(server.identity, server.identityConfig) + if err != nil { + t.Fatalf("derive the outbound configuration from the supplied source: %v", err) + } + switch { + case built.RootCAs == nil: + t.Error("the outbound client verifies the platform against system roots, not the anchor the supplied source named") + case built.GetClientCertificate == nil && len(built.Certificates) == 0: + t.Error("the outbound client presents no identity, so the platform cannot tell this workload from anything else that reached it") + case built.MinVersion != tls.VersionTLS13: + t.Errorf("the outbound floor is 0x%04x, want TLS 1.3", built.MinVersion) + } + // And the gateway a handler is handed carries it, rather than the + // unauthenticated fallback. + if server.gatewayFor(http.Header{}).transport == nil { + t.Error("a handler's gateway carries no transport, so it dials the platform unauthenticated") + } + }) +} + +// TestAMismatchedLeafIsRefusedBeforeTheMint is the ordering half of the same +// finding: the principal check is worth nothing after the fact. +// +// openCredential used to run before listen(), so a leaf issued for a +// neighbouring workload — the wrong Secret mounted, a Certificate issued for +// another service — had already been presented to the host, together with this +// workload's projected service-account token, by the time the listener refused +// it. The refusal was real and the disclosure had already happened. +func TestAMismatchedLeafIsRefusedBeforeTheMint(t *testing.T) { + mint := newHostMint(t, &hostMint{}) + bootIdentity(t, mint, "spiffe://codefly.test/ns/solutions/sa/another-workload") + + // Not bootFails: that helper re-provisions the environment, which would + // replace the mismatched leaf with a matching one and assert nothing. + ln, err := takeListener(New(Manifest{ID: testSolutionID})).start(context.Background()) + if ln != nil { + _ = ln.Close() + } + if err == nil { + t.Fatal("the boot came up serving a leaf issued for another workload") + } + if !strings.Contains(err.Error(), "another-workload") { + t.Errorf("the boot was refused with %v, want a refusal naming the identity actually served", err) + } + if got := mint.count(); got != 0 { + t.Errorf("the host was asked %d time(s) before the leaf was checked: a mismatched identity must reach nothing, and the projected token travels on that request", got) + } +} + +// TestAnAuthenticatedCallerIsNotAutomaticallyAnAuthorisedOne is the confused +// deputy a second reviewer found behind "authenticated peers, both directions". +// +// Verifying a caller's certificate against the cell's anchor answers whether it +// holds an identity the platform issued. In a cell that is every workload, +// including the modules this solution consumes. So any of them could reach a +// handler or a passthrough route directly, bypassing the admission the host +// decides on its own routes — and since gatewayFor reads x-org-id and +// x-session-id as headers the gateway stamped from a verified bearer, such a +// caller could set them itself and have this runtime mint capabilities under +// *this workload's* attestation for an org and session nobody authenticated. +// The deputy being confused is the one process the issuer trusts to say which +// module is asking. +// +// Authentication was added in the previous round and authorisation was not, +// which is why the first reviewer asked for "authenticated, authorized peers". +func TestAnAuthenticatedCallerIsNotAutomaticallyAnAuthorisedOne(t *testing.T) { + mint := newHostMint(t, &hostMint{}) + cell := newCell(t) + certFile, keyFile, bundleFile, _, _ := cell.workload(t, testPrincipal) + mint.serveTLS(t, cell) + authorityValues(t) + tokenFile := filepath.Join(t.TempDir(), "token") + writeFile(t, tokenFile, "projected-token") + func() { ln, port := boundPort(t); provideListener(t, ln); t.Setenv("PORT", port) }() + t.Setenv("GATEWAY_URL", mint.URL) + t.Setenv(CredentialMintURLEnvironmentVariable, mint.URL+credentialMintPath) + t.Setenv(ProjectedTokenFileEnvironmentVariable, tokenFile) + t.Setenv(IdentityCertFileEnvironmentVariable, certFile) + t.Setenv(IdentityKeyFileEnvironmentVariable, keyFile) + t.Setenv(IdentityTrustBundleFileEnvironmentVariable, bundleFile) + t.Setenv(IdentityAllowedCallersFileEnvironmentVariable, identitiesFile(t, testGatewayPrincipal)) + t.Setenv(IdentityMintPeersFileEnvironmentVariable, identitiesFile(t, testGatewayPrincipal)) + t.Setenv(IdentityGatewayPeersFileEnvironmentVariable, identitiesFile(t, testGatewayPrincipal)) + t.Setenv(ContractProfileEnvironmentVariable, localProfile) + t.Setenv("ASSETS_DIR", t.TempDir()) + + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + reached := make(chan struct{}, 8) + server := New(Manifest{ID: testSolutionID}).Handle("/thing", func(context.Context, *Gateway) (any, error) { + reached <- struct{}{} + return map[string]string{"ok": "yes"}, nil + }) + ln, err := takeListener(server).start(ctx) + if err != nil { + t.Fatalf("boot: %v", err) + } + served := make(chan error, 1) + go func() { served <- server.serve(ctx, ln) }() + t.Cleanup(func() { cancel(); <-served }) + base := "https://127.0.0.1:" + os.Getenv("PORT") + + call := func(identity string) (*http.Response, error) { + client := &http.Client{Transport: &http.Transport{TLSClientConfig: &tls.Config{ + RootCAs: cell.roots, + Certificates: []tls.Certificate{*cell.identity(t, identity)}, + MinVersion: tls.VersionTLS13, + }}} + request, err := http.NewRequest(http.MethodGet, base+"/thing", nil) + if err != nil { + t.Fatal(err) + } + request.Header.Set("authorization", viewerBearer()) + // The headers a caller must not be able to assert for itself. + request.Header.Set(orgHeader, "org-the-caller-chose") + request.Header.Set(sessionHeader, "session-the-caller-chose") + request.Header.Set(workcontext.InstallationIDHeaderName, testInstallation) + return client.Do(request) + } + + t.Run("the gateway is served", func(t *testing.T) { + resp, err := call(testGatewayPrincipal) + if err != nil { + t.Fatalf("the admitted caller was refused, so this check is just a broken listener: %v", err) + } + _ = resp.Body.Close() + select { + case <-reached: + default: + t.Error("the admitted caller did not reach the handler") + } + }) + + for _, identity := range []string{ + // A module this solution consumes: holds a cell certificate, and is + // precisely the party that must not be able to drive this runtime's + // mints. + "spiffe://codefly.test/ns/modules/sa/things", + // Any other workload in the trust domain. + "spiffe://codefly.test/ns/solutions/sa/someone-else", + } { + t.Run("refused: "+identity, func(t *testing.T) { + resp, err := call(identity) + if err == nil { + defer func() { _ = resp.Body.Close() }() + t.Errorf("a caller authenticated as %s was served %d: holding a certificate from this cell's anchor is not the same as being a caller this solution serves, and this one set its own %s and %s", + identity, resp.StatusCode, orgHeader, sessionHeader) + } + select { + case <-reached: + t.Error("the handler ran for a caller this solution does not admit") + default: + } + }) + } +} + +// TestARotatedLeafIsStillHeldToTheFrozenPrincipal closes the half of the +// leaf check that only ran at boot. +// +// A source with no per-connection callback still rotates its leaf — that is what +// GetCertificate is for, and the projected pair is deliberately short-lived. The +// principal check ran once, against whatever was current then, so a rotation +// landing a certificate for a *different* workload (the wrong Secret replaced, a +// mesh CA re-issuing under a changed identity) would be served for the life of +// the process, with the one check that would have caught it having run before +// the file changed. +func TestARotatedLeafIsStillHeldToTheFrozenPrincipal(t *testing.T) { + c := newCell(t) + certFile, keyFile, bundleFile, _, _ := c.workload(t, testPrincipal) + server := New(Manifest{ID: testSolutionID}) + server.cfg = config{port: listenOn(t, server), identityCertFile: certFile, identityKeyFile: keyFile, + trustBundleFile: bundleFile, allowedCallersFile: identitiesFile(t, testGatewayPrincipal), mintPeersFile: identitiesFile(t, testGatewayPrincipal), + gatewayPeersFile: identitiesFile(t, testGatewayPrincipal)} + server.principal = testPrincipal + + config, err := server.serverIdentity() + if err != nil { + t.Fatalf("the boot refused a conforming pair: %v", err) + } + if _, err := config.GetCertificate(&tls.ClientHelloInfo{ServerName: "localhost"}); err != nil { + t.Fatalf("the leaf current at boot was refused: %v", err) + } + + // The projection is replaced with a pair for another workload, under the + // same anchor so nothing else about it is wrong. + rivalCert, rivalKey, _, _, _ := c.workload(t, "spiffe://codefly.test/ns/solutions/sa/another-workload") + writeFile(t, certFile, readFile(t, rivalCert)) + writeFile(t, keyFile, readFile(t, rivalKey)) + + _, err = config.GetCertificate(&tls.ClientHelloInfo{ServerName: "localhost"}) + if err == nil { + t.Fatal("a leaf issued for another workload was served after rotation: the principal check only ran at boot, against a file that has since changed") + } + if !strings.Contains(err.Error(), "another-workload") || !strings.Contains(err.Error(), testPrincipal) { + t.Errorf("refusal %q does not name both the identity served and this workload's frozen principal", err) + } +} + +// anchorAnnouncingIdentity is a source that says what its anchor is, which is +// the only sound way for one that rotates to be re-checkable. +type anchorAnnouncingIdentity struct { + config *tls.Config + anchor func() *x509.CertPool +} + +func (a anchorAnnouncingIdentity) ServerTLSConfig() (*tls.Config, error) { return a.config, nil } +func (a anchorAnnouncingIdentity) PeerAnchor() (*x509.CertPool, error) { return a.anchor(), nil } + +// TestASourcesAnchorIsAskedForAndNeverFabricated covers both halves of how a +// consumer's anchor is obtained, which pull in opposite directions. +// +// A *x509.CertPool carried on the base configuration is one object fixed when +// the source built it, so a source that rotates its anchor had every peer +// judged by the pool this process booted with. The previous answer to that was +// to call the source's GetConfigForClient with a fabricated ClientHelloInfo and +// read the pool off the answer — which is the blocker this cutover already has +// a name for: a source keyed on the hello answers a synthetic probe with one +// thing and the real handshake with another, demonstrably. Trading a stale +// anchor for a forged one is not a fix. +// +// So the source is asked directly, and a source that can only answer through a +// handshake is refused. +func TestASourcesAnchorIsAskedForAndNeverFabricated(t *testing.T) { + boot := newCell(t) + current := newCell(t) + + t.Run("a source that announces its anchor is followed", func(t *testing.T) { + announced := boot.roots + source := anchorAnnouncingIdentity{ + config: &tls.Config{MinVersion: tls.VersionTLS13, ClientAuth: tls.RequireAndVerifyClientCert, ClientCAs: boot.roots}, + anchor: func() *x509.CertPool { return announced }, + } + anchor, err := peerAnchorFrom(source, source.config) + if err != nil { + t.Fatalf("peerAnchorFrom: %v", err) + } + if anchor != boot.roots { + t.Fatal("the announced anchor was not used") + } + // It rotates, and nothing is rebuilt. + announced = current.roots + anchor, err = peerAnchorFrom(source, source.config) + if err != nil { + t.Fatalf("peerAnchorFrom after the rotation: %v", err) + } + if anchor != current.roots { + t.Error("the anchor came from the base configuration after the source rotated: a pool on the base is fixed when the source built it, so every peer would be judged by the one this process booted with") + } + }) + + t.Run("a source carrying only a base anchor uses it", func(t *testing.T) { + config := &tls.Config{MinVersion: tls.VersionTLS13, ClientAuth: tls.RequireAndVerifyClientCert, ClientCAs: boot.roots} + anchor, err := peerAnchorFrom(unaskableIdentity{config: config}, config) + if err != nil { + t.Fatalf("peerAnchorFrom: %v", err) + } + if anchor != boot.roots { + t.Error("a source that carries its anchor on the configuration it returned had it ignored") + } + }) + + t.Run("a per-handshake anchor with no way to ask for it is refused", func(t *testing.T) { + // The shape that used to be answered with a fabricated hello: the pool + // exists only inside the callback's answer. + config := &tls.Config{ + MinVersion: tls.VersionTLS13, + ClientAuth: tls.RequireAndVerifyClientCert, + GetConfigForClient: func(*tls.ClientHelloInfo) (*tls.Config, error) { + return &tls.Config{MinVersion: tls.VersionTLS13, ClientAuth: tls.RequireAndVerifyClientCert, ClientCAs: current.roots}, nil + }, + } + anchor, err := peerAnchorFrom(unaskableIdentity{config: config}, config) + if err == nil { + t.Fatal("an anchor was obtained from a source that can only answer through a handshake: the only way to get one is to fabricate a ClientHelloInfo, and a source keyed on the hello answers a probe and a real handshake differently — which is the blocker this runtime was already held up on, one field along") + } + if anchor != nil { + t.Error("a refusal returned a pool anyway") + } + if !strings.Contains(err.Error(), "PeerAnchor") { + t.Errorf("the refusal %q does not name the interface that fixes it", err) + } + }) +} + +// unaskableIdentity is a source that does NOT implement PeerAnchorSource, +// which the test sources here otherwise do. It exists so the refusal for a +// source that can only answer through a handshake stays reachable. +type unaskableIdentity struct{ config *tls.Config } + +func (u unaskableIdentity) ServerTLSConfig() (*tls.Config, error) { return u.config, nil } + +// TestASourceCannotReplaceThePostureItIsHeldTo: the posture checks enumerate +// what a conforming listener must have, which a source can satisfy completely +// while handing back a configuration whose clock, randomness, session secrets +// or ticket encoding it controls. +// +// Resumption is the sharpest of them: a resumed connection presents no +// certificate, so a source encoding its own tickets decides who gets in +// without any check in this file seeing a handshake. +func TestASourceCannotReplaceThePostureItIsHeldTo(t *testing.T) { + c := newCell(t) + certFile, keyFile, bundleFile, _, _ := c.workload(t, testPrincipal) + conforming := func(t *testing.T) *tls.Config { + t.Helper() + config, err := projectedIdentity{certFile: certFile, keyFile: keyFile, trustBundleFile: bundleFile}.ServerTLSConfig() + if err != nil { + t.Fatal(err) + } + return config + } + // The control: the shape every case below starts from is accepted. + if err := usableServerIdentity(conforming(t)); err != nil { + t.Fatalf("the conforming projected configuration was refused, so nothing below is about the field it changes: %v", err) + } + + for _, tc := range []struct { + name string + lower func(*tls.Config) + }{ + {"a clock of its own", func(cfg *tls.Config) { + cfg.Time = func() time.Time { return time.Unix(0, 0) } + }}, + //nolint:staticcheck // Deliberately sets the deprecated field, to prove the refusal fires. + {"a randomness source of its own", func(cfg *tls.Config) { cfg.Rand = strings.NewReader("not random") }}, + {"the session secrets written out", func(cfg *tls.Config) { cfg.KeyLogWriter = io.Discard }}, + {"resumption tickets it encodes", func(cfg *tls.Config) { + cfg.WrapSession = func(tls.ConnectionState, *tls.SessionState) ([]byte, error) { return nil, nil } + }}, + {"resumption tickets it decodes", func(cfg *tls.Config) { + cfg.UnwrapSession = func([]byte, tls.ConnectionState) (*tls.SessionState, error) { return nil, nil } + }}, + {"no verification at all", func(cfg *tls.Config) { cfg.InsecureSkipVerify = true }}, + {"a resumption ticket key of its own", func(cfg *tls.Config) { + //nolint:staticcheck // Deliberately sets the deprecated field, to prove the refusal fires. + cfg.SessionTicketKey = [32]byte{1} + }}, + } { + t.Run(tc.name, func(t *testing.T) { + config := conforming(t) + tc.lower(config) + if err := usableServerIdentity(config); err == nil { + t.Error("a configuration that keeps every enumerated requirement and replaces a posture Go owns was accepted: the checks answer whether the description is acceptable, which is not whether the handshake is") + } + }) + } + + // And a per-connection answer is held to the same list, not a copy of its + // reasoning: the answer replaces the base for that connection. + // + // The answer carries a real anchor, which is the part this case got wrong + // before: without one it was refused for the missing pool, so the + // assertion held whether or not the denylist was applied to the answer at + // all, and the mutant that stopped applying it survived. + anchor, err := peerAnchor(bundleFile) + if err != nil { + t.Fatal(err) + } + answered := func(lower func(*tls.Config)) *tls.Config { + base := conforming(t) + base.GetConfigForClient = func(*tls.ClientHelloInfo) (*tls.Config, error) { + answer := conforming(t) + answer.GetConfigForClient = nil + answer.ClientCAs = anchor + lower(answer) + return answer, nil + } + holdPerConnectionPosture(base, testPrincipal, func() ([]string, error) { return []string{testGatewayPrincipal}, nil }) + return base + } + + // The control: a conforming answer, with the anchor, is served. + if _, err := answered(func(*tls.Config) {}).GetConfigForClient(&tls.ClientHelloInfo{}); err != nil { + t.Fatalf("a conforming per-connection answer was refused, so nothing below is about the denylist: %v", err) + } + for _, tc := range []struct { + name, names string + lower func(*tls.Config) + }{ + {"the session secrets written out", "KeyLogWriter", func(cfg *tls.Config) { cfg.KeyLogWriter = io.Discard }}, + {"a clock of its own", "Time", func(cfg *tls.Config) { cfg.Time = func() time.Time { return time.Unix(0, 0) } }}, + {"its own chain verifier", "VerifyPeerCertificate", func(cfg *tls.Config) { + cfg.VerifyPeerCertificate = func([][]byte, [][]*x509.Certificate) error { return nil } + }}, + } { + t.Run("per connection: "+tc.name, func(t *testing.T) { + _, err := answered(tc.lower).GetConfigForClient(&tls.ClientHelloInfo{}) + if err == nil { + t.Fatalf("a per-connection answer setting %s was served: the configuration a callback returns replaces the base one, so it is held to the same posture", tc.names) + } + // And refused for THAT, not for something else the answer happens + // to be missing. + if !strings.Contains(err.Error(), tc.names) { + t.Errorf("the refusal %q does not name %s, so this case does not cover it", err, tc.names) + } + }) + } +} + +// TestASourcesConfigurationIsNotMutated: serverIdentity wraps GetCertificate, +// composes over VerifyConnection and wraps GetConfigForClient — all writes — +// and it did them to the configuration the source handed back. +// +// A source is under no obligation to return a fresh one. Returning a +// configuration it builds once and keeps is the ordinary way to write a source, +// and then those writes land on a value another handshake may be reading, and a +// second resolution wraps the wrapper. holdPerConnectionPosture says exactly +// this about a per-connection answer, in those words, and this function did the +// thing that comment forbids one level up. +func TestASourcesConfigurationIsNotMutated(t *testing.T) { + c := newCell(t) + certFile, keyFile, bundleFile, _, _ := c.workload(t, testPrincipal) + shared, err := projectedIdentity{certFile: certFile, keyFile: keyFile, trustBundleFile: bundleFile}.ServerTLSConfig() + if err != nil { + t.Fatal(err) + } + before := struct { + verify bool + perConn bool + getCert bool + numCerts int + }{ + verify: shared.VerifyConnection != nil, + perConn: shared.GetConfigForClient != nil, + getCert: shared.GetCertificate != nil, + numCerts: len(shared.Certificates), + } + + server := New(Manifest{ID: testSolutionID}).Identity(staticIdentity{config: shared}) + server.cfg = config{allowedCallersFile: identitiesFile(t, testGatewayPrincipal)} + server.principal = testPrincipal + config, err := server.serverIdentity() + if err != nil { + t.Fatalf("boot: %v", err) + } + if config == shared { + t.Fatal("serverIdentity returned the source's own configuration: everything it applies is a write, so the source's value is being modified while it may be in use elsewhere") + } + if (shared.VerifyConnection != nil) != before.verify { + t.Error("the source's VerifyConnection was written: the admission check belongs on this runtime's copy, not on the value the source keeps") + } + if (shared.GetConfigForClient != nil) != before.perConn { + t.Error("the source's GetConfigForClient was wrapped in place") + } + if (shared.GetCertificate != nil) != before.getCert || len(shared.Certificates) != before.numCerts { + t.Error("the source's certificate selection was rewritten in place") + } + // And the copy this runtime serves does carry the checks, or the test + // above is satisfied by a runtime that applies nothing at all. + if config.VerifyConnection == nil { + t.Error("the configuration this listener serves carries no caller admission") + } +} diff --git a/internal/seam/seam.go b/internal/seam/seam.go new file mode 100644 index 0000000..e877767 --- /dev/null +++ b/internal/seam/seam.go @@ -0,0 +1,34 @@ +// Package seam is how the passthrough test harness reaches the solution +// runtime's passthrough without the runtime exposing a way to build one in +// production. +// +// Server.PassthroughHandler used to be exported, and that was a public +// production bypass: New(...).Consumes(...).Credential(source). +// PassthroughHandler(gatewayURL) returned a usable, credential-bearing handler +// having skipped validate(), the mTLS boot, the caller allow-list, the +// published ceiling and authenticated outbound. A deployment that called it +// completed a viewer mint and a module call over plaintext and answered 200 — +// with the viewer's bearer and this workload's credential on the wire — and +// supplying a real credential source defeated the one mitigation it had. +// +// It had to stay reachable from passthroughtest, which is a consumer-facing +// package in this module and the documented seam for running the real +// passthrough against a fake host. Go's internal-package rule is what makes +// those two facts compatible: the root package registers the constructor here +// at init, passthroughtest reads it, and nothing outside this module can import +// this package at all. The gate is the compiler rather than a flag, a name or a +// runtime check, and the remaining looseness of the seam — a plaintext fake +// host, no contract, no listener — is now a property of this module's own tests +// instead of an API a solution can call. +package seam + +import "net/http" + +// Passthrough builds the passthrough for an already-configured *solution.Server +// against gatewayURL, with consumesJSON standing in for the api.consumes +// projection a boot resolves. The root package sets it; passthroughtest calls +// it. +// +// The server is untyped because the root package imports this one to register +// the function, so this one cannot name its types. The implementation asserts. +var Passthrough func(server any, gatewayURL, consumesJSON string) (http.Handler, error) diff --git a/mcp.go b/mcp.go index e2a1501..86f5593 100644 --- a/mcp.go +++ b/mcp.go @@ -2,6 +2,8 @@ package solution import ( "context" + "encoding/json" + "errors" "fmt" "log" "net/http" @@ -10,6 +12,7 @@ import ( codefly "github.com/codefly-dev/sdk-go" "github.com/modelcontextprotocol/go-sdk/auth" + "github.com/modelcontextprotocol/go-sdk/jsonrpc" "github.com/modelcontextprotocol/go-sdk/mcp" "github.com/modelcontextprotocol/go-sdk/oauthex" ) @@ -175,7 +178,7 @@ func (s *Server) mountMCP(mux *http.ServeMux) error { } // MCPEnvironment is what Serve resolves from the composition for the MCP -// surface, supplied instead by a caller of MCPHandler. +// surface, supplied instead by a caller of mcpHandler. type MCPEnvironment struct { // GatewayURL is the host gateway a tool call's Work Context is minted // through and its composed-module reads go to. @@ -189,20 +192,43 @@ type MCPEnvironment struct { PublicURL string } -// MCPHandler is the MCP surface exactly as Serve serves it — the same endpoint, -// the same metadata document, the same refusals, mounted by the same code — -// against an environment the caller supplies instead of the one Serve resolves. -// Nothing else of the solution is served, and nothing registers anywhere. +// mcpHandler is the MCP surface exactly as Serve serves it, for a test in this +// package. +// +// NOT exported, and it must not become so. It was `MCPHandler`, and its only +// callers are tests: an exported method that hands back a servable handler is +// the PassthroughHandler defect again — whoever mounts it serves the viewer's +// bearer and this workload's credential over whatever they mounted it on, +// having skipped validate(), the mTLS boot, the caller allow-list, the ceiling +// and authenticated outbound. This one also writes boot configuration from +// values its caller supplies, which is the one thing loadConfig exists to be +// the only source of. +// +// It is the same endpoint, the same metadata document, the same refusals, +// mounted by the same code, against an environment the caller supplies instead +// of the one Serve resolves. Nothing else of the solution is served, and +// nothing registers anywhere. +// +// TestNoExportedPathHandsOutAServableHandler fails if an exported path +// reappears; it is what caught this. // // It exists for tests: a solution drives its own tools under the runtime's // identity handling, with a fake host standing in for the gateway. A solution // serves with Serve, which overwrites the environment set here when it resolves // its own. -func (s *Server) MCPHandler(env MCPEnvironment) (http.Handler, error) { +func (s *Server) mcpHandler(env MCPEnvironment) (http.Handler, error) { if s.mcp == nil { return nil, fmt.Errorf("solution %q declares no MCP server (ServeMCP)", s.manifest.ID) } s.cfg.gatewayURL, s.cfg.mcpIssuerURL, s.cfg.mcpPublicURL = env.GatewayURL, env.IssuerURL, env.PublicURL + // And the derived set, because this overwrites what it was derived FROM. + // + // cfg.dialled is assembled in loadConfig; assigning the gateway URL here + // without rebuilding it left the set empty on exactly the seam a consumer + // runs, so the net that stops a tool naming an address this runtime + // dialled was silently off there. A derived field whose sources are + // reassigned is stale until it is not. + s.cfg.dialled = dialledAddresses(s.cfg.gatewayURL, s.cfg.mintURL) // The caller named both, so they are as explicit as the composition's // declared values: what validate() refuses is an issuer nobody chose, and // what a refusal has to name for a public URL is whoever set it. @@ -248,6 +274,46 @@ func ViewerFromContext(ctx context.Context) (*Gateway, error) { return gw, nil } +// actsForNobody reports whether an MCP method runs nothing on a viewer's +// behalf: the handshake, the keepalive, and the listings that describe what +// this solution declares. +// +// Everything else is held to this execution's credential. The list is what a +// method must be ON to skip that check, so a method nobody here has heard of is +// checked. +func actsForNobody(method string) bool { + switch method { + // The handshake, the keepalive and the listings that describe what this + // solution declares. + case "initialize", "ping", + "tools/list", "prompts/list", "resources/list", "resources/templates/list", + // The notifications the SDK itself sends, named one by one. + // + // This was strings.HasPrefix(method, "notifications/"), which is a + // DENYLIST wearing an allowlist's clothes: the namespace is open, the + // SDK's extension API lets author code register methods under it, and + // one registered there would have run with a viewer's gateway and no + // credential check — the round-nine defect, still reachable. An exact + // list cannot be widened by someone else's naming. + "notifications/initialized", "notifications/cancelled", + "notifications/progress", "notifications/roots/list_changed": + return true + } + return false +} + +// mcpCredentialUnavailableCode is what an MCP client is told when this process +// will not act for a viewer: -32001, in JSON-RPC's implementation-defined +// server range, beside the SDK's own CodeResourceNotFound (-32002). +// +// I claimed this could not be set because the SDK's wire-error type was +// internal. That was wrong, and wrong because I grepped for the type and +// stopped: package jsonrpc exports it as an alias +// (jsonrpc.Error = jsonrpc2.WireError) precisely so a server can set a code. A +// client that only gets a generic protocol error cannot tell "this solution +// cannot act right now" from a broken tool, so it cannot back off. +const mcpCredentialUnavailableCode = -32001 + // mcpViewer binds each incoming MCP request to the viewer it arrived as, so a // tool handler reads composed modules on their behalf and not as the solution. // @@ -270,9 +336,88 @@ func (s *Server) mcpViewer(next mcp.MethodHandler) mcp.MethodHandler { if call, ok := req.(*mcp.CallToolRequest); ok && call.Params != nil { log.Printf("solution %q: mcp tool %q", s.manifest.ID, call.Params.Name) } - id := stampedIdentity(header) - gw := newGateway(s.cfg.gatewayURL, id.bearer, id.orgID, id.sessionID) - return next(context.WithValue(ctx, mcpViewerKey{}, gw), method, req) + // Through gatewayFor, not newGateway. A raw gateway carries the + // viewer's headers and nothing else: no credential acquirer, so a tool + // call could not attest which module was asking; no attestation report + // and no terminal hook; and — the one that matters most — none of the + // published contract's ceilings, so a mint from an MCP tool call was + // held to nothing the contract claims. Every other path a viewer + // reaches this runtime on is built here, and this one has to be too. + gw := s.gatewayFor(header) + // A tool's error becomes content the client reads, below this + // middleware, so the runtime's own errors are sanitized before the + // author ever holds one. See Gateway.surfaced. + gw.sanitizeErrors = true + // The credential gate, before anything author-written runs. + // + // This is R4-3 on the MCP path, and it arrived with the surface rather + // than being missed in it: a plain handler and a ViewerBearer + // passthrough are both held here, and a tool call was not. A tool + // receives a gateway carrying the viewer's bearer, so running one while + // the issuer has withdrawn this execution's credential is acting for a + // viewer without authority — and a tool that mints nothing still sends + // that bearer wherever it dials. + // + // An ALLOWLIST of the methods that act for nobody, not a denylist of + // the ones that act. A denylist over a method set this package does not + // own is the mistake the TLS posture made three times: whatever it does + // not name, the far side keeps. A method the SDK adds tomorrow is gated + // here by default, and the cost of that being wrong is a refusal rather + // than an ungated viewer action. + if !actsForNobody(method) { + if err := s.actingForAViewer(ctx); err != nil { + // The sanitized sentence, not the error: it wraps whatever the + // source said — a mint URL, an issuer's own text, a dial + // failure naming an internal address — and an MCP client is as + // much a viewer's channel as a browser is. + log.Printf("solution %q: refusing an MCP %s for a viewer: %v", s.manifest.ID, method, err) + return nil, &jsonrpc.Error{ + Code: mcpCredentialUnavailableCode, + Message: credentialRefusalForAViewer(err), + } + } + } + result, err := next(context.WithValue(ctx, mcpViewerKey{}, gw), method, req) + // An error RESULT, which is not an error at all as far as this + // middleware is concerned. + // + // The SDK's typed AddTool wrapper turns a tool's returned error into + // CallToolResult{IsError: true} and returns a NIL err, so the branch + // below never runs on that path and the text travels as content. The + // runtime's own errors are sanitized where they leave its hands now + // (Gateway.surfaced, bearerTransport), which removes the issuer's + // words — but http.Client.Do prepends the URL it dialled to whatever + // the transport returned, and the author did not author that. + // + // So this is a net over the one thing left: if what a tool is about to + // tell a client contains an address THIS RUNTIME configured, the + // content is replaced wholesale and the original goes to the log. It + // matches on exact values the runtime knows — its gateway and mint + // URLs — not on a pattern, so a tool's own message about its own + // domain is untouched. + // + // It is a net and not the boundary. The boundary is surfaced; this + // catches what net/http adds after it. + if err == nil { + result = s.withoutRuntimeAddresses(result, method) + } + if err != nil { + // Sanitized on the way out, through the same function the handler + // routes use. A tool that called ForModule and was refused handed + // the viewer's agent client the whole chain: "mint returned HTTP + // 503", and with the issuer unreachable the mint's own URL and + // dial address. The gate above stops a tool running without a + // credential; this stops what a tool learned about the issuer + // reaching the client when one fails mid-call. + // + // The author's own errors are sanitized by the same rule as a + // handler's, which is what that rule is for: a tool's error is + // written for a model and read by whoever is driving it. + _, message := handlerErrorResponse(err) + log.Printf("solution %q: mcp %s failed: %v", s.manifest.ID, method, err) + return result, errors.New(message) + } + return result, nil } } @@ -361,6 +506,29 @@ func (s *Server) requireStampedViewer(next http.Handler) http.Handler { stamped, sessionHeader, detail), http.StatusForbidden) return } + // And this execution's own credential, at the HTTP boundary. + // + // The method-layer gate in mcpViewer refuses the same thing, and both + // are here on purpose. This one answers a real HTTP 503, which + // JSON-RPC has no equivalent for at all, and it holds anything that + // reaches this handler without going through the method middleware. + // + // The method layer is not codeless: it carries -32001 through + // jsonrpc.Error. This comment said the SDK's wire-error type was + // internal and a code could not be set, which was my own false claim — + // I grepped for the type, found internal/jsonrpc2, and stopped looking + // for the exported alias. Corrected in the code one commit and left + // standing here. + if err := s.actingForAViewer(r.Context()); err != nil { + // The sanitized sentence, never the source's error: it wraps + // whatever the issuer said, which is a mint URL and sometimes a + // dial failure naming an internal address. An agent client is a + // viewer's channel like a browser is, and this was the disclosure + // executed round eight reproduced. + log.Printf("solution %q: refusing an MCP request for a viewer: %v", s.manifest.ID, err) + http.Error(w, credentialRefusalForAViewer(err), http.StatusServiceUnavailable) + return + } next.ServeHTTP(w, r) }) } @@ -516,26 +684,34 @@ func firstForwarded(r *http.Request, name string) string { // MCPConfigurationGroup names the workspace-configuration group a composition // declares for a solution that serves MCP, and MCPIssuerURLKey and -// MCPPublicURLKey the two values in it. They are resolved through the SDK the -// way the registration secret is (see SolutionRegistrationSecretGroup), because -// a Codefly composition has no way to set a bare environment variable on a -// service: every value a service receives is either an endpoint the SDK +// MCPPublicURLKey the two values in it. They are resolved through the SDK +// because a Codefly composition has no way to set a bare environment variable +// on a service: every value a service receives is either an endpoint the SDK // resolves or a declared configuration the render projects. Both were read as // bare environment variables when ServeMCP landed, which named a provisioning // path that does not exist. // +// This used to say "the way the registration secret is", naming +// SolutionRegistrationSecretGroup. That group and the registrations it fed are +// deleted; the comparison outlived the thing it compared to. +// // MCPIssuerURLKey is the origin MCP clients authenticate against — the host as // an authorization server. A deployment must supply it: what the SDK resolves // is the address this composition dials the host at, in-cluster and reachable // by no client. It is the one value a composition has to declare for MCP, and // it stops being one when module-saas-starter#1003 settles what `iss` is. // -// MCPPublicURLKey overrides the public MCP URL, which the runtime otherwise -// derives by construction (see resolveMCPPublicURL). It exists for a host whose -// gateway fronts solutions under some other route; it must end in MCPPath, -// because the metadata document's own URL is derived from it by swapping that -// suffix — the same pairing the registration token URLs use, and refused at -// boot for the same reason when it cannot be made. +// MCPPublicURLKey is the public MCP URL. A deployed MCP surface must declare +// it: the runtime can derive one from PUBLIC_URL, but the derivation builds +// /solutions//mcp, which encodes the route the HOST serves this +// solution on — a layout this runtime does not know. The derivation stands for +// a local run, where whoever made the guess can check it. +// +// It must end in MCPPath, because the metadata document's own URL is derived +// from it by swapping that suffix, and a value that cannot be paired is +// refused at boot rather than guessed at. The pairing is siblingURL, which the +// deleted registration token URLs used for the same reason — it is the only +// caller left. const ( MCPConfigurationGroup = "mcp" MCPIssuerURLKey = "issuer-url" @@ -624,10 +800,10 @@ func (c config) validateMCP() error { if u, err := url.Parse(c.mcpIssuerURL); err != nil || !u.IsAbs() || u.Host == "" { if c.environmentLoadErr != nil { return fmt.Errorf("unresolved host issuer %q, and loading Codefly's injected environment failed first: %w — nothing the SDK resolves can be trusted to be absent until that is fixed", - c.mcpIssuerURL, c.environmentLoadErr) + redactedURL(c.mcpIssuerURL), c.environmentLoadErr) } return fmt.Errorf("unresolved host issuer %q: ServeMCP publishes it as the authorization server of this MCP resource, so a client has nowhere to authenticate without it. Provision the workspace configuration %s with the origin MCP clients authenticate against and declare that group as a workspace-configuration dependency of this backend, or ensure the SDK resolves the host frontend's http endpoint", - c.mcpIssuerURL, mcpConfigurationValue(MCPIssuerURLKey)) + redactedURL(c.mcpIssuerURL), mcpConfigurationValue(MCPIssuerURLKey)) } // In a deployment neither value below can be resolved from inside the // cluster, for the reason the loopback refusals above exist: the runtime @@ -640,24 +816,74 @@ func (c config) validateMCP() error { if c.mcpPublicURL == "" { // Only reachable with PUBLIC_URL unresolved and no override // declared: the identifier is otherwise derived by construction. - return fmt.Errorf("no public MCP URL in the deployed runtime context %q: it is derived as %s%s, and PUBLIC_URL resolved nothing — so the identifier would be reconstructed per request from the forwarded headers, missing the route the gateway strips (and naming the scheme of the hop, not the client's), and an MCP client binds its token to the identifier it dialled and rejects any other. Set PUBLIC_URL to the origin clients reach this product at, or provision the workspace configuration %s with the URL they dial, ending in %s", - c.runtimeContext, gatewaySolutionsRoute, MCPPath, mcpConfigurationValue(MCPPublicURLKey), MCPPath) + // + // And this must NOT offer PUBLIC_URL as the remedy. It did, and + // setting it in a deployment reaches the very next refusal below — + // a derived identifier is refused in a deployed context, because + // the derivation encodes the route the host serves this solution + // on. So the diagnostic sent an operator one step further and then + // failed the boot again, which is worse than the first refusal + // because it reads as progress. There is one remedy in a + // deployment and this names only that one. + return fmt.Errorf("no public MCP URL in the deployed runtime context %q: the identifier an MCP client binds its token to has to be the one it dialled, and nothing here knows that — PUBLIC_URL resolved nothing, so it would be reconstructed per request from the forwarded headers, missing the route the gateway strips and naming the scheme of the hop rather than the client's. Provision the workspace configuration %s with the URL clients dial, ending in %s, and declare the %s group as a workspace-configuration dependency of this backend. Setting PUBLIC_URL does not answer this in a deployment: a derived identifier is refused here too, because the derivation encodes the route the HOST serves this solution on", + c.runtimeContext, mcpConfigurationValue(MCPPublicURLKey), MCPPath, MCPConfigurationGroup) } if !c.mcpIssuerExplicit { return fmt.Errorf("no %s provisioned in the deployed runtime context %q: the issuer resolved from the SDK (%q) is the address this composition dials the host at, which no MCP client can reach, and it would be published as this resource's authorization server. Provision that workspace configuration with the origin clients authenticate against and declare the %s group as a workspace-configuration dependency of this backend", - mcpConfigurationValue(MCPIssuerURLKey), c.runtimeContext, c.mcpIssuerURL, MCPConfigurationGroup) + mcpConfigurationValue(MCPIssuerURLKey), c.runtimeContext, redactedURL(c.mcpIssuerURL), MCPConfigurationGroup) } + // Declared, not derived. The derived identifier satisfied a check for + // "a public URL is set", so this refusal never fired in the one case + // it matters: a deployment with PUBLIC_URL resolved, where + // /solutions//mcp is this runtime encoding the HOST's + // route layout. That is what the manifest URL was changed to stop + // doing in this cutover — the route a client reaches a solution + // through is the host's to resolve, and a runtime guessing it was + // wrong in every deployment but the author's machine. Locally the + // derivation stands, because there the guess is checkable by the + // person making it. + if c.mcpPublicURL != "" && !c.mcpPublicExplicit { + return fmt.Errorf("the public MCP URL in the deployed runtime context %q is derived (%q), not declared: it is built as %s%s, which encodes the route the HOST serves this solution on — a layout this runtime does not know and must not assume. Provision the workspace configuration %s with the URL clients actually dial, ending in %s", + c.runtimeContext, redactedURL(c.mcpPublicURL), gatewaySolutionsRoute, MCPPath, + mcpConfigurationValue(MCPPublicURLKey), MCPPath) + } + } + // After the deployed block, whose refusals name the specific value an + // operator has not provisioned; this one reports a defect in a value they + // did. Put first, it answered "not https" for an in-cluster address whose + // real problem is that no MCP client can reach it at all. + if err := usablePublishedURL("host issuer", c.mcpIssuerURL, + mcpConfigurationValue(MCPIssuerURLKey), deployedRuntimeContext(c.runtimeContext)); err != nil { + return err } if c.mcpPublicURL == "" { return nil } if u, err := url.Parse(c.mcpPublicURL); err != nil || !u.IsAbs() || u.Host == "" { return fmt.Errorf("unusable %s (%q): it is the resource identifier an MCP client binds its token to, so it must be the absolute URL clients dial", - c.mcpPublicURLSource(), c.mcpPublicURL) + c.mcpPublicURLSource(), redactedURL(c.mcpPublicURL)) + } + // Before the pairing check, because a URL carrying a query or a fragment + // cannot pair either — and the pairing refusal would then report the wrong + // defect while printing the value verbatim. It did exactly that: an + // identifier ending "/mcp?token=s3cr3t" was refused as "unpairable", with + // the token in the message and so in the boot log. + // + // Both URLs are held to what the mint and gateway URLs are held to. They + // are published rather than dialled — the issuer as this resource's + // authorization server, the public URL as the identifier a client binds a + // token to — which makes the same three defects worse rather than milder: + // userinfo is a credential this runtime publishes to every client that + // reads the document, a query travels with the identifier, and a fragment + // is never sent so the identifier is not the one it appears to be. + deployed := deployedRuntimeContext(c.runtimeContext) + if err := usablePublishedURL(c.mcpPublicURLSource(), c.mcpPublicURL, + mcpConfigurationValue(MCPPublicURLKey), deployed); err != nil { + return err } if siblingURL(c.mcpPublicURL, MCPPath, ProtectedResourceMetadataPath) == "" { return fmt.Errorf("unpairable %s (%q): it must end in %s, because the metadata document a 401 points a client to is derived from it by swapping that suffix", - c.mcpPublicURLSource(), c.mcpPublicURL, MCPPath) + c.mcpPublicURLSource(), redactedURL(c.mcpPublicURL), MCPPath) } return nil } @@ -672,3 +898,194 @@ func (c config) mcpPublicURLSource() string { } return "public MCP URL derived from PUBLIC_URL" } + +// usablePublishedURL holds a URL this runtime publishes to what validate() +// holds the addresses it dials to. +// +// The same three refusals, for a reason that is stronger here: a dialled URL +// carrying userinfo is a secret this process logs and net/http then strips, and +// a published one is a secret handed to every client that reads the document. +// https is required only in a deployment, because http://localhost is what a +// developer machine serves and refusing it there would refuse the local run +// this runtime is also meant to support. +func usablePublishedURL(name, value, fixWith string, deployed bool) error { + u, err := url.Parse(value) + if err != nil || !u.IsAbs() || u.Host == "" { + return fmt.Errorf("unusable %s (%q): it is published to MCP clients, so it must be an absolute URL. Provision %s", name, redactedURL(value), fixWith) + } + if deployed && u.Scheme != "https" { + return fmt.Errorf("%s (%q) is not https in a deployed runtime context: MCP clients authenticate against it and dial it, so a plaintext hop hands their credentials to anything on the path. Provision %s with an https URL", + name, redactedURL(value), fixWith) + } + switch { + case u.User != nil: + return fmt.Errorf("%s carries userinfo (%q): this URL is published in a document clients read, so credentials in it are disclosed to every one of them. Provision %s with the origin and path and nothing else", + name, redactedURL(value), fixWith) + case u.RawQuery != "": + return fmt.Errorf("%s carries a query string (%q): clients bind a token to this identifier and dial it, and anything secret in a query is logged by every hop that sees it. Provision %s with the origin and path and nothing else", + name, redactedURL(value), fixWith) + case u.Fragment != "": + return fmt.Errorf("%s carries a fragment (%q): a fragment is never sent, so this is not the identifier it appears to be. Provision %s with the URL clients actually dial", + name, redactedURL(value), fixWith) + } + return nil +} + +// namedInJSON decodes one JSON document and reports the first string value or +// key inside it that names something the given test matches. +// +// Decoded rather than textual, because JSON escaping means the serialized form +// and the form a client reads are not the same string: `https:\/\/host` and +// `https://host` are one value with two spellings, and a search over the bytes +// sees only the one it was given. A document this runtime cannot decode is +// reported as offending for the same reason an unmarshalable value is — it +// cannot be cleared if it cannot be read. +func namedInJSON(document []byte, named func(string) (string, bool)) (string, bool) { + var decoded any + if err := json.Unmarshal(document, &decoded); err != nil { + return "undecodable structured content: " + err.Error(), true + } + var walk func(any) (string, bool) + walk = func(value any) (string, bool) { + switch typed := value.(type) { + case string: + if _, found := named(typed); found { + return typed, true + } + case []any: + for _, item := range typed { + if offending, found := walk(item); found { + return offending, true + } + } + case map[string]any: + for key, item := range typed { + // The KEY as well: a map keyed by the address discloses it + // just as a value does. + if _, found := named(key); found { + return key, true + } + if offending, found := walk(item); found { + return offending, true + } + } + } + return "", false + } + return walk(decoded) +} + +// withoutRuntimeAddresses replaces an error result that names an address this +// runtime dialled. +// +// Only an IsError result, and only when the text contains the gateway or mint +// URL this process was configured with: a tool's own refusal is its own to +// word, and rewriting every error result would take that away. +func (s *Server) withoutRuntimeAddresses(result mcp.Result, method string) mcp.Result { + call, ok := result.(*mcp.CallToolResult) + if !ok || call == nil || !call.IsError { + return result + } + // The TEXT content, and the STRUCTURED content. + // + // This read only Content. The pinned SDK serializes StructuredContent as + // well, so a result with generic text and + // StructuredContent{"error": err.Error()} went to the agent client with + // the mint's address in it — the net was installed for exactly that + // disclosure and looked in one of the two places the SDK sends. + // + // Structured content is arbitrary JSON, so it is rendered once and + // searched as a whole rather than walked field by field: a rule that + // inspected a field named "error" would be a rule about this round's + // example. + named := func(text string) (string, bool) { + for _, dialled := range s.cfg.dialled { + if dialled != "" && strings.Contains(text, dialled) { + return dialled, true + } + } + return "", false + } + replaced := func(where, offending string) mcp.Result { + log.Printf("solution %q: an MCP %s error result named an address this runtime dialled (in its %s); replaced: %s", + s.manifest.ID, method, where, offending) + return &mcp.CallToolResult{ + IsError: true, + Content: []mcp.Content{&mcp.TextContent{ + Text: "this solution could not complete the call", + }}, + } + } + // SERIALIZE EVERYTHING FIRST, THEN INSPECT, THEN PUBLISH AN OWNED SNAPSHOT. + // + // The order is the whole fix, and getting it wrong twice is why it is + // spelled out. First this inspected the object and returned it for the SDK + // to serialize later, so a MarshalJSON answering differently the second + // time disclosed on the second call. Then the structured half was frozen + // and the text half was not — and the text was read BEFORE the structured + // marshalling ran, so a structured value holding a pointer to the result's + // own *mcp.TextContent could rewrite the text, from inside MarshalJSON, + // after it had passed inspection. `{}` is a clean serialization of a value + // whose side effect put the mint's address in a sibling field. + // + // So: every marshaller runs first, then nothing is inspected that is not + // about to be published, and what is published is a value built here from + // strings and bytes this runtime owns. Nothing the caller still holds a + // pointer to can change it, and no marshaller is asked twice. + var rendered []byte + if call.StructuredContent != nil { + // Marshalling failure is itself a reason to replace: a structured + // value this runtime cannot read is one it cannot clear either, and + // the SDK is about to serialize it. + var err error + if rendered, err = json.Marshal(call.StructuredContent); err != nil { + return replaced("structured content", "unreadable structured content: "+err.Error()) + } + } + // Only now are the texts read, and they are COPIED as they are read, so + // what is inspected below and what is published are the same strings. + snapshot := make([]mcp.Content, 0, len(call.Content)) + for _, content := range call.Content { + text, isText := content.(*mcp.TextContent) + if !isText { + // Content this runtime cannot read is content it cannot clear. + // Replacing is the fail-closed answer: the alternative is + // publishing a type whose serialization nothing here inspected. + return replaced("non-text content", fmt.Sprintf("%T", content)) + } + snapshot = append(snapshot, &mcp.TextContent{ + Text: text.Text, + Annotations: text.Annotations, + Meta: text.Meta, + }) + } + for _, content := range snapshot { + text := content.(*mcp.TextContent) + if _, found := named(text.Text); found { + return replaced("text content", text.Text) + } + } + if rendered != nil { + // The DECODED strings, not the serialization. + // + // This searched the marshalled bytes for the configured URL, and JSON + // has more than one spelling for the same string: a json.RawMessage + // holding {"error":"https:\/\/mint…"} keeps those escapes through + // Marshal, the search for "https://mint…" misses, and the client + // decodes the address back out. So the bytes are decoded and every + // string inside them — and every key — is checked, which is the + // representation the client actually sees. + if offending, found := namedInJSON(rendered, named); found { + return replaced("structured content", offending) + } + } + clean := &mcp.CallToolResult{ + IsError: true, + Content: snapshot, + Meta: call.Meta, + } + if rendered != nil { + clean.StructuredContent = json.RawMessage(rendered) + } + return clean +} diff --git a/mcp_test.go b/mcp_test.go index 49452ea..9c2e6be 100644 --- a/mcp_test.go +++ b/mcp_test.go @@ -4,6 +4,8 @@ import ( "bytes" "context" "encoding/json" + "errors" + "fmt" "io" "log" "net/http" @@ -11,10 +13,13 @@ import ( "os" "strings" "sync" + "sync/atomic" "testing" "github.com/codefly-dev/core/resources" codefly "github.com/codefly-dev/sdk-go" + "github.com/codefly-dev/sdk-go/workcontext" + "github.com/modelcontextprotocol/go-sdk/jsonrpc" "github.com/modelcontextprotocol/go-sdk/mcp" ) @@ -25,6 +30,9 @@ import ( // Everything the runtime's MCP surface knows about the caller comes from here. type stampedTransport struct { bearer, userID, orgID, sessionID, credentialKind string + // installation is what the gateway stamps as x-codefly-installation-id, + // which every mint the runtime runs names. + installation string // prefix is the path the gateway strips before the solution sees the // request, forwarded as x-forwarded-prefix. prefix string @@ -35,15 +43,17 @@ func (s stampedTransport) RoundTrip(r *http.Request) (*http.Response, error) { r.Header.Del(orgHeader) r.Header.Del(sessionHeader) r.Header.Del(credentialKindHeader) + r.Header.Del(workcontext.InstallationIDHeaderName) if s.bearer != "" { r.Header.Set("authorization", s.bearer) } for name, value := range map[string]string{ - userHeader: s.userID, - orgHeader: s.orgID, - sessionHeader: s.sessionID, - credentialKindHeader: s.credentialKind, - forwardedPrefixHeader: s.prefix, + userHeader: s.userID, + orgHeader: s.orgID, + sessionHeader: s.sessionID, + credentialKindHeader: s.credentialKind, + workcontext.InstallationIDHeaderName: s.installation, + forwardedPrefixHeader: s.prefix, } { if value != "" { r.Header.Set(name, value) @@ -56,11 +66,12 @@ func (s stampedTransport) RoundTrip(r *http.Request) (*http.Response, error) { // same org and session every other test's viewer acts under. func viewerStamp() stampedTransport { return stampedTransport{ - bearer: "Bearer viewer-token", + bearer: viewerBearer(), userID: "viewer-principal", orgID: viewerOrg, sessionID: viewerSession, credentialKind: "user_session", + installation: testInstallation, } } @@ -82,8 +93,15 @@ type askOutput struct { // host. register adds the tools the test drives. func serveMCP(t *testing.T, env MCPEnvironment, register func(*mcp.Server)) *httptest.Server { t.Helper() - s := New(Manifest{ID: mcpServerName, Title: "Wiki"}).ServeMCP(mcpServerName, "v1.2.3", register) - handler, err := s.MCPHandler(env) + // A credential for this workload, because a tool call attests which module + // is asking and this runtime refuses to send a mint it cannot attest for. + // Before the cutover nothing here held one; the refusal is the point, and + // a test seam that skipped it would be testing a path no deployment has. + mint := newHostMint(t, &hostMint{}) + s := New(Manifest{ID: mcpServerName, Title: "Wiki"}). + ServeMCP(mcpServerName, "v1.2.3", register). + Credential(stubCredentialSource{credential: mintedCredential(t, mint, testProjectionAudience)}) + handler, err := s.mcpHandler(env) if err != nil { t.Fatalf("mount MCP: %v", err) } @@ -159,7 +177,7 @@ func TestMCPToolCallRunsAsTheViewer(t *testing.T) { } mint := <-host.mints - if mint.Bearer != "Bearer viewer-token" { + if mint.Bearer != viewerBearer() { t.Errorf("mint presented bearer %q, want the viewer's — accounts must resolve the viewer as owner", mint.Bearer) } if mint.OrgID != viewerOrg { @@ -552,6 +570,7 @@ func TestMCPIsStateless(t *testing.T) { } req.Header.Set("authorization", stamp.bearer) req.Header.Set(sessionHeader, stamp.sessionID) + req.Header.Set(workcontext.InstallationIDHeaderName, testInstallation) stream, err := http.DefaultClient.Do(req) if err != nil { t.Fatalf("GET %s: %v", MCPPath, err) @@ -592,11 +611,11 @@ func TestServeMCPRefusesAMisDeclaredSurface(t *testing.T) { } } -// TestMCPHandlerRefusesWithoutAServeMCPDeclaration keeps the test seam honest: +// TestTheMCPSeamRefusesWithoutAServeMCPDeclaration keeps the test seam honest: // it serves the surface Serve serves, and there is none to serve here. -func TestMCPHandlerRefusesWithoutAServeMCPDeclaration(t *testing.T) { - if _, err := New(Manifest{ID: "wiki"}).MCPHandler(MCPEnvironment{IssuerURL: testIssuer}); err == nil { - t.Error("MCPHandler accepted a solution that declares no MCP server") +func TestTheMCPSeamRefusesWithoutAServeMCPDeclaration(t *testing.T) { + if _, err := New(Manifest{ID: "wiki"}).mcpHandler(MCPEnvironment{IssuerURL: testIssuer}); err == nil { + t.Error("mcpHandler accepted a solution that declares no MCP server") } } @@ -671,8 +690,8 @@ func TestServeMCPRefusesARedeclaredSurface(t *testing.T) { if !strings.Contains(err.Error(), "ServeMCP") { t.Errorf("refusal %q does not name ServeMCP", err) } - if _, err := s.MCPHandler(MCPEnvironment{IssuerURL: testIssuer}); err == nil { - t.Error("MCPHandler served a redeclared surface") + if _, err := s.mcpHandler(MCPEnvironment{IssuerURL: testIssuer}); err == nil { + t.Error("mcpHandler served a redeclared surface") } } @@ -732,11 +751,31 @@ func TestValidateRefusesADeployedMCPThatNobodyAddressed(t *testing.T) { t.Errorf("refusal %q does not name %s", err, mcpConfigurationValue(MCPIssuerURLKey)) } - told := addressed - told.mcpIssuerExplicit = true - told.mcpIssuerURL = "https://app.example.com" + // An operator who declared the issuer but let the identifier be derived is + // still refused, and this is the case that used to pass. + // + // The derived value is /solutions//mcp: this runtime + // encoding the route the HOST serves it on. That assumption is what the + // Module Federation manifest URL was changed to stop making in this + // cutover, for the same reason — the route a client reaches a solution + // through is the host's to resolve, and a runtime guessing it is right + // only on the machine of whoever wrote the guess. A derived value also + // satisfied the "a public URL is set" check above, so nothing refused it. + halfTold := addressed + halfTold.mcpIssuerExplicit = true + halfTold.mcpIssuerURL = "https://app.example.com" + err = halfTold.validateMCP() + if err == nil { + t.Fatal("a deployed MCP surface was accepted with an identifier derived from PUBLIC_URL: a client binds its token to the URL it dialled, and this one is this runtime's guess about the host's route") + } + if !strings.Contains(err.Error(), mcpConfigurationValue(MCPPublicURLKey)) { + t.Errorf("refusal %q does not name %s, the configuration that fixes it", err, mcpConfigurationValue(MCPPublicURLKey)) + } + + told := halfTold + told.mcpPublicExplicit = true if err := told.validateMCP(); err != nil { - t.Errorf("a deployed MCP surface an operator addressed was refused: %v", err) + t.Errorf("a deployed MCP surface an operator addressed in full was refused: %v", err) } // A local run resolves a host it can actually reach and is unaffected: the @@ -762,6 +801,101 @@ func TestViewerFromContextOutsideAServedRequest(t *testing.T) { } } +// TestAnMCPToolCallIsHeldToThePublishedCeiling is the gap integrating this +// surface with the cutover exposed. +// +// The MCP middleware built its viewer gateway with newGateway, which carries +// the caller's headers and nothing else. Every other path a viewer reaches this +// runtime on is built by gatewayFor, which attaches the credential controller, +// the attestation report, the terminal hook and — the one that matters here — +// the ceilings of the contract this process published. So a mint driven from an +// MCP tool call was held to nothing the contract claims, while the identical +// mint from a handler or the passthrough was refused. +// +// The ceiling is declared, not derived, for the reason AGENTS.md gives: one +// computed from what the code asks for is satisfied by construction. A surface +// that bypasses it is the same defect by another route. +// +// Driven through the real middleware, by a tool reading its own viewer with +// ViewerFromContext. The first version of this test called gatewayFor directly +// and passed against the broken code, which is the mistake it exists to catch. +func TestAnMCPToolCallIsHeldToThePublishedCeiling(t *testing.T) { + mint := newHostMint(t, &hostMint{}) + var ( + seen atomic.Bool + ceilings atomic.Bool + audience atomic.Bool + acquirer atomic.Bool + reporting atomic.Bool + terminal atomic.Bool + ) + register := func(srv *mcp.Server) { + mcp.AddTool(srv, &mcp.Tool{Name: "inspect", Description: "reports its own viewer"}, + func(ctx context.Context, _ *mcp.CallToolRequest, _ askInput) (*mcp.CallToolResult, askOutput, error) { + gw, err := ViewerFromContext(ctx) + if err != nil { + return nil, askOutput{}, err + } + seen.Store(true) + ceilings.Store(gw.ceilings != nil) + _, declared := gw.ceilings["wiki-api"] + audience.Store(declared) + acquirer.Store(gw.workload != nil) + reporting.Store(gw.report != nil) + terminal.Store(gw.terminal != nil) + return nil, askOutput{Status: 200}, nil + }) + } + + server := New(Manifest{ID: mcpServerName, Title: "Wiki"}). + ServeMCP(mcpServerName, "v1.2.3", register). + Credential(stubCredentialSource{credential: mintedCredential(t, mint, testProjectionAudience)}) + // A resolved contract with one audience, which is what the ceiling is read + // off; contractResolved is the flag that makes it governing. + server.contract = effectiveContract{ + Solution: mcpServerName, + Bindings: []contractBinding{{Audience: "wiki-api", Ceiling: []Scope{{ResourceKind: "collection", Actions: []string{"read"}}}}}, + } + server.contractResolved = true + + handler, err := server.mcpHandler(MCPEnvironment{IssuerURL: testIssuer}) + if err != nil { + t.Fatalf("mount MCP: %v", err) + } + host := httptest.NewServer(handler) + t.Cleanup(host.Close) + session := connectMCP(t, host, viewerStamp()) + result, err := session.CallTool(context.Background(), &mcp.CallToolParams{ + Name: "inspect", + Arguments: map[string]any{"question": "anything"}, + }) + if err != nil { + t.Fatalf("tools/call: %v", err) + } + if result.IsError { + t.Fatalf("tools/call failed: %v", toolErrorText(result)) + } + + if !seen.Load() { + t.Fatal("the tool never ran, so this test asserted nothing about the gateway it would have been given") + } + if !ceilings.Load() { + t.Error("the gateway an MCP tool call receives carries no ceilings: a mint driven from a tool call is held to nothing the published contract claims, while the identical mint from a handler is refused") + } + if !audience.Load() { + t.Error("the gateway's ceilings do not name the declared audience") + } + if !acquirer.Load() { + t.Error("the gateway has no credential acquirer, so a tool call cannot attest which module is asking") + } + if !reporting.Load() { + t.Error("the gateway has no attestation report, so a refused mint is reported by nothing") + } + if !terminal.Load() { + t.Error("the gateway has no terminal hook, so a refusal that should end the process does not") + } +} + // --- the composition's half: a derived identifier and one declared value --- // mcpConfigurationCarrier is the environment variable Codefly's render projects @@ -799,7 +933,7 @@ func TestMCPPublicURLIsDerivedFromTheProductOrigin(t *testing.T) { t.Setenv("PORT", "8080") t.Setenv("PUBLIC_URL", "https://app.example.com/") - cfg := loadConfig(context.Background(), "wiki") + cfg := loadConfig(context.Background(), "wiki", nil) if want := "https://app.example.com/solutions/wiki" + MCPPath; cfg.mcpPublicURL != want { t.Fatalf("mcpPublicURL = %q, want the derived %q", cfg.mcpPublicURL, want) } @@ -827,7 +961,7 @@ func TestMCPPublicURLIsDerivedFromTheProductOrigin(t *testing.T) { // No origin to derive from: the identifier falls back to the forwarded // headers, which validate() refuses in a deployment. t.Setenv("PUBLIC_URL", "") - if got := loadConfig(context.Background(), "wiki").mcpPublicURL; got != "" { + if got := loadConfig(context.Background(), "wiki", nil).mcpPublicURL; got != "" { t.Errorf("with no PUBLIC_URL, mcpPublicURL = %q, want empty", got) } } @@ -843,7 +977,7 @@ func TestMCPPublicURLOverrideIsADeclaredConfiguration(t *testing.T) { const declared = "https://mcp.example.com/wiki" + MCPPath declareMCPConfiguration(t, MCPPublicURLKey, declared+"/") - cfg := loadConfig(context.Background(), "wiki") + cfg := loadConfig(context.Background(), "wiki", nil) if cfg.mcpPublicURL != declared { t.Fatalf("mcpPublicURL = %q, want the declared %q", cfg.mcpPublicURL, declared) } @@ -853,7 +987,7 @@ func TestMCPPublicURLOverrideIsADeclaredConfiguration(t *testing.T) { // The bare environment variable is gone: an operator who exports the name // the runtime used to read gets the derived URL, not theirs. t.Setenv("MCP_PUBLIC_URL", "https://exported.example.com/mcp") - if got := loadConfig(context.Background(), "wiki").mcpPublicURL; got != declared { + if got := loadConfig(context.Background(), "wiki", nil).mcpPublicURL; got != declared { t.Errorf("mcpPublicURL = %q, want the declared %q — a bare MCP_PUBLIC_URL must have no effect", got, declared) } } @@ -870,7 +1004,7 @@ func TestMCPIssuerIsADeclaredConfiguration(t *testing.T) { const declared = "https://login.example.com" declareMCPConfiguration(t, MCPIssuerURLKey, declared+"/") - cfg := loadConfig(context.Background(), "wiki") + cfg := loadConfig(context.Background(), "wiki", nil) if cfg.mcpIssuerURL != declared { t.Fatalf("mcpIssuerURL = %q, want the declared %q", cfg.mcpIssuerURL, declared) } @@ -879,7 +1013,7 @@ func TestMCPIssuerIsADeclaredConfiguration(t *testing.T) { } // The bare environment variable is gone here too. t.Setenv("HOST_ISSUER_URL", "https://exported.example.com") - if got := loadConfig(context.Background(), "wiki").mcpIssuerURL; got != declared { + if got := loadConfig(context.Background(), "wiki", nil).mcpIssuerURL; got != declared { t.Errorf("mcpIssuerURL = %q, want the declared %q — a bare HOST_ISSUER_URL must have no effect", got, declared) } } @@ -895,7 +1029,7 @@ func TestDeployedMCPRefusalsNameTheConfigurationThroughTheSDK(t *testing.T) { t.Setenv("PORT", "8080") t.Setenv("CODEFLY__RUNTIME_CONTEXT", "kubernetes") t.Setenv("GATEWAY_URL", "http://auth-gateway.saas.svc.cluster.local:8080") - cfg := loadConfig(context.Background(), "wiki") + cfg := loadConfig(context.Background(), "wiki", nil) cfg.mcp = true return cfg } @@ -924,9 +1058,11 @@ func TestDeployedMCPRefusalsNameTheConfigurationThroughTheSDK(t *testing.T) { }) t.Run("no issuer the composition declared", func(t *testing.T) { - cfg := deployed(t) + // For the deployed environment it sets, not the config it returns: + // PUBLIC_URL lands after it, so the config has to be loaded again. + deployed(t) t.Setenv("PUBLIC_URL", "https://app.example.com") - cfg = loadConfig(context.Background(), "wiki") + cfg := loadConfig(context.Background(), "wiki", nil) cfg.mcp = true if cfg.mcpIssuerExplicit { t.Fatal("an issuer nobody declared reads as explicit") @@ -943,17 +1079,839 @@ func TestDeployedMCPRefusalsNameTheConfigurationThroughTheSDK(t *testing.T) { } }) - t.Run("the one declared value is enough", func(t *testing.T) { + t.Run("a derived identifier is refused however well PUBLIC_URL resolved", func(t *testing.T) { + // This test asserted the opposite: that declaring the issuer alone was + // enough, because the identifier is "derived by construction" from + // PUBLIC_URL. The derivation builds /solutions//mcp, + // which is this runtime encoding the route the HOST serves it on — the + // same assumption the Module Federation manifest URL was changed to + // stop making in this cutover, for the same reason: the route a client + // reaches a solution through is the host's to resolve, and a runtime + // guessing it is right only on the machine of whoever wrote the guess. + // + // A derived value satisfied the old "a public URL is set" check, so the + // deployed refusal never fired in the one case that matters. declareMCPConfiguration(t, MCPIssuerURLKey, "https://login.example.com") - cfg := deployed(t) + deployed(t) t.Setenv("PUBLIC_URL", "https://app.example.com") - cfg = loadConfig(context.Background(), "wiki") + cfg := loadConfig(context.Background(), "wiki", nil) + cfg.mcp = true + err := cfg.validateMCP() + if err == nil { + t.Fatal("a deployed MCP surface was accepted with an identifier derived from PUBLIC_URL: a client binds its token to the URL it dialled, and this one is a guess about the host's route layout") + } + if !strings.Contains(err.Error(), mcpConfigurationValue(MCPPublicURLKey)) { + t.Errorf("the refusal %q does not name %s, the configuration that fixes it", err, mcpConfigurationValue(MCPPublicURLKey)) + } + }) + + t.Run("both declared values are enough", func(t *testing.T) { + declareMCPConfiguration(t, MCPIssuerURLKey, "https://login.example.com") + declareMCPConfiguration(t, MCPPublicURLKey, "https://app.example.com/wiki"+MCPPath) + deployed(t) + cfg := loadConfig(context.Background(), "wiki", nil) cfg.mcp = true if err := cfg.validateMCP(); err != nil { - t.Fatalf("a deployed MCP surface with the issuer declared and PUBLIC_URL resolved was refused: %v", err) + t.Fatalf("a deployed MCP surface with both values declared was refused: %v", err) + } + if want := "https://app.example.com/wiki" + MCPPath; cfg.mcpPublicURL != want { + t.Errorf("mcpPublicURL = %q, want the declared %q", cfg.mcpPublicURL, want) + } + }) + + t.Run("a published URL is held to what a dialled one is", func(t *testing.T) { + for _, tc := range []struct{ name, value, names string }{ + {"userinfo", "https://ops:s3cr3t@app.example.com/wiki" + MCPPath, "userinfo"}, + {"a query string", "https://app.example.com/wiki" + MCPPath + "?token=s3cr3t", "query"}, + {"a fragment", "https://app.example.com/wiki" + MCPPath + "#frag", "fragment"}, + {"plaintext", "http://app.example.com/wiki" + MCPPath, "https"}, + } { + t.Run(tc.name, func(t *testing.T) { + declareMCPConfiguration(t, MCPIssuerURLKey, "https://login.example.com") + declareMCPConfiguration(t, MCPPublicURLKey, tc.value) + deployed(t) + cfg := loadConfig(context.Background(), "wiki", nil) + cfg.mcp = true + err := cfg.validateMCP() + if err == nil { + t.Fatalf("a deployed MCP surface published %q: it is the identifier clients bind a token to and dial, so it is held to what the addresses this runtime dials are held to", tc.value) + } + if !strings.Contains(err.Error(), tc.names) { + t.Errorf("the refusal %q does not say what is wrong (%s)", err, tc.names) + } + if strings.Contains(err.Error(), "s3cr3t") { + t.Errorf("the refusal %q repeats the secret it is refusing", err) + } + }) + } + }) +} + +// TestAnMCPToolDoesNotRunWhileThisExecutionHoldsNoCredential is R4-3 on the +// MCP path, which the integration of main reopened. +// +// A plain handler and a ViewerBearer passthrough are both refused while the +// issuer has withdrawn this execution's credential — "fail closed, with no +// exception for a counterpart's current state". A tool call was not: executed +// round eight reproduced one being SERVED over mTLS with a real MCP client, +// with the viewer's bearer reaching the gateway. +// +// Refused at the HTTP boundary, which answers a real 503. An earlier version of +// this gated only at the method layer so that tools/list could still answer, +// on the grounds that refusing it would make a withdrawn credential look like +// a solution serving no MCP at all. That reasoning was weak: a 503 carrying +// the sanitized sentence says "temporarily cannot act", which is accurate, +// where a 404 would be the indistinguishable answer. And a client that +// connects and lists tools it can never call is worse served than one told to +// back off. The method-layer gate stays as well, for anything reaching the +// method middleware another way. +func TestAnMCPToolDoesNotRunWhileThisExecutionHoldsNoCredential(t *testing.T) { + for _, tc := range []struct { + name string + err error + }{ + {"a terminal refusal", workcontext.ErrMintRefused}, + {"a transient unavailability", workcontext.ErrMintUnavailable}, + } { + t.Run(tc.name, func(t *testing.T) { + var ran atomic.Bool + register := func(srv *mcp.Server) { + mcp.AddTool(srv, &mcp.Tool{Name: "act", Description: "must not run"}, + func(ctx context.Context, _ *mcp.CallToolRequest, _ askInput) (*mcp.CallToolResult, askOutput, error) { + ran.Store(true) + return nil, askOutput{Status: 200}, nil + }) + } + server := New(Manifest{ID: mcpServerName, Title: "Wiki"}). + ServeMCP(mcpServerName, "v1.2.3", register). + Credential(refusingSource{err: fmt.Errorf("%w: the issuer said so, at https://mint.internal.example/platform/_credential (dial tcp 10.4.1.9:8443: connection refused)", tc.err)}) + + handler, err := server.mcpHandler(MCPEnvironment{IssuerURL: testIssuer}) + if err != nil { + t.Fatalf("mount MCP: %v", err) + } + host := httptest.NewServer(handler) + t.Cleanup(host.Close) + + // A tools/call the way a client sends it, by hand: the SDK's + // client cannot complete its handshake against a surface that is + // refusing, which is the point. + body := `{"jsonrpc":"2.0","id":1,"method":"tools/call","params":{"name":"act","arguments":{"question":"anything"}}}` + req, err := http.NewRequest(http.MethodPost, host.URL+MCPPath, strings.NewReader(body)) + if err != nil { + t.Fatal(err) + } + req.Header.Set("content-type", "application/json") + stamp := viewerStamp() + req.Header.Set("authorization", stamp.bearer) + req.Header.Set(userHeader, stamp.userID) + req.Header.Set(orgHeader, stamp.orgID) + req.Header.Set(sessionHeader, stamp.sessionID) + req.Header.Set(workcontext.InstallationIDHeaderName, testInstallation) + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatalf("post a tool call: %v", err) + } + defer drainAndClose(resp) + + if resp.StatusCode != http.StatusServiceUnavailable { + t.Errorf("a tool call answered %d while this execution held no usable credential, want 503: the viewer's bearer would otherwise reach author code under authority this process does not have", resp.StatusCode) + } + if ran.Load() { + t.Error("the tool ran: the gate has to refuse before anything author-written executes") + } + said, err := io.ReadAll(resp.Body) + if err != nil { + t.Fatal(err) + } + for _, leak := range []string{"mint.internal.example", "_credential", "10.4.1.9", "connection refused"} { + if strings.Contains(string(said), leak) { + t.Errorf("the refusal discloses %q to an MCP client: %s", leak, said) + } + } + }) + } + + t.Run("the refusal is 503 and not 404", func(t *testing.T) { + // A withdrawn credential must not read as "this solution serves no + // MCP": one is a back-off, the other sends a client looking for an + // endpoint that does not exist. + server := New(Manifest{ID: mcpServerName, Title: "Wiki"}). + ServeMCP(mcpServerName, "v1.2.3", readCollectionTool). + Credential(refusingSource{err: workcontext.ErrMintRefused}) + handler, err := server.mcpHandler(MCPEnvironment{IssuerURL: testIssuer}) + if err != nil { + t.Fatalf("mount MCP: %v", err) + } + host := httptest.NewServer(handler) + t.Cleanup(host.Close) + + req, err := http.NewRequest(http.MethodPost, host.URL+MCPPath, strings.NewReader(`{"jsonrpc":"2.0","id":1,"method":"initialize","params":{}}`)) + if err != nil { + t.Fatal(err) + } + req.Header.Set("content-type", "application/json") + stamp := viewerStamp() + req.Header.Set("authorization", stamp.bearer) + req.Header.Set(orgHeader, stamp.orgID) + req.Header.Set(sessionHeader, stamp.sessionID) + req.Header.Set(workcontext.InstallationIDHeaderName, testInstallation) + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + defer drainAndClose(resp) + if resp.StatusCode == http.StatusNotFound { + t.Error("a withdrawn credential answered 404: an MCP client cannot tell that from a solution that serves no MCP at all") + } + if resp.StatusCode != http.StatusServiceUnavailable { + t.Errorf("status = %d, want 503", resp.StatusCode) + } + }) + + t.Run("every method that is not allowlisted is gated at the method layer too", func(t *testing.T) { + // The method-layer gate's property, not an enumeration: a method this + // file has never heard of is held to the credential. A denylist over + // the SDK's method set is the mistake the TLS posture made three times. + for _, method := range []string{"tools/call", "resources/read", "prompts/get", "completion/complete", "some/method-added-next-release", + // The namespace that used to be exempted by prefix. It is open — + // the SDK's extension API lets author code register under it — so + // a method nobody here named must be gated, not waved through for + // looking like a notification. + "notifications/anything-an-author-registers", "notifications/tools/call"} { + if actsForNobody(method) { + t.Errorf("%q is treated as acting for nobody, so a viewer action on it would skip the credential gate", method) + } + } + for _, method := range []string{"initialize", "ping", "tools/list", "prompts/list", "resources/list", + "notifications/initialized", "notifications/cancelled", "notifications/progress", "notifications/roots/list_changed"} { + if !actsForNobody(method) { + t.Errorf("%q is gated at the method layer though it runs nothing for a viewer", method) + } + } + }) +} + +// TestAToolsOwnFailureIsSanitizedOnTheWayOut is the other half of R8-1, and +// the half my first answer to it did not test. +// +// The gate refuses a tool when this execution holds no credential. This is the +// case the gate lets through: the credential is fine when the call starts and +// the viewer's mint fails during it. A tool that called ForModule and was +// refused handed the agent client the whole chain — "mint returned HTTP 503", +// and with the issuer unreachable the mint's own URL and dial address. +// +// A tool's error is written for a model and read by whoever is driving it, so +// it is sanitized by the same rule a handler's is. The mutation that relays +// err.Error() verbatim survived until this test existed, because the gate test +// refuses before any tool runs and never reaches the out-path. +func TestAToolsOwnFailureIsSanitizedOnTheWayOut(t *testing.T) { + // A gateway whose mint refuses, and a workload credential that is + // perfectly good — so the gate passes and the failure happens inside the + // tool. + gw := newWorkContextGateway(t, &workContextGateway{mintStatus: http.StatusServiceUnavailable}) + register := func(srv *mcp.Server) { + mcp.AddTool(srv, &mcp.Tool{Name: "read", Description: "reads a module as the viewer"}, + func(ctx context.Context, _ *mcp.CallToolRequest, _ askInput) (*mcp.CallToolResult, askOutput, error) { + viewer, err := ViewerFromContext(ctx) + if err != nil { + return nil, askOutput{}, err + } + // The real shape: a tool minting for the viewer, refused. + if _, err := viewer.ForModule(ctx, "documents", Scope{ResourceKind: "documents", Actions: []string{"read"}}); err != nil { + return nil, askOutput{}, err + } + return nil, askOutput{Status: 200}, nil + }) + } + server := New(Manifest{ID: mcpServerName, Title: "Wiki"}). + ServeMCP(mcpServerName, "v1.2.3", register). + Credential(attestingSource(t)) + server.cfg.gatewayURL = gw.URL + + handler, err := server.mcpHandler(MCPEnvironment{GatewayURL: gw.URL, IssuerURL: testIssuer}) + if err != nil { + t.Fatalf("mount MCP: %v", err) + } + host := httptest.NewServer(handler) + t.Cleanup(host.Close) + session := connectMCP(t, host, viewerStamp()) + + result, err := session.CallTool(context.Background(), &mcp.CallToolParams{ + Name: "read", + Arguments: map[string]any{"question": "anything"}, + }) + // Either shape is a failure the client sees; what matters is what it says. + said := "" + switch { + case err != nil: + said = err.Error() + case result != nil && result.IsError: + said = toolErrorText(result) + default: + t.Fatal("a tool whose viewer mint was refused answered as though it had succeeded") + } + for _, leak := range []string{gw.URL, "StartTask", "503", "mint returned"} { + if strings.Contains(said, leak) { + t.Errorf("the tool's failure discloses %q to the MCP client: %s", leak, said) + } + } + if said == "" { + t.Error("the client was told nothing at all") + } +} + +// TestTheMethodLayerGateRefusesOnItsOwn exercises the MCP method middleware +// directly, without the HTTP boundary in front of it. +// +// Both gates refuse a viewer action while this execution holds no credential, +// and after the HTTP one was added the method one had nothing holding it: the +// mutation that removes it survived, because every test reaching a tool goes +// through requireStampedViewer first and is refused there. Adding the better +// check orphaned the one behind it. +// +// It is not redundant. It is what holds a method dispatched without passing +// through that handler — and it is the layer that knows the method name, so it +// is where the allowlist lives. +func TestTheMethodLayerGateRefusesOnItsOwn(t *testing.T) { + server := New(Manifest{ID: mcpServerName, Title: "Wiki"}). + ServeMCP(mcpServerName, "v1.2.3", readCollectionTool). + Credential(refusingSource{err: fmt.Errorf("%w: at https://mint.internal.example/platform/_credential", workcontext.ErrMintUnavailable)}) + + var reached atomic.Bool + next := func(context.Context, string, mcp.Request) (mcp.Result, error) { + reached.Store(true) + return nil, nil + } + gated := server.mcpViewer(next) + + t.Run("a method that acts for a viewer is refused", func(t *testing.T) { + reached.Store(false) + _, err := gated(context.Background(), "tools/call", &mcp.CallToolRequest{}) + if err == nil { + t.Fatal("the method middleware admitted a tool call while this execution held no usable credential: it is what holds a method dispatched without the HTTP handler in front of it") + } + if reached.Load() { + t.Error("the call reached the next handler, so nothing was gated") + } + for _, leak := range []string{"mint.internal.example", "_credential"} { + if strings.Contains(err.Error(), leak) { + t.Errorf("the refusal discloses %q: %s", leak, err) + } + } + // And it carries the CODE, which is the whole point of item 8: a + // client that gets a generic protocol error cannot tell "this solution + // cannot act right now" from a broken tool, so it cannot back off. + // The mutation that drops the typed error survived until this. + var wire *jsonrpc.Error + if !errors.As(err, &wire) { + t.Fatalf("the refusal is %T, not a *jsonrpc.Error: without a code an MCP client cannot distinguish this from any other server error", err) + } + if wire.Code != mcpCredentialUnavailableCode { + t.Errorf("the refusal carries code %d, want %d", wire.Code, mcpCredentialUnavailableCode) } - if want := "https://app.example.com/solutions/wiki" + MCPPath; cfg.mcpPublicURL != want { - t.Errorf("mcpPublicURL = %q, want the derived %q", cfg.mcpPublicURL, want) + }) + + t.Run("a method that acts for nobody passes", func(t *testing.T) { + reached.Store(false) + if _, err := gated(context.Background(), "tools/list", &mcp.ListToolsRequest{}); err != nil { + t.Errorf("the method middleware refused tools/list, which runs nothing for a viewer: %v", err) + } + if !reached.Load() { + t.Error("tools/list never reached the next handler") + } + }) +} + +// TestNoMCPRefusalEverPrintsASecret is item 4, as a property rather than an +// audit. +// +// Every refusal that quotes a URL goes through redactedURL, which drops +// userinfo, the query and the fragment — url.Redacted() masks the password and +// keeps the username, which is how a great many tokens are carried. I had +// redacted two paths and missed six, so this drives every refusal validateMCP +// can produce with a secret in each position and asserts none of them comes +// back out. +func TestNoMCPRefusalEverPrintsASecret(t *testing.T) { + const secret = "s3cr3tvalue" + // Every shape that reaches a refusal: unresolved, in-cluster, derived, + // unpairable, plaintext, and a secret in each of the three positions a URL + // can hide one. + withSecret := []string{ + "https://ops:" + secret + "@app.example.com/wiki" + MCPPath, + "https://app.example.com/wiki" + MCPPath + "?token=" + secret, + "https://app.example.com/wiki" + MCPPath + "#" + secret, + "http://app.example.com/wiki" + MCPPath + "?token=" + secret, + "https://" + secret + ":x@app.example.com/not-the-suffix", + "not-a-url-at-all?token=" + secret, + // Scheme-less and opaque forms. url.Parse accepts + // "ops:SECRET@host/path" as scheme "ops" with everything after the + // colon opaque, so User is nil and RawQuery empty and the ordinary + // strips touch nothing — the secret came through redactedURL intact. + "ops:" + secret + "@app.example.com/wiki" + MCPPath, + "mailto:ops:" + secret + "@app.example.com", + "app.example.com:8443/wiki" + MCPPath + "?token=" + secret, + "https://app.example.com/wiki?token=" + secret, + } + for _, runtimeContext := range []string{"", "kubernetes"} { + for _, value := range withSecret { + for _, which := range []string{"issuer", "public"} { + cfg := config{mcp: true, runtimeContext: runtimeContext, + mcpIssuerURL: "https://login.example.com", mcpIssuerExplicit: true, + mcpPublicURL: "https://app.example.com/wiki" + MCPPath, mcpPublicExplicit: true} + switch which { + case "issuer": + cfg.mcpIssuerURL = value + case "public": + cfg.mcpPublicURL = value + } + err := cfg.validateMCP() + if err == nil { + continue // accepted; nothing was printed + } + if strings.Contains(err.Error(), secret) { + t.Errorf("a refusal printed the secret in %s=%q (runtime context %q): %v", + which, value, runtimeContext, err) + } + } + } + } +} + +// A transport failure through gw.HTTPClient() is deliberately NOT sanitized, +// and a test asserting otherwise was deleted rather than weakened. +// +// Two reasons, found by writing it. http.Client.Do wraps whatever the +// transport returns in its own *url.Error carrying the URL, so a transport +// wrapper cannot remove the address. And the author built that request from +// BaseURL(), so the address is already theirs — the error discloses nothing +// they did not have. What the runtime owes them is not handing over what they +// could not otherwise see, which is Gateway.surfaced's job and is covered by +// TestAToolsOwnFailureIsSanitizedOnTheWayOut. + +// TestNoExportedGatewayMethodRelaysTheRuntimesError is round eleven's item 3, +// and it is a gate rather than one more case. +// +// ForModule's mint errors were sanitized and its exported SIBLING's were not: +// WorkContextPrincipals reaches the same mint, produced the same errors — the +// status, the issuer's code, its message — and handed them straight back. A +// tool relayed the whole chain to an agent client. Two callers of one mint, +// one of them sanitized. +// +// So this drives every exported Gateway method that can return a runtime +// error, on a gateway that asked for sanitizing, and asserts none of them +// carries the issuer's words. A method added later is one line here, and the +// one-by-one version of this test is how the second path was missed. +func TestNoExportedGatewayMethodRelaysTheRuntimesError(t *testing.T) { + gw := newWorkContextGateway(t, &workContextGateway{}) + server := New(Manifest{ID: testSolutionID}).Credential(attestingSource(t)) + server.cfg = config{gatewayURL: gw.URL} + + header := http.Header{} + header.Set("authorization", viewerBearer()) + header.Set(orgHeader, viewerOrg) + header.Set(sessionHeader, viewerSession) + header.Set(workcontext.InstallationIDHeaderName, testInstallation) + + viewer := server.gatewayFor(header) + // The MCP path, where a tool's error becomes content a client reads and + // nothing downstream can tell it from the author's own. + viewer.sanitizeErrors = true + + // A derived gateway first, so the delegation exists; then the host starts + // refusing and the held capability lapses, which is what a tool holding a + // gateway hits mid-call. + acting, err := viewer.ForModule(context.Background(), "documents", Scope{ResourceKind: "documents", Actions: []string{"read"}}) + if err != nil { + t.Fatalf("the first derive failed, so nothing below is about sanitizing: %v", err) + } + gw.mintStatus = http.StatusServiceUnavailable + lapseCachedCapabilities(acting) + + for _, probe := range []struct { + name string + call func() error + }{ + {"ForModule", func() error { + _, err := viewer.ForModule(context.Background(), "documents", Scope{ResourceKind: "documents", Actions: []string{"read"}}) + return err + }}, + {"WorkContextPrincipals", func() error { + _, err := acting.WorkContextPrincipals(context.Background()) + return err + }}, + } { + t.Run(probe.name, func(t *testing.T) { + err := probe.call() + if err == nil { + t.Fatal("the refusing host was not reached, so this probe asserted nothing") + } + // The issuer's own words, the status it answered with, and the + // address this runtime dialled. + for _, leak := range []string{gw.URL, "StartTask", "503", "permission_denied", "no such authority", "rejected"} { + if strings.Contains(err.Error(), leak) { + t.Errorf("%s relays %q to whoever the author hands this to: %v", probe.name, leak, err) + } + } + }) + } +} + +// TestAToolResultNeverCarriesTheRuntimesInternals is B1: the path where the +// middleware sees no error at all. +// +// The SDK's typed AddTool wrapper turns a tool's returned error into +// CallToolResult{IsError: true} and returns a NIL error, so the middleware's +// error branch never runs and whatever the tool was handed travels as content. +// The shape that exposed it: a tool derives a gateway, the held capability +// lapses, and the re-mint happens INSIDE the round trip — so the refusal +// carries the issuer's status, its code and its words, and http.Client.Do +// prepends the address it dialled. +// +// Measured before the fix, the client was told: +// +// Get "http://127.0.0.1:50122/v1/documents/collection": work context mint +// for "documents" rejected (status 503, "permission_denied"): caller holds +// no such authority +// +// Two layers answer it, because one was not enough. bearerTransport sanitizes +// the mint error where it leaves this runtime, which removes the issuer's +// words; and the middleware replaces an error result that still names an +// address this runtime dialled, which is what net/http added after the author +// had no further say. +// +// My earlier reasoning is what let this through: I argued a transport error +// only names an address the author already holds from BaseURL(), which is true +// of a dial failure and false of a renewal that happens inside the transport. +func TestAToolResultNeverCarriesTheRuntimesInternals(t *testing.T) { + gw := newWorkContextGateway(t, &workContextGateway{}) + register := func(srv *mcp.Server) { + mcp.AddTool(srv, &mcp.Tool{Name: "read", Description: "reads a module as the viewer"}, + func(ctx context.Context, _ *mcp.CallToolRequest, _ askInput) (*mcp.CallToolResult, askOutput, error) { + viewer, err := ViewerFromContext(ctx) + if err != nil { + return nil, askOutput{}, err + } + acting, err := viewer.ForModule(ctx, "documents", Scope{ResourceKind: "documents", Actions: []string{"read"}}) + if err != nil { + return nil, askOutput{}, err + } + // The host starts refusing and the capability lapses, so the + // re-mint lands inside the round trip below. + gw.mintStatus = http.StatusServiceUnavailable + lapseCachedCapabilities(acting) + request, err := http.NewRequestWithContext(ctx, http.MethodGet, acting.BaseURL()+modulePath, nil) + if err != nil { + return nil, askOutput{}, err + } + resp, err := acting.HTTPClient().Do(request) + if err != nil { + // Relayed verbatim, which is what an author does. + return nil, askOutput{}, err + } + drainAndClose(resp) + return nil, askOutput{Status: resp.StatusCode}, nil + }) + } + server := New(Manifest{ID: mcpServerName, Title: "Wiki"}). + ServeMCP(mcpServerName, "v1.2.3", register). + Credential(attestingSource(t)) + server.cfg.gatewayURL = gw.URL + + handler, err := server.mcpHandler(MCPEnvironment{GatewayURL: gw.URL, IssuerURL: testIssuer}) + if err != nil { + t.Fatalf("mount MCP: %v", err) + } + host := httptest.NewServer(handler) + t.Cleanup(host.Close) + session := connectMCP(t, host, viewerStamp()) + + result, callErr := session.CallTool(context.Background(), &mcp.CallToolParams{ + Name: "read", + Arguments: map[string]any{"question": "anything"}, + }) + said := "" + switch { + case callErr != nil: + said = callErr.Error() + case result != nil && result.IsError: + said = toolErrorText(result) + default: + t.Fatal("the tool reported success though its module read could not be authenticated") + } + for _, leak := range []string{gw.URL, "127.0.0.1", modulePath, "StartTask", "503", "permission_denied", "no such authority", "work context mint"} { + if strings.Contains(said, leak) { + t.Errorf("the tool result discloses %q to the MCP client: %s", leak, said) + } + } + if said == "" { + t.Error("the client was told nothing at all") + } +} + +// TestAToolsOwnRefusalIsLeftAlone is the other half of B1's fix: the net must +// not rewrite what an author actually wrote. +// +// Replacing every error result would take a tool's own vocabulary away, and a +// model driving it reads that text. The replacement fires only when the +// content names an address this runtime dialled. +func TestAToolsOwnRefusalIsLeftAlone(t *testing.T) { + gw := newWorkContextGateway(t, &workContextGateway{}) + register := func(srv *mcp.Server) { + mcp.AddTool(srv, &mcp.Tool{Name: "ask", Description: "refuses in its own words"}, + func(context.Context, *mcp.CallToolRequest, askInput) (*mcp.CallToolResult, askOutput, error) { + return nil, askOutput{}, errors.New("no such collection in this wiki") + }) + } + server := New(Manifest{ID: mcpServerName, Title: "Wiki"}). + ServeMCP(mcpServerName, "v1.2.3", register). + Credential(attestingSource(t)) + server.cfg.gatewayURL = gw.URL + handler, err := server.mcpHandler(MCPEnvironment{GatewayURL: gw.URL, IssuerURL: testIssuer}) + if err != nil { + t.Fatalf("mount MCP: %v", err) + } + host := httptest.NewServer(handler) + t.Cleanup(host.Close) + session := connectMCP(t, host, viewerStamp()) + + result, err := session.CallTool(context.Background(), &mcp.CallToolParams{ + Name: "ask", + Arguments: map[string]any{"question": "anything"}, + }) + if err != nil { + t.Fatalf("tools/call: %v", err) + } + if !strings.Contains(toolErrorText(result), "no such collection in this wiki") { + t.Errorf("the tool's own refusal was rewritten: %q — a tool's vocabulary is its own, and a model driving it reads that text", toolErrorText(result)) + } +} + +// TestAStructuredToolResultNeverCarriesTheRuntimesInternals is round fifteen's +// minor, and it narrows a closure claim I made. +// +// withoutRuntimeAddresses read only Content. The pinned SDK serializes +// StructuredContent too, so a result with generic text and +// StructuredContent{"error": err.Error()} reached the agent client with the +// mint's address in it. The net was installed for exactly that disclosure and +// was looking in one of the two places the SDK sends. +func TestAStructuredToolResultNeverCarriesTheRuntimesInternals(t *testing.T) { + server := New(Manifest{ID: testSolutionID}) + server.cfg = config{ + gatewayURL: "https://gateway.internal.test", + mintURL: "https://mint.internal.test/platform/_credential", + } + server.cfg.dialled = dialledAddresses(server.cfg.gatewayURL, server.cfg.mintURL) + + t.Run("the address is in the structured content alone", func(t *testing.T) { + result := server.withoutRuntimeAddresses(&mcp.CallToolResult{ + IsError: true, + Content: []mcp.Content{&mcp.TextContent{Text: "the call did not complete"}}, + StructuredContent: map[string]any{ + "error": `Post "https://mint.internal.test/platform/_credential": dial tcp 127.0.0.1:1 connect: connection refused`, + }, + }, "tools/call") + call, ok := result.(*mcp.CallToolResult) + if !ok { + t.Fatalf("the filter returned %T", result) + } + if call.StructuredContent != nil { + t.Errorf("the replaced result still carries structured content: %v", call.StructuredContent) + } + rendered, err := json.Marshal(call) + if err != nil { + t.Fatal(err) + } + for _, secret := range []string{"mint.internal.test", "gateway.internal.test"} { + if strings.Contains(string(rendered), secret) { + t.Errorf("the serialized result names %q, which is an address this runtime dialled and nothing a viewer's agent client may be told: %s", secret, rendered) + } + } + }) + + t.Run("a tool's own structured refusal is left alone", func(t *testing.T) { + // The net matches on exact configured values, so an author's own + // structured error about their own domain is untouched. + own := &mcp.CallToolResult{ + IsError: true, + Content: []mcp.Content{&mcp.TextContent{Text: "the wiki rejected that page title"}}, + StructuredContent: map[string]any{"error": "title must not be empty", "field": "title"}, + } + // The CONTENT survives, which is what "left alone" means. Pointer + // identity used to stand in for it and no longer can: the filter + // returns the bytes it checked, so what a client reads is exactly + // what was inspected rather than a re-serialization of a value this + // runtime does not own. + got, ok := server.withoutRuntimeAddresses(own, "tools/call").(*mcp.CallToolResult) + if !ok { + t.Fatalf("the filter returned %T", got) + } + if !got.IsError || len(got.Content) != 1 { + t.Fatalf("the filter reshaped a tool's own refusal: %+v", got) + } + if text, isText := got.Content[0].(*mcp.TextContent); !isText || text.Text != "the wiki rejected that page title" { + t.Errorf("the filter replaced a tool's own message: %+v", got.Content[0]) + } + rendered, err := json.Marshal(got.StructuredContent) + if err != nil { + t.Fatal(err) + } + for _, want := range []string{"title must not be empty", "title"} { + if !strings.Contains(string(rendered), want) { + t.Errorf("a tool's own structured refusal lost %q: %s", want, rendered) + } } }) + + t.Run("the address is JSON-escaped, which is the same string to a client", func(t *testing.T) { + // Round sixteen's escape: json.RawMessage keeps `https:\/\/host` + // through Marshal, so a search over the serialized bytes for + // "https://host" misses — and the client decodes the address straight + // back out. The decoded strings are what get checked now. + escaped := json.RawMessage(`{"error":"https:\/\/mint.internal.test\/platform\/_credential failed"}`) + result := server.withoutRuntimeAddresses(&mcp.CallToolResult{ + IsError: true, + Content: []mcp.Content{&mcp.TextContent{Text: "the call did not complete"}}, + StructuredContent: escaped, + }, "tools/call") + call, ok := result.(*mcp.CallToolResult) + if !ok { + t.Fatalf("the filter returned %T", result) + } + if call.StructuredContent != nil { + t.Errorf("the escaped address survived the filter: %v", call.StructuredContent) + } + // And what a client would actually read must not contain it either. + rendered, err := json.Marshal(call) + if err != nil { + t.Fatal(err) + } + var asAClientSeesIt any + if err := json.Unmarshal(rendered, &asAClientSeesIt); err != nil { + t.Fatal(err) + } + again, _ := json.Marshal(asAClientSeesIt) + if strings.Contains(string(again), "mint.internal.test") { + t.Errorf("a client decoding this result reads the mint's address: %s", again) + } + }) + + t.Run("an address used as a structured KEY is caught too", func(t *testing.T) { + result := server.withoutRuntimeAddresses(&mcp.CallToolResult{ + IsError: true, + Content: []mcp.Content{&mcp.TextContent{Text: "the call did not complete"}}, + StructuredContent: map[string]any{"https://mint.internal.test/platform/_credential": "refused"}, + }, "tools/call") + if call, ok := result.(*mcp.CallToolResult); !ok || call.StructuredContent != nil { + t.Errorf("an address in a structured KEY survived the filter: %+v", result) + } + }) + + t.Run("a marshaler that answers differently the second time cannot disclose", func(t *testing.T) { + // The confirming round's finding: the filter inspected ONE + // serialization and returned the original object, which the SDK then + // serialized again. A MarshalJSON that answers `{}` first and the + // mint's address afterwards passed the check and disclosed on the + // second call. No concurrency needed — marshalling is not a pure + // function of a value this runtime did not write. + shifty := &shiftingMarshaler{later: `{"error":"https://mint.internal.test/platform/_credential"}`} + result := server.withoutRuntimeAddresses(&mcp.CallToolResult{ + IsError: true, + Content: []mcp.Content{&mcp.TextContent{Text: "the call did not complete"}}, + StructuredContent: shifty, + }, "tools/call") + // What a client reads is the SDK's serialization of what came back. + published, err := json.Marshal(result) + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(published), "mint.internal.test") { + t.Errorf("the published result names the mint's address, which the filter had approved on an earlier serialization: %s", published) + } + if shifty.calls > 1 { + t.Errorf("the value was marshalled %d times, so the bytes that were checked are not the bytes that were published", shifty.calls) + } + }) + + t.Run("a marshaler that rewrites sibling text cannot disclose", func(t *testing.T) { + // The confirming round's minor, and the sharper half of it: the text + // was inspected BEFORE the structured content was marshalled, so a + // structured value holding a pointer to the result's own TextContent + // could rewrite the text from inside MarshalJSON — after it had + // passed inspection — and serialize as a clean `{}` itself. No second + // marshaller call and no concurrency. + text := &mcp.TextContent{Text: "the call did not complete"} + result := server.withoutRuntimeAddresses(&mcp.CallToolResult{ + IsError: true, + Content: []mcp.Content{text}, + StructuredContent: sabotagingMarshaler{text: text, to: "https://mint.internal.test/platform/_credential"}, + }, "tools/call") + published, err := json.Marshal(result) + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(published), "mint.internal.test") { + t.Errorf("the published result names the mint's address: a marshaler rewrote text that had already passed inspection: %s", published) + } + }) + + t.Run("content this runtime cannot read is replaced", func(t *testing.T) { + // Fail closed: content whose serialization nothing here inspected is + // not published. + result := server.withoutRuntimeAddresses(&mcp.CallToolResult{ + IsError: true, + Content: []mcp.Content{&mcp.ImageContent{Data: []byte("x"), MIMEType: "image/png"}}, + }, "tools/call") + call, ok := result.(*mcp.CallToolResult) + if !ok { + t.Fatalf("the filter returned %T", result) + } + if len(call.Content) != 1 { + t.Fatalf("want the single replacement message, got %+v", call.Content) + } + if text, isText := call.Content[0].(*mcp.TextContent); !isText || !strings.Contains(text.Text, "could not complete") { + t.Errorf("content the filter cannot inspect was published as-is: %+v", call.Content[0]) + } + }) + + t.Run("structured content this runtime cannot read is replaced", func(t *testing.T) { + // It cannot be cleared if it cannot be read, and the SDK is about to + // serialize it either way. + result := server.withoutRuntimeAddresses(&mcp.CallToolResult{ + IsError: true, + Content: []mcp.Content{&mcp.TextContent{Text: "the call did not complete"}}, + StructuredContent: map[string]any{"error": func() {}}, + }, "tools/call") + call, ok := result.(*mcp.CallToolResult) + if !ok || call.StructuredContent != nil { + t.Errorf("unreadable structured content survived the filter: %+v", result) + } + }) +} + +// shiftingMarshaler answers `{}` the first time and something else after, which +// is the shape that defeats checking one serialization and publishing another. +type shiftingMarshaler struct { + calls int + later string +} + +func (s *shiftingMarshaler) MarshalJSON() ([]byte, error) { + s.calls++ + if s.calls == 1 { + return []byte(`{}`), nil + } + return []byte(s.later), nil +} + +// sabotagingMarshaler serializes as `{}` and, as a side effect, rewrites a +// sibling TextContent the result still carries. +type sabotagingMarshaler struct { + text *mcp.TextContent + to string +} + +func (v sabotagingMarshaler) MarshalJSON() ([]byte, error) { + v.text.Text = v.to + return []byte(`{}`), nil } diff --git a/operations.go b/operations.go index 1a1a326..e15a779 100644 --- a/operations.go +++ b/operations.go @@ -86,8 +86,8 @@ type InterfaceInfo struct { // "/solutions/": the backend serves them at its own root, but a caller // only ever reaches them prefixed. BasePath string - // OperationPrefix starts every operationId ("Wiki" renders "/sources" as - // "Wiki_Sources"); Tag is the one tag every operation carries. + // OperationPrefix starts every operationId ("Notes" renders "/sources" as + // "Notes_Sources"); Tag is the one tag every operation carries. OperationPrefix string Tag string } diff --git a/operations_test.go b/operations_test.go index e30aa5f..d3cde86 100644 --- a/operations_test.go +++ b/operations_test.go @@ -110,7 +110,7 @@ func TestOperationsServesEachDeclaredRoute(t *testing.T) { if err != nil { t.Fatal(err) } - resp.Body.Close() + _ = resp.Body.Close() if resp.StatusCode != http.StatusOK { t.Fatalf("status = %d", resp.StatusCode) } diff --git a/passthrough.go b/passthrough.go index 6035b94..354d8a1 100644 --- a/passthrough.go +++ b/passthrough.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "fmt" + "log" "net/http" "os" "path/filepath" @@ -14,6 +15,8 @@ import ( "connectrpc.com/connect" "github.com/codefly-dev/core/solution/manifest" + "github.com/codefly-dev/sdk-go/workcontext" + "github.com/codefly-dev/solution-runtime-go/internal/seam" "google.golang.org/protobuf/encoding/protojson" "google.golang.org/protobuf/proto" "google.golang.org/protobuf/reflect/protoreflect" @@ -142,7 +145,24 @@ const ( // chainable and may be called more than once; Serve refuses a declaration that // cannot be served before it listens. func (s *Server) Consumes(modules ...ConsumedModule) *Server { - s.consumed = append(s.consumed, modules...) + // Frozen at the declaration boundary, for the reason Contract() gives: + // `append` copies the structs and leaves every slice inside them pointing + // at the caller's arrays. These are the ASKED scopes — the other operand + // of the ceiling comparison and what the published contract reports — so a + // caller editing one after boot changes what this process claims to ask + // for. + for _, module := range modules { + module.Scopes = frozenScopes(module.Scopes) + methods := make([]ConsumedMethod, len(module.Methods)) + for i, method := range module.Methods { + method.Scopes = frozenScopes(method.Scopes) + methods[i] = method + } + if module.Methods != nil { + module.Methods = methods + } + s.consumed = append(s.consumed, module) + } return s } @@ -286,11 +306,14 @@ func (s *Server) validatePassthrough() (map[string]passthroughRoute, error) { return routes, checkConsumed(s.consumed, consumed) } -// mountPassthrough mounts the declared routes on mux at PassthroughPathPrefix, -// as serve does and as PassthroughHandler does: the one place the passthrough -// is wired, so a test through PassthroughHandler serves what Serve serves. -// Serve has already resolved and checked the routes; a test that calls serve -// directly has not, and resolves them here. +// mountPassthrough mounts the declared routes on mux at PassthroughPathPrefix: +// the one place the passthrough is wired, so the test seam serves what Serve +// serves. Serve has already resolved and checked the routes; a test that calls +// serve directly has not, and resolves them here. +// +// It named Server.PassthroughHandler as the other caller, which was removed — +// an exported constructor for a credential-bearing handler with no boot behind +// it. The seam reached through internal/seam replaced it. func (s *Server) mountPassthrough(mux *http.ServeMux) error { if len(s.consumed) == 0 { return nil @@ -306,35 +329,40 @@ func (s *Server) mountPassthrough(mux *http.ServeMux) error { return nil } -// PassthroughEnvironment is what Serve resolves from the composition for the -// passthrough, supplied instead by a caller of PassthroughHandler. -type PassthroughEnvironment struct { - // GatewayURL is the host gateway every call goes through: the Work Context - // mint and the consumed modules' /v1/ prefixes. - GatewayURL string - // Consumes is the solution's api.consumes projection, which Serve reads - // from CODEFLY__API_CONSUMES. Every declared module must be in it. - Consumes []manifest.ConsumedAPI +// init registers the passthrough seam for package passthroughtest, which is the +// only caller that can reach it: internal/seam is importable inside this module +// and nowhere else. +func init() { + seam.Passthrough = passthroughSeam } -// PassthroughHandler is the consumed-module passthrough exactly as Serve -// serves it — the declaration checked by the same boot check, mounted by the -// same code at PassthroughPathPrefix — against an environment the caller -// supplies instead of the one Serve resolves. Nothing else of the solution is -// served, and nothing registers anywhere. +// passthroughSeam is what Server.PassthroughHandler used to be, minus the +// "exported" part. +// +// Exported, it was a production bypass: a solution could build a usable +// credential-bearing handler that had skipped validate(), the mTLS boot, the +// caller allow-list, the published ceiling and authenticated outbound, and a +// deployment calling it completed a viewer mint and a module call over +// plaintext, 200, with the viewer's bearer and this workload's own credential +// on the wire. Supplying a credential source — which a real deployment does — +// defeated the "no source, nothing to mint with" mitigation it relied on. Fail +// closed admits no exception for a shape that exists to make testing +// convenient, so the shape moved behind the compiler instead: see +// internal/seam. // -// It exists for tests: package passthroughtest builds on it, with a fake host -// standing in for the gateway. A solution serves with Serve, which overwrites -// the environment set here when it resolves its own. -func (s *Server) PassthroughHandler(env PassthroughEnvironment) (http.Handler, error) { +// What it builds is otherwise unchanged, and deliberately so — it is the +// declaration checked by the same boot check and mounted by the same code at +// PassthroughPathPrefix, so a consumer's test serves what Serve serves. Serve +// overwrites the environment set here when it resolves its own. +func passthroughSeam(server any, gatewayURL, consumesJSON string) (http.Handler, error) { + s, ok := server.(*Server) + if !ok { + return nil, fmt.Errorf("the passthrough seam was handed a %T rather than a solution server", server) + } if len(s.consumed) == 0 { return nil, fmt.Errorf("solution %q declares no consumed modules (Consumes)", s.manifest.ID) } - projection, err := json.Marshal(env.Consumes) - if err != nil { - return nil, err - } - s.cfg.gatewayURL, s.cfg.apiConsumes = env.GatewayURL, string(projection) + s.cfg.gatewayURL, s.cfg.apiConsumes = gatewayURL, consumesJSON routes, err := s.validatePassthrough() if err != nil { return nil, fmt.Errorf("solution %q: %w", s.manifest.ID, err) @@ -450,7 +478,23 @@ func (s *Server) forward(ctx context.Context, route passthroughRoute, req *conne // this method (or the bearer alone, for a ViewerBearer module), and the // declared pin merged into the request. func (s *Server) authorize(ctx context.Context, route passthroughRoute, header http.Header, msg *dynamicpb.Message) (*Gateway, error) { - gw := newGateway(s.cfg.gatewayURL, header.Get("authorization"), header.Get(orgHeader), header.Get(sessionHeader)) + // Whether or not this route mints anything. A ViewerBearer route forwards + // the viewer's bearer and asks the credential for nothing, so it was the + // one route that never consulted it — and it kept forwarding that bearer + // to the gateway after the issuer refused this build. See + // actingForAViewer: the credential is what authorises this process to act + // for a viewer, not merely what it mints with. + if err := s.actingForAViewer(ctx); err != nil { + // Sanitized, for the reason wrapRequest records: the error wraps what + // the source said, and this branch handed it to the page verbatim. + code := connect.CodeUnavailable + if errors.Is(err, ErrCredentialRefused) { + code = connect.CodeFailedPrecondition + } + log.Printf("solution: refusing to act for a viewer on %s: %v", route.module.As, err) + return nil, connect.NewError(code, errors.New(credentialRefusalForAViewer(err))) + } + gw := s.gatewayFor(header) if !route.module.ViewerBearer { acting, err := gw.ForModule(ctx, route.module.As, route.scopes()...) if err != nil { @@ -532,6 +576,42 @@ func relayedError(err error) error { return connect.NewError(connect.CodeDeadlineExceeded, errors.New("the module did not answer in time")) case errors.Is(err, context.Canceled): return connect.NewError(connect.CodeCanceled, errors.New("the call was canceled")) + case errors.Is(err, workcontext.ErrRevoked): + // Checked before ErrNotAttested, which now wraps it: a renewal the + // issuer refuses because the state the credential is sealed to has + // moved arrives here as both, and the first matching branch decides + // what a page is told. Reported as unavailable it reads as "retry + // shortly", which is the one thing that cannot help. + // + // The capability was sound when it was minted and the state moved under + // it — an installation revision, a principal's epoch, a build + // incarnation, a binding. The holder's answer to every one of those is + // the same and it is not "retry": it is to mint again. Reported as the + // unavailable fallthrough below, a page or a client policy would retry + // against a credential guaranteed to keep failing; reported as a plain + // denial, a condition one mint fixes would reach the viewer as their + // own authorization failing. Aborted is neither, and a caller that + // re-asks gets a fresh mint because the cache no longer holds a current + // one. + return connect.NewError(connect.CodeAborted, errors.New("the authority this solution presented has been superseded; the call was not made")) + case errors.Is(err, ErrNotAttested): + // This solution could not attest which module is asking, so it did not + // ask. The viewer's authority is not in question and the condition is + // one renewal away, which is what unavailable says and what neither + // internal nor permission_denied would. + return connect.NewError(connect.CodeUnavailable, errors.New("this solution cannot currently act for the viewer against the module")) + case errors.Is(err, workcontext.ErrNotACoreToken), errors.Is(err, workcontext.ErrInvalid): + // A capability this solution could not present is this solution's + // problem, not a module that is briefly unreachable, and not the + // viewer's authorization. Reported as the unavailable fallthrough it + // would be retried against a credential that will be exactly as + // unusable next time, and the one signal that the issuer handed back + // something this runtime cannot carry would be spent on a retry loop. + // The three are kept apart from each other at the point they are + // logged, not here: "another format", "no seal" and "bad capability" + // have different owners, and core deliberately does not let one + // errors.Is branch reach all three. + return connect.NewError(connect.CodeInternal, errors.New("this solution could not present a usable credential for the module")) } return connect.NewError(connect.CodeUnavailable, errors.New("the module call failed")) } diff --git a/passthrough_scopes_test.go b/passthrough_scopes_test.go index 092d88b..20bfb81 100644 --- a/passthrough_scopes_test.go +++ b/passthrough_scopes_test.go @@ -38,12 +38,12 @@ func mintedActions(m mintRequest) string { func TestPassthroughMintsOnlyTheCalledMethodsScopes(t *testing.T) { gw := newModuleGateway(t, http.StatusOK, `{"entryId":"e1"}`) - s := passthroughServer(gw.URL, perMethodModule()) + s := passthroughServer(t, gw.URL, perMethodModule()) - if status, body := call(t, s, searchPath, "Bearer viewer", `{"entryId":"e1"}`); status != http.StatusOK { + if status, body := call(t, s, searchPath, viewerBearer(), `{"entryId":"e1"}`); status != http.StatusOK { t.Fatalf("search: status %d: %v", status, body) } - if status, body := call(t, s, getPath, "Bearer viewer", `{"entryId":"e1"}`); status != http.StatusOK { + if status, body := call(t, s, getPath, viewerBearer(), `{"entryId":"e1"}`); status != http.StatusOK { t.Fatalf("get: status %d: %v", status, body) } if len(gw.mints) != 2 { @@ -67,10 +67,10 @@ func TestPassthroughMethodScopesReplaceTheModules(t *testing.T) { module := perMethodModule() module.Scopes = []Scope{{ResourceKind: "things", Actions: []string{"read"}}} module.Methods[0].Scopes = nil // Search falls back to the module's. - s := passthroughServer(gw.URL, module) + s := passthroughServer(t, gw.URL, module) - call(t, s, searchPath, "Bearer viewer", `{}`) - call(t, s, getPath, "Bearer viewer", `{"entryId":"e1"}`) + call(t, s, searchPath, viewerBearer(), `{}`) + call(t, s, getPath, viewerBearer(), `{"entryId":"e1"}`) if len(gw.mints) != 2 || mintedActions(gw.mints[0]) != "things:read" || mintedActions(gw.mints[1]) != "things:inspect" { t.Fatalf("mints = %+v, want module scope for search and method scope (alone) for get", gw.mints) } @@ -79,9 +79,9 @@ func TestPassthroughMethodScopesReplaceTheModules(t *testing.T) { func TestPassthroughRefusesOnlyTheMethodWhoseAuthorityTheViewerLacks(t *testing.T) { gw := newModuleGateway(t, http.StatusOK, `{"entryId":"e1"}`) gw.deny = "inspect" - s := passthroughServer(gw.URL, perMethodModule()) + s := passthroughServer(t, gw.URL, perMethodModule()) - status, body := call(t, s, getPath, "Bearer member", `{"entryId":"e1"}`) + status, body := call(t, s, getPath, sealedBearer("member"), `{"entryId":"e1"}`) if status != http.StatusForbidden || body["code"] != "permission_denied" { t.Fatalf("get without inspect: status %d, body %v; want 403 permission_denied", status, body) } @@ -92,7 +92,7 @@ func TestPassthroughRefusesOnlyTheMethodWhoseAuthorityTheViewerLacks(t *testing. t.Fatalf("the module was called without a capability: %v", gw.calls) } // The member's own method is unaffected, and its mint was not widened. - status, body = call(t, s, searchPath, "Bearer member", `{"entryId":"e1"}`) + status, body = call(t, s, searchPath, sealedBearer("member"), `{"entryId":"e1"}`) if status != http.StatusOK || body["entryId"] != "e1" { t.Fatalf("search after a refused get: status %d, body %v", status, body) } diff --git a/passthrough_test.go b/passthrough_test.go index 169c553..f457a1b 100644 --- a/passthrough_test.go +++ b/passthrough_test.go @@ -3,6 +3,7 @@ package solution import ( "context" "encoding/json" + "fmt" "io" "net/http" "net/http/httptest" @@ -14,7 +15,7 @@ import ( "time" "github.com/codefly-dev/core/solution/manifest" - codefly "github.com/codefly-dev/sdk-go" + "github.com/codefly-dev/sdk-go/workcontext" "google.golang.org/protobuf/reflect/protoreflect" ) @@ -38,8 +39,18 @@ type moduleGateway struct { status int reply string // deny, when set, is an action the viewer does not hold: a mint asking for - // it is refused the way accounts refuses one. + // it is refused the way accounts refuses one — with 403. deny string + // conflict, when set, is an action whose mint is answered with 409 rather + // than 403: the status that means "the state you were sealed to has moved" + // for a capability and "this viewer lacks that authority" for a viewer, + // which is the ambiguity nothing in this runtime may resolve by guessing. + // + // It exists because a test named after a CONFLICT was driving `deny` and + // therefore asserting against a 403. A 409-specific regression — inferring + // supersession and poisoning the process — would have survived it, which a + // reviewer caught twice before it was fixed. + conflict string mu sync.Mutex mints []mintRequest @@ -58,9 +69,17 @@ func newModuleGateway(t *testing.T, status int, reply string) *moduleGateway { var mint mintRequest _ = json.Unmarshal(body, &mint) mint.Bearer = r.Header.Get("authorization") + mint.WorkContext = r.Header.Get(workcontext.HeaderName) g.mints = append(g.mints, mint) for _, scope := range mint.AuthorityScopes { for _, action := range scope.Actions { + if g.conflict != "" && action == g.conflict { + writeJSON(w, http.StatusConflict, map[string]any{ + "code": "aborted", + "message": "the state this capability was sealed to has moved", + }) + return + } if g.deny != "" && action == g.deny { writeJSON(w, http.StatusForbidden, map[string]any{ "code": "permission_denied", @@ -71,7 +90,7 @@ func newModuleGateway(t *testing.T, status int, reply string) *moduleGateway { } } writeJSON(w, http.StatusOK, map[string]any{ - "token": "context-" + mint.Audience + ".1", "orgId": mint.OrgID, + "token": capability("context-" + mint.Audience + ".1"), "orgId": mint.OrgID, "ownerPrincipalId": "viewer", "currentActorPrincipalId": "viewer", "expiresAt": time.Now().Add(5 * time.Minute).UTC().Format(time.RFC3339Nano), }) @@ -86,6 +105,13 @@ func newModuleGateway(t *testing.T, status int, reply string) *moduleGateway { return g } +// observedMints is every mint this fake gateway was asked for. +func (g *moduleGateway) observedMints() []mintRequest { + g.mu.Lock() + defer g.mu.Unlock() + return append([]mintRequest{}, g.mints...) +} + // call sends one Connect unary JSON request, as connect-es does, to the // solution's passthrough and returns the status and decoded body. func call(t *testing.T, s *Server, path, bearer, body string) (int, map[string]any) { @@ -102,6 +128,7 @@ func call(t *testing.T, s *Server, path, bearer, body string) (int, map[string]a } req.Header.Set(orgHeader, "org-1") req.Header.Set(sessionHeader, "session-1") + req.Header.Set(workcontext.InstallationIDHeaderName, testInstallation) rec := httptest.NewRecorder() s.passthroughHandler(routes).ServeHTTP(rec, req) var out map[string]any @@ -113,19 +140,37 @@ func call(t *testing.T, s *Server, path, bearer, body string) (int, map[string]a return rec.Code, out } -func passthroughServer(gatewayURL string, modules ...ConsumedModule) *Server { - s := New(Manifest{ID: "test"}).Consumes(modules...) +// passthroughServer is a solution serving the passthrough against a fake +// gateway, holding a real execution credential. +// +// The credential is not optional furniture here: minting for a viewer is +// fail-closed, so a server without one refuses every call (see +// TestAServerWithNoCredentialSourceMintsNothing). Every passthrough test +// therefore stands up the same mint a booted runtime would. +func passthroughServer(t *testing.T, gatewayURL string, modules ...ConsumedModule) *Server { + t.Helper() + s := New(Manifest{ID: "test"}).Consumes(modules...).Credential(attestingSource(t)) s.cfg.gatewayURL = gatewayURL return s } +// attestingSource is an execution credential from a fake host, for a test that +// is about something other than the mint. +func attestingSource(t *testing.T) CredentialSource { + t.Helper() + mint := newHostMint(t, &hostMint{}) + tokenFile := filepath.Join(t.TempDir(), "token") + writeFile(t, tokenFile, "projected") + return mintClientFor(t, mint, tokenFile) +} + const searchPath = "/modules/things/things.v1.Things/Search" func TestPassthroughAnswersADeclaredMethodAsTheViewer(t *testing.T) { gw := newModuleGateway(t, http.StatusOK, `{"entryId":"e1","big":"7","secret":"s3cret","sub":{"name":"x"}}`) - s := passthroughServer(gw.URL, passthroughModule()) + s := passthroughServer(t, gw.URL, passthroughModule()) - status, body := call(t, s, searchPath, "Bearer viewer", `{"entryId":"e1","pageSize":3}`) + status, body := call(t, s, searchPath, viewerBearer(), `{"entryId":"e1","pageSize":3}`) if status != http.StatusOK { t.Fatalf("status %d: %v", status, body) } @@ -141,10 +186,10 @@ func TestPassthroughAnswersADeclaredMethodAsTheViewer(t *testing.T) { if err := json.Unmarshal([]byte(gw.bodys[0]), &sent); err != nil || sent["entryId"] != "e1" || sent["pageSize"] != float64(3) { t.Fatalf("module received %q", gw.bodys[0]) } - if got := gw.calls[0].Header.Get(codefly.WorkContextHeaderName); got != "context-things.1" { + if got := gw.calls[0].Header.Get(workcontext.HeaderName); got != capability("context-things.1") { t.Fatalf("work context = %q, want the one minted for the module", got) } - if gw.calls[0].Header.Get("authorization") != "Bearer viewer" { + if gw.calls[0].Header.Get("authorization") != viewerBearer() { t.Fatal("the viewer's bearer was not forwarded") } if len(gw.mints) != 1 || gw.mints[0].Audience != "things" || gw.mints[0].OrgID != "org-1" || gw.mints[0].SessionID != "session-1" || @@ -157,29 +202,29 @@ func TestPassthroughForwardsOnlyTheBearerToAModuleThatAuthenticatesTheViewer(t * gw := newModuleGateway(t, http.StatusOK, `{"entryId":"e1"}`) module := passthroughModule() module.Scopes, module.ViewerBearer = nil, true - s := passthroughServer(gw.URL, module) + s := passthroughServer(t, gw.URL, module) - if status, body := call(t, s, searchPath, "Bearer viewer", `{"entryId":"e1"}`); status != http.StatusOK { + if status, body := call(t, s, searchPath, viewerBearer(), `{"entryId":"e1"}`); status != http.StatusOK { t.Fatalf("status %d: %v", status, body) } if len(gw.mints) != 0 { t.Fatalf("minted %d capabilities for a module that reads the bearer", len(gw.mints)) } - if gw.calls[0].Header.Get(codefly.WorkContextHeaderName) != "" || gw.calls[0].Header.Get("authorization") != "Bearer viewer" { + if gw.calls[0].Header.Get(workcontext.HeaderName) != "" || gw.calls[0].Header.Get("authorization") != viewerBearer() { t.Fatal("want the viewer's bearer and no capability") } } func TestPassthroughServesOnlyWhatIsDeclared(t *testing.T) { gw := newModuleGateway(t, http.StatusOK, `{}`) - s := passthroughServer(gw.URL, passthroughModule()) + s := passthroughServer(t, gw.URL, passthroughModule()) for _, path := range []string{ "/modules/things/things.v1.Things/Get", // the module's, but not declared "/modules/other/things.v1.Things/Search", // not a consumed module "/modules/things/../things/x", // not a procedure "/modules/things/things.v1.Things/Search/", // not the procedure } { - status, body := call(t, s, path, "Bearer viewer", `{}`) + status, body := call(t, s, path, viewerBearer(), `{}`) if status != http.StatusNotFound || body["code"] != "not_found" { t.Errorf("%s: status %d %v, want not_found", path, status, body) } @@ -191,7 +236,7 @@ func TestPassthroughServesOnlyWhatIsDeclared(t *testing.T) { func TestPassthroughRequiresTheViewersBearer(t *testing.T) { gw := newModuleGateway(t, http.StatusOK, `{}`) - s := passthroughServer(gw.URL, passthroughModule()) + s := passthroughServer(t, gw.URL, passthroughModule()) status, body := call(t, s, searchPath, "", `{}`) if status != http.StatusUnauthorized || body["code"] != "unauthenticated" { t.Fatalf("status %d %v", status, body) @@ -203,11 +248,11 @@ func TestPassthroughRequiresTheViewersBearer(t *testing.T) { func TestPassthroughForwardsOnlyTheMethodsOwnFields(t *testing.T) { gw := newModuleGateway(t, http.StatusOK, `{}`) - s := passthroughServer(gw.URL, passthroughModule()) + s := passthroughServer(t, gw.URL, passthroughModule()) // A field the method's request does not declare is not the page's to add: // the request is re-encoded from the method's message, so it never reaches // the module, and it can name no path or prefix of its own. - status, body := call(t, s, searchPath, "Bearer viewer", `{"entryId":"e1","path":"/v1/other","prefix":"/v1/admin"}`) + status, body := call(t, s, searchPath, viewerBearer(), `{"entryId":"e1","path":"/v1/other","prefix":"/v1/admin"}`) if status != http.StatusOK { t.Fatalf("status %d %v", status, body) } @@ -230,8 +275,8 @@ func TestPassthroughRelaysTheModulesRefusal(t *testing.T) { } { t.Run(tc.name, func(t *testing.T) { gw := newModuleGateway(t, tc.status, tc.reply) - s := passthroughServer(gw.URL, passthroughModule()) - status, body := call(t, s, searchPath, "Bearer viewer", `{}`) + s := passthroughServer(t, gw.URL, passthroughModule()) + status, body := call(t, s, searchPath, viewerBearer(), `{}`) if status != tc.wantStatus || body["code"] != tc.wantCode || body["message"] != tc.wantMessage { t.Fatalf("status %d %v, want %d %s %q", status, body, tc.wantStatus, tc.wantCode, tc.wantMessage) } @@ -323,9 +368,9 @@ func TestPassthroughMergesTheDeclaredPinIntoEveryRequest(t *testing.T) { pin.Set(pin.Descriptor().Fields().ByName("ids"), ids) module := passthroughModule() module.Methods[0].Pin = pin - s := passthroughServer(gw.URL, module) + s := passthroughServer(t, gw.URL, module) - if status, body := call(t, s, searchPath, "Bearer viewer", `{"tenant":"page-tenant","ids":["page-id"]}`); status != http.StatusOK { + if status, body := call(t, s, searchPath, viewerBearer(), `{"tenant":"page-tenant","ids":["page-id"]}`); status != http.StatusOK { t.Fatalf("status %d %v", status, body) } var sent struct { @@ -372,3 +417,78 @@ func TestInterfaceArtifactCarriesTheManifestVersionAndBothSurfaces(t *testing.T) t.Fatal("a backend with no service manifest rendered a version") } } + +// TestAViewerBearerRouteRefusesToActWithoutACredential: a ViewerBearer route +// mints nothing, so it asked the credential for nothing — and it was therefore +// the one route that kept forwarding the viewer's bearer to the gateway after +// the issuer refused this build. +// +// The credential is what authorises this process to act for a viewer, not +// merely what it mints with. A route that forwards a viewer's bearer under a +// withdrawn credential is acting without authority, whatever it does not mint. +func TestAViewerBearerRouteRefusesToActWithoutACredential(t *testing.T) { + gw := newModuleGateway(t, http.StatusOK, `{"entry_id":"e1"}`) + bearerOnly := ConsumedModule{ + As: "pages", + ViewerBearer: true, + Methods: []ConsumedMethod{{Name: "/things.v1.Things/Search", Response: MustFieldMask(thingMessage("Thing"), "entry_id", "big")}}, + } + build := func(t *testing.T, source CredentialSource) (*Server, passthroughRoute) { + t.Helper() + server := New(Manifest{ID: testSolutionID}).Consumes(bearerOnly) + if source != nil { + server.Credential(source) + } + server.cfg.profile = localProfile + server.cfg.gatewayURL = gw.URL + route := passthroughRoute{module: bearerOnly, method: bearerOnly.Methods[0]} + return server, route + } + + header := http.Header{} + header.Set("authorization", viewerBearer()) + header.Set(orgHeader, "org-1") + header.Set(sessionHeader, "session-1") + header.Set(workcontext.InstallationIDHeaderName, testInstallation) + + t.Run("a refused build", func(t *testing.T) { + server, route := build(t, failingCredentialSource{ + err: fmt.Errorf("%w: this build is not the one the presence document approved", workcontext.ErrMintRefused), + }) + if _, err := server.authorize(context.Background(), route, header, nil); err == nil { + t.Fatal("a ViewerBearer route acted for a viewer while this execution's credential was refused: it forwards that viewer's bearer to the gateway under an authority the issuer has withdrawn") + } + }) + + t.Run("an issuer that cannot answer", func(t *testing.T) { + server, route := build(t, failingCredentialSource{ + err: fmt.Errorf("%w: the issuer cannot reach its own dependencies", workcontext.ErrMintUnavailable), + }) + if _, err := server.authorize(context.Background(), route, header, nil); err == nil { + t.Fatal("a ViewerBearer route acted for a viewer while this execution's credential could not be obtained") + } + }) + + // The survive-control: with a credential the issuer approves, the same + // route authorises normally. Without this, a route refusing everything + // passes both cases above. + t.Run("an approved credential", func(t *testing.T) { + tokenFile := filepath.Join(t.TempDir(), "token") + writeFile(t, tokenFile, "projected") + mint := newHostMint(t, &hostMint{}) + server, route := build(t, mintClientFor(t, mint, tokenFile)) + if _, err := server.authorize(context.Background(), route, header, nil); err != nil { + t.Fatalf("a ViewerBearer route was refused while this process holds an approved credential: %v", err) + } + }) + + // And a server with no credential source at all still serves: a solution + // that mints nothing and declares nothing to mint with is a valid shape, + // and the ask is only about a source that exists and is failing. + t.Run("no credential source declared", func(t *testing.T) { + server, route := build(t, nil) + if _, err := server.authorize(context.Background(), route, header, nil); err != nil { + t.Fatalf("a ViewerBearer route was refused on a solution that declares no credential source: %v", err) + } + }) +} diff --git a/passthroughtest/identity.go b/passthroughtest/identity.go new file mode 100644 index 0000000..f66dbb3 --- /dev/null +++ b/passthroughtest/identity.go @@ -0,0 +1,115 @@ +package passthroughtest + +import ( + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "fmt" + "math/big" + "net" + "net/url" + "time" +) + +// The trust domain and the two identities this fake host issues. +// +// sdk-go#51 holds the mint endpoint to WHICH party it is, read from the single +// URI SAN on its leaf, so the fake host needs a real cell rather than +// httptest's shared certificate: that one carries no SPIFFE ID at all, and a +// mint client configured with AdmittedPeers refuses it — correctly. The seam +// exists to run a consumer's passthrough against the posture a deployment has, +// so the fake host grew a CA instead of the client losing the check. +const ( + trustDomain = "passthroughtest" + mintPeerIdentity = "spiffe://" + trustDomain + "/ns/host/sa/mint" + workloadIdentitry = "spiffe://" + trustDomain + "/ns/solutions/sa/passthroughtest" +) + +// mintCell is the fake host's own certificate authority: a root, the mint +// endpoint's leaf, and the workload leaf the mint client presents. +type mintCell struct { + roots *x509.CertPool + mint tls.Certificate + workload tls.Certificate +} + +// newMintCell issues the cell. Everything is ephemeral and in memory: this is +// a test seam, and a key on disk would outlive the run that made it. +func newMintCell() (*mintCell, error) { + rootKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + return nil, fmt.Errorf("the fake host could not generate its root key: %w", err) + } + rootTemplate := &x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{CommonName: trustDomain + " root"}, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(24 * time.Hour), + KeyUsage: x509.KeyUsageCertSign | x509.KeyUsageDigitalSignature, + BasicConstraintsValid: true, + IsCA: true, + } + rootDER, err := x509.CreateCertificate(rand.Reader, rootTemplate, rootTemplate, &rootKey.PublicKey, rootKey) + if err != nil { + return nil, fmt.Errorf("the fake host could not sign its root: %w", err) + } + root, err := x509.ParseCertificate(rootDER) + if err != nil { + return nil, fmt.Errorf("the fake host's root does not parse: %w", err) + } + roots := x509.NewCertPool() + roots.AddCert(root) + + // The mint endpoint. It is a server AND the thing whose identity the client + // admits, so it carries exactly one URI SAN and the names httptest serves + // on. + mint, err := leaf(root, rootKey, 2, mintPeerIdentity, + []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + []string{"localhost"}, []net.IP{net.ParseIP("127.0.0.1"), net.ParseIP("::1")}) + if err != nil { + return nil, err + } + // The workload. One URI SAN as well, because this is what the real mint + // endpoint reads to decide which solution is asking. + workload, err := leaf(root, rootKey, 3, workloadIdentitry, + []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, nil, nil) + if err != nil { + return nil, err + } + return &mintCell{roots: roots, mint: *mint, workload: *workload}, nil +} + +// leaf issues one end-entity certificate with a single SPIFFE URI SAN. +func leaf( + root *x509.Certificate, rootKey *ecdsa.PrivateKey, serial int64, identity string, + usages []x509.ExtKeyUsage, dnsNames []string, ips []net.IP, +) (*tls.Certificate, error) { + spiffeID, err := url.Parse(identity) + if err != nil { + return nil, fmt.Errorf("the fake host's identity %q does not parse: %w", identity, err) + } + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + return nil, fmt.Errorf("the fake host could not generate a key for %s: %w", identity, err) + } + template := &x509.Certificate{ + SerialNumber: big.NewInt(serial), + Subject: pkix.Name{CommonName: identity}, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(24 * time.Hour), + KeyUsage: x509.KeyUsageDigitalSignature, + ExtKeyUsage: usages, + BasicConstraintsValid: true, + URIs: []*url.URL{spiffeID}, + DNSNames: dnsNames, + IPAddresses: ips, + } + der, err := x509.CreateCertificate(rand.Reader, template, root, &key.PublicKey, rootKey) + if err != nil { + return nil, fmt.Errorf("the fake host could not sign %s: %w", identity, err) + } + return &tls.Certificate{Certificate: [][]byte{der}, PrivateKey: key}, nil +} diff --git a/passthroughtest/mint_race_test.go b/passthroughtest/mint_race_test.go new file mode 100644 index 0000000..9bbda28 --- /dev/null +++ b/passthroughtest/mint_race_test.go @@ -0,0 +1,37 @@ +package passthroughtest + +import ( + "strings" + "sync" + "testing" +) + +// TestConcurrentWorkloadMintsDoNotRaceOnTheCounter: the increment was locked +// and the sequence it names the execution with was read back after unlocking, +// which the race detector reports once enough mints are in flight. It is a +// defect in this harness rather than in the runtime, and a harness that trips +// -race fails a consumer's suite for a reason that is not theirs. +// +// Internal to the package, unlike the rest of the suite, because what it drives +// is the fake host's own mint endpoint rather than a solution's passthrough. +func TestConcurrentWorkloadMintsDoNotRaceOnTheCounter(t *testing.T) { + host := NewHost(t) + client := host.server.Client() + var wait sync.WaitGroup + for range 32 { + wait.Add(1) + go func() { + defer wait.Done() + resp, err := client.Post(host.URL()+workloadMintPath, "application/json", strings.NewReader("{}")) + if err != nil { + t.Error(err) + return + } + _ = resp.Body.Close() + }() + } + wait.Wait() + if got := host.WorkloadMints(); got != 32 { + t.Errorf("the host counted %d mints, want 32", got) + } +} diff --git a/passthroughtest/passthroughtest.go b/passthroughtest/passthroughtest.go index 48eca10..d5a6f5d 100644 --- a/passthroughtest/passthroughtest.go +++ b/passthroughtest/passthroughtest.go @@ -2,9 +2,14 @@ // test, against a fake host, so a consuming solution can exercise the calls its // page makes — unary and server-streaming — without a composition. // -// It serves the real passthrough: Server.PassthroughHandler, the handler Serve -// mounts, checked by the same boot check, over an httptest server. Only the -// host is fake. Host stands in for the gateway the passthrough calls through: +// It serves the real passthrough — the handler Serve mounts, checked by the +// same boot check — over an httptest server, reached through internal/seam. +// Only the host is fake. The seam is not public API: Server.PassthroughHandler +// was exported once, and a deployment calling it completed a viewer mint and a +// module call over plaintext with no validate(), no mTLS boot, no caller +// allow-list, no ceiling and no authenticated outbound. This package is the one +// caller Go's internal rule lets reach it, and because any module can import +// *this* package, its constructors refuse a non-test binary. Host stands in for the gateway the passthrough calls through: // it mints the viewer's Work Context (the accounts StartTask procedure the // gateway routes) and forwards each /v1//* call to the module address the // test gave it, typically an httptest server of the test's own that answers @@ -20,20 +25,27 @@ package passthroughtest import ( + "context" + "crypto/tls" + "crypto/x509" "encoding/json" "fmt" + corework "github.com/codefly-dev/core/workcontext" "net/http" "net/http/httptest" "net/http/httputil" "net/url" + "os" + "path/filepath" "strings" "sync" "testing" "time" "github.com/codefly-dev/core/solution/manifest" - codefly "github.com/codefly-dev/sdk-go" + "github.com/codefly-dev/sdk-go/workcontext" solution "github.com/codefly-dev/solution-runtime-go" + "github.com/codefly-dev/solution-runtime-go/internal/seam" ) // startTaskProcedure is the accounts procedure the gateway routes a Work @@ -41,19 +53,44 @@ import ( const startTaskProcedure = "/saas.accounts.v1.WorkContextService/StartTask" // Viewer is the identity the page calls as: the bearer the browser sends, and -// the organization and session the gateway stamps from it (x-org-id, -// x-session-id) before the call reaches the solution. +// the organization, session and installation the gateway stamps from it +// (x-org-id, x-session-id, x-codefly-installation-id) before the call reaches +// the solution. type Viewer struct { Bearer string OrgID string SessionID string + // InstallationID is the installation the gateway stamps, which every mint + // the runtime runs names. Empty is refused. + InstallationID string } // DefaultViewer is the viewer Solution.Client calls as. -var DefaultViewer = Viewer{ - Bearer: "Bearer passthroughtest-viewer", - OrgID: "passthroughtest-org", - SessionID: "passthroughtest-session", +// +// The bearer is a real sealed capability from core's authority, not a +// placeholder string. Every mint the runtime runs names the installation it +// acts under, and the viewer's comes from the seal of the capability they +// arrived with — so a placeholder bearer is a request no installation can be +// read from, and this seam would hand every consumer the refusal path instead +// of the behaviour they are testing. +// +// A consumer supplying their own Viewer needs a sealed bearer too. SealedViewer +// builds one, and the refusal names what is missing if they pass something +// else. +var DefaultViewer = SealedViewer("passthroughtest-viewer") + +// SealedViewer is a viewer whose bearer is a capability sealed to core's +// fixture installation, which is what the runtime reads the mint's installation +// from. seed names the capability so two viewers can differ. +func SealedViewer(seed string) Viewer { + return Viewer{ + Bearer: "Bearer " + capability(seed), + OrgID: "passthroughtest-org", + SessionID: "passthroughtest-session", + // The installation the gateway stamps beside the org and the session. + // Every mint the runtime runs names one and refuses without it. + InstallationID: corework.FixtureInstallation, + } } // Mint is one Work Context the host was asked to mint: the passthrough's @@ -100,19 +137,66 @@ type Refusal struct { type Host struct { server *httptest.Server + // dir holds the projected files this fake host stands in for. + dir string mu sync.Mutex modules map[string]*httputil.ReverseProxy refuse func(Mint) *Refusal mints []Mint calls []Call + // workloadMints counts the solution's own execution credentials. + workloadMints int + // mintServer serves the workload mint over TLS, separately from the + // gateway's own plaintext server. + // + // The SDK's mint client refuses plain HTTP outright — the projected + // service-account token travels on that request — and it requires the + // roots that may sign the endpoint, with no system-pool fallback. Both are + // right, and neither can be satisfied by the plaintext server the rest of + // this host uses. + // + // Only the mint moves. Making the whole host TLS would need a SPIFFE leaf + // from a test CA for the gateway's own identity, because the runtime's + // outbound client admits a platform destination by its certificate — and + // the seam deliberately runs without that boot. That remains the + // fidelity gap seam.go already names; this closes the half the SDK now + // requires rather than pretending to close both. + mintServer *httptest.Server + // cell is the fake host's own CA, because the mint endpoint is now held to + // WHICH party it is and not merely to a certificate from an accepted root. + cell *mintCell } // NewHost starts a fake host, closed when the test ends. func NewHost(t testing.TB) *Host { + mustBeATest() t.Helper() - h := &Host{modules: map[string]*httputil.ReverseProxy{}} + h := &Host{modules: map[string]*httputil.ReverseProxy{}, dir: t.TempDir()} h.server = httptest.NewServer(http.HandlerFunc(h.serveHTTP)) t.Cleanup(h.server.Close) + cell, err := newMintCell() + if err != nil { + t.Fatalf("start the fake host: %v", err) + } + h.cell = cell + // mTLS, with the client certificate REQUIRED and verified. + // + // The real endpoint receives the projected service-account token, and + // sdk-go#51 makes the client present this workload's X.509-SVID on that + // request. A fake host that merely tolerated a client certificate would + // let a consumer's green run mean less than it looks like: the seam is + // where a consumer finds out their passthrough works against the posture + // a deployment has. + mint := httptest.NewUnstartedServer(http.HandlerFunc(h.serveHTTP)) + mint.TLS = &tls.Config{ + MinVersion: tls.VersionTLS13, + Certificates: []tls.Certificate{cell.mint}, + ClientAuth: tls.RequireAndVerifyClientCert, + ClientCAs: cell.roots, + } + mint.StartTLS() + h.mintServer = mint + t.Cleanup(h.mintServer.Close) return h } @@ -168,6 +252,59 @@ func (h *Host) Calls() []Call { return append([]Call(nil), h.calls...) } +// workloadMintPath is where this fake host mints the solution's own execution +// credential — the same path the runtime derives from its resolved gateway. +// +// It exists because minting for a viewer is fail-closed: a solution that cannot +// attest which module is asking does not ask. A seam that handed the +// passthrough no credential would therefore be a seam in which every call is +// refused, so the fake host mints one, exactly as the real host does. +const workloadMintPath = "/platform/_credential" + +// mintWorkload answers the solution's own mint: one credential per execution, +// sealed to this fake host's installation. +func (h *Host) mintWorkload(w http.ResponseWriter, r *http.Request) { + // The sequence is copied under the lock and the copy is what names the + // task: reading h.workloadMints again after unlocking is a read of shared + // state that concurrent mints race on, which the race detector reports with + // thirty-two of them in flight. + h.mu.Lock() + h.workloadMints++ + execution := h.workloadMints + h.mu.Unlock() + token, _, err := standInAuthority().Start(r.Context(), corework.StartInput{ + TenantID: corework.FixtureTenant, + OwnerPrincipalID: corework.FixturePrincipal, + OwnerPrincipalKind: "human", + TaskID: fmt.Sprintf("passthroughtest-execution-%d", execution), + Audience: WorkloadAudience, + OrganizationID: corework.FixtureOrganization, + InstallationID: corework.FixtureInstallation, + TTL: 10 * time.Minute, + Execution: corework.Execution{ + ImageDigest: corework.FixtureImageDigest, + BuildIncarnation: corework.FixtureBuildIncarnation, + }, + }) + if err != nil { + writeJSON(w, http.StatusInternalServerError, map[string]any{"code": 13, "message": err.Error()}) + return + } + writeJSON(w, http.StatusOK, map[string]string{"work_context": token}) +} + +// WorkloadAudience is the audience this fake host mints the solution's own +// execution credential for. +const WorkloadAudience = "passthroughtest-workload" + +// WorkloadMints is how many execution credentials this host was asked for. A +// correct run asks once, however many calls the page makes. +func (h *Host) WorkloadMints() int { + h.mu.Lock() + defer h.mu.Unlock() + return h.workloadMints +} + // consumes is the api.consumes projection of the routed modules. func (h *Host) consumes() []manifest.ConsumedAPI { h.mu.Lock() @@ -180,6 +317,10 @@ func (h *Host) consumes() []manifest.ConsumedAPI { } func (h *Host) serveHTTP(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == workloadMintPath { + h.mintWorkload(w, r) + return + } if r.URL.Path == startTaskProcedure { h.mint(w, r) return @@ -193,7 +334,7 @@ func (h *Host) serveHTTP(w http.ResponseWriter, r *http.Request) { return } call := Call{Module: as, Method: r.Method, Path: r.URL.Path, Bearer: r.Header.Get("authorization")} - if presented := r.Header.Get(codefly.WorkContextHeaderName); presented != "" { + if presented := r.Header.Get(workcontext.HeaderName); presented != "" { for i := range h.mints { if h.mints[i].Token == presented { mint := h.mints[i] @@ -257,7 +398,7 @@ func (h *Host) record(mint *Mint, granted bool) { h.mu.Lock() defer h.mu.Unlock() if granted { - mint.Token = fmt.Sprintf("passthroughtest-%d.%s", len(h.mints)+1, mint.Audience) + mint.Token = capability(fmt.Sprintf("passthroughtest-%d-%s", len(h.mints)+1, mint.Audience)) } h.mints = append(h.mints, *mint) } @@ -280,13 +421,85 @@ type Solution struct { // modules host routes as the solution's api.consumes: the real handler, and // the error Serve would refuse the declaration with at boot. func Handler(host *Host, consumes ...solution.ConsumedModule) (http.Handler, error) { - server := solution.New(solution.Manifest{ID: "passthroughtest"}).Consumes(consumes...) - return server.PassthroughHandler(solution.PassthroughEnvironment{GatewayURL: host.URL(), Consumes: host.consumes()}) + mustBeATest() + source, err := host.credentialSource() + if err != nil { + return nil, err + } + server := solution.New(solution.Manifest{ID: "passthroughtest"}). + Consumes(consumes...). + Credential(source) + projection, err := json.Marshal(host.consumes()) + if err != nil { + return nil, err + } + // Through internal/seam, because building a credential-bearing handler + // without the boot is no longer something the runtime's public API can do: + // exported, that constructor was a production bypass, and this package is + // the one caller Go's internal-package rule still lets reach it. + return seam.Passthrough(server, host.URL(), string(projection)) +} + +// credentialSource is the solution's own execution credential, obtained from +// this fake host through the SDK's mint client — the same client a booted +// runtime uses, so the seam exercises the attestation rather than skipping it. +func (h *Host) credentialSource() (solution.CredentialSource, error) { + tokenFile := filepath.Join(h.dir, "projected-token") + if err := os.WriteFile(tokenFile, []byte("passthroughtest-projected-token"), 0o600); err != nil { + return nil, err + } + return workcontext.NewMintClient(workcontext.MintOptions{ + URL: h.mintServer.URL + workloadMintPath, + // The audience is named as a pinned value now, not passed as a string + // beside the pin, so the seam supplies a pin that answers it. + Audience: workcontext.AuthorityValue{Name: authorityGroup, Key: audienceKey}, + Authority: fixedAuthority{audience: WorkloadAudience}, + ProjectedToken: workcontext.ProjectedTokenFile(tokenFile), + ProjectionAudience: "accounts", + // The three readers sdk-go#51 takes in place of a pool by value. They + // are readers here for the same reason they are readers in a booted + // runtime: each is consulted during the handshake, so what this seam + // exercises is the check and not a snapshot of it. + TrustAnchor: func() (*x509.CertPool, error) { return h.cell.roots, nil }, + ClientCertificate: func() (*tls.Certificate, error) { return &h.cell.workload, nil }, + AdmittedPeers: func() ([]string, error) { return []string{mintPeerIdentity}, nil }, + }) +} + +// mustBeATest refuses to build any of this outside a test binary. +// +// This package is the one caller Go's internal-package rule lets reach the +// passthrough seam, and that made it the bypass it was built to close: the root +// package's exported constructor is gone, but Handler is exported from a package +// any module can import, so a deployment importing it got the same +// credential-bearing handler with no validate(), no mTLS boot, no caller +// allow-list, no ceiling and no authenticated outbound. Taking a testing.TB is +// not the gate it resembles — testing.TB's unexported method only stops a +// type *declaring* the interface, and a struct that EMBEDS testing.TB satisfies +// it in any program, so the signature proves nothing about where the call came +// from. +// +// testing.Testing() does prove it: it is true exactly when the binary was built +// by `go test`. It was the wrong tool for the root package, where the +// legitimate caller is production code and only the bypass is a test; it is the +// right one here, where every legitimate caller is a test and only the bypass +// is production. +func mustBeATest() { refuseOutsideTest(testing.Testing()) } + +// refuseOutsideTest takes the answer rather than asking for it, so the refusal +// itself is reachable from a test. Written as one function reading +// testing.Testing() directly, the only branch that matters is the one no test +// can enter — a mutation that neutered it left every gate here passing. +func refuseOutsideTest(isTest bool) { + if !isTest { + panic("passthroughtest is a test seam and this is not a test binary: it builds a handler that holds a real execution credential without validate(), the mTLS boot, the caller allow-list, the published ceiling or authenticated outbound, so a deployment reaching it would serve the viewer's bearer and this workload's credential over whatever it was mounted on. Boot the runtime with solution.Serve instead") + } } // Start serves the passthrough for the declaration against host, until the // test ends. A declaration Serve would refuse fails the test. func Start(t testing.TB, host *Host, consumes ...solution.ConsumedModule) *Solution { + mustBeATest() t.Helper() handler, err := Handler(host, consumes...) if err != nil { @@ -325,6 +538,9 @@ func (t viewerTransport) RoundTrip(r *http.Request) (*http.Response, error) { "authorization": t.viewer.Bearer, "x-org-id": t.viewer.OrgID, "x-session-id": t.viewer.SessionID, + // Stamped beside them, because every mint the runtime runs names the + // installation it acts under. + "x-codefly-installation-id": t.viewer.InstallationID, } { r.Header.Del(header) if value != "" { @@ -333,3 +549,88 @@ func (t viewerTransport) RoundTrip(r *http.Request) (*http.Response, error) { } return t.base.RoundTrip(r) } + +// capability is a stand-in issuer's capability, minted per seed. +// +// It is minted by **core's** authority, from core's own conformance fixture +// identities and key — the one implementation of a Work Context, which this +// runtime neither signs nor parses. The private key is public by design (core +// derives it from a seed in its own source), so this is a real sealed +// capability that a conforming verifier accepts, without a signer or a payload +// struct living here. That matters beyond convenience: a fixture issuer with +// its own encoding is how two implementations of a capability format start. +// +// Memoised by seed so a test can recompute the capability it expects to have +// been presented: every mint carries a fresh nonce, so minting the same seed +// twice would otherwise produce two different strings. +func capability(seed string) string { + capabilityMu.Lock() + defer capabilityMu.Unlock() + if issued, ok := issuedCapabilities[seed]; ok { + return issued + } + token, _, err := standInAuthority().Start(context.Background(), corework.StartInput{ + TenantID: corework.FixtureTenant, + OwnerPrincipalID: corework.FixturePrincipal, + OwnerPrincipalKind: "human", + TaskID: seed, + Audience: corework.FixtureAudience, + OrganizationID: corework.FixtureOrganization, + InstallationID: corework.FixtureInstallation, + TTL: 10 * time.Minute, + Execution: corework.Execution{ + ImageDigest: corework.FixtureImageDigest, + BuildIncarnation: corework.FixtureBuildIncarnation, + }, + }) + if err != nil { + panic("stand-in capability: " + err.Error()) + } + issuedCapabilities[seed] = token + return token +} + +var ( + capabilityMu sync.Mutex + issuedCapabilities = map[string]string{} +) + +// standInAuthority is core's minter, configured from core's fixture identities. +// +// Built once through sync.OnceValue rather than on a nil check: the check was a +// data race, reached from the fake host's mint handler and from capability() at +// the same time, so a consumer running this seam under -race could fail in a +// file they do not own, for a reason that has nothing to do with their test. +var standInAuthority = sync.OnceValue(func() *corework.Authority { + _, key := corework.FixtureKeyPair() + return &corework.Authority{ + Issuer: corework.FixtureIssuer, + KeyID: corework.FixtureKeyID, + Key: key, + Revisions: corework.FixtureRevisions(), + Seals: corework.FixtureSeals(), + } +}) + +// authorityGroup and audienceKey name the pinned value this seam's mint reads +// its audience from. They mirror the root package's own constants; the seam +// cannot import them without importing the package it is a seam for. +const ( + authorityGroup = "module-authority" + audienceKey = "AUDIENCE" +) + +// fixedAuthority is an AuthorityPin whose values never drift, which is what a +// fake host should be: the drift refusal is the runtime's behaviour to exercise +// elsewhere, and a seam that drifted at random would fail a consumer's suite +// for a reason they did not write. +type fixedAuthority struct{ audience string } + +func (fixedAuthority) Recheck(context.Context) error { return nil } + +func (a fixedAuthority) Value(name, key string) (string, error) { + if name == authorityGroup && key == audienceKey { + return a.audience, nil + } + return "", fmt.Errorf("passthroughtest: no pinned authority value %s/%s", name, key) +} diff --git a/passthroughtest/passthroughtest_internal_test.go b/passthroughtest/passthroughtest_internal_test.go new file mode 100644 index 0000000..ea96c29 --- /dev/null +++ b/passthroughtest/passthroughtest_internal_test.go @@ -0,0 +1,44 @@ +package passthroughtest + +import ( + "strings" + "testing" +) + +// TestTheSeamRefusesANonTestBinary is the behavioural half of the seam's gate. +// +// This package is importable by any module — unlike internal/seam, which Go +// keeps inside this one — so Handler and Start were an exported, production +// reachable path to a handler holding a real execution credential, having +// skipped validate(), the mTLS boot, the caller allow-list, the ceiling and +// authenticated outbound. Taking a testing.TB does not prevent that: the +// unexported method stops a type declaring the interface, not one embedding +// it, so a few lines of production code satisfy it. +func TestTheSeamRefusesANonTestBinary(t *testing.T) { + defer func() { + recovered := recover() + if recovered == nil { + t.Fatal("the seam did not refuse a non-test binary: a deployment importing this package builds a credential-bearing handler with none of the boot behind it") + } + message, ok := recovered.(string) + if !ok { + t.Fatalf("the refusal panicked with %T, which says nothing to whoever hits it", recovered) + } + for _, named := range []string{"test binary", "credential", "solution.Serve"} { + if !strings.Contains(message, named) { + t.Errorf("the refusal does not mention %q, so it does not say what was wrong or what to do instead: %s", named, message) + } + } + }() + refuseOutsideTest(false) +} + +// And it does not refuse the only callers it has. Every legitimate use of this +// package is a test, which is what makes testing.Testing() the right gate here +// and the wrong one for the root package. +func TestTheSeamAdmitsATestBinary(t *testing.T) { + refuseOutsideTest(true) + if !testing.Testing() { + t.Fatal("testing.Testing() is false inside a test, so the gate this package relies on does not mean what it is used for") + } +} diff --git a/passthroughtest/passthroughtest_test.go b/passthroughtest/passthroughtest_test.go index ed6870b..2e4bf82 100644 --- a/passthroughtest/passthroughtest_test.go +++ b/passthroughtest/passthroughtest_test.go @@ -13,7 +13,7 @@ import ( "time" "connectrpc.com/connect" - codefly "github.com/codefly-dev/sdk-go" + "github.com/codefly-dev/sdk-go/workcontext" solution "github.com/codefly-dev/solution-runtime-go" "github.com/codefly-dev/solution-runtime-go/passthroughtest" "google.golang.org/genproto/googleapis/api/annotations" @@ -215,7 +215,7 @@ func TestAUnaryCallIsAnsweredByTheModuleAsTheViewer(t *testing.T) { t.Fatalf("mint = %+v, want the module's audience in the viewer's org and session", mint) } upstream.mu.Lock() - presented := upstream.headers[0].Get(codefly.WorkContextHeaderName) + presented := upstream.headers[0].Get(workcontext.HeaderName) upstream.mu.Unlock() if got := presented; got != mint.Token { t.Fatalf("the module was presented %q, want the minted %q", got, mint.Token) diff --git a/redirect_test.go b/redirect_test.go new file mode 100644 index 0000000..eaf07fc --- /dev/null +++ b/redirect_test.go @@ -0,0 +1,356 @@ +package solution + +import ( + "context" + "crypto/tls" + "io" + "net/http" + "net/http/httptest" + "net/http/httptrace" + "path/filepath" + "strings" + "sync" + "sync/atomic" + + "github.com/codefly-dev/sdk-go/workcontext" + "testing" +) + +// trap is a destination this runtime must never dial: it records anything that +// reaches it, so a test can assert absence rather than infer it. +type trap struct { + *httptest.Server + mu sync.Mutex + hits []http.Header +} + +func newTrap(t *testing.T) *trap { + t.Helper() + tr := &trap{} + tr.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + tr.mu.Lock() + tr.hits = append(tr.hits, r.Header.Clone()) + tr.mu.Unlock() + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte(`{}`)) + })) + t.Cleanup(tr.Close) + return tr +} + +func (tr *trap) received() []http.Header { + tr.mu.Lock() + defer tr.mu.Unlock() + return append([]http.Header(nil), tr.hits...) +} + +// TestACredentialBearingRequestIsNeverRedirected is the blocker a second +// reviewer found, and every piece of it looked right on its own. +// +// The boot's outbound client did set CheckRedirect. gatewayFor copied only its +// .Transport, so every gateway client was an http.Client literal with Go's +// default policy: follow up to ten hops. Go copies the original request's +// headers onto a redirected request and strips only Authorization, +// WWW-Authenticate and Cookie — so the viewer's capability and the installation +// headers travelled, and on the mint this workload's own credential in +// x-codefly-work-context travelled with them. bearerTransport then set the +// bearer again on the redirected hop, putting back the one header Go strips. +// A 307 re-sends the body too. +// +// So one `307 Location: http://elsewhere/` from the gateway, from accounts, or +// from any module the gateway relays handed a third party the viewer's unscoped +// bearer, the capability minted for them, and this workload's credential — in +// cleartext, because nothing about an http:// hop involves the TLS +// configuration that protects the first one. +// +// The assertion is absence at the destination, on every credential-bearing +// path, with the trap on plain http so a followed hop is unmistakable. +func TestACredentialBearingRequestIsNeverRedirected(t *testing.T) { + for _, tc := range []struct { + name string + // redirect reports whether this request is the one to bounce + redirect func(*http.Request) bool + }{ + {"the viewer's mint is redirected", func(r *http.Request) bool { + return r.URL.Path == workContextStartTaskProcedure + }}, + {"the module call is redirected", func(r *http.Request) bool { + return r.URL.Path != workContextStartTaskProcedure + }}, + {"every hop is redirected", func(*http.Request) bool { return true }}, + } { + t.Run(tc.name, func(t *testing.T) { + tr := newTrap(t) + gw := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if tc.redirect(r) { + // 307 keeps the method and re-sends the body, which is the + // worst case: the mint's own payload is replayed too. + http.Redirect(w, r, tr.URL+"/stolen", http.StatusTemporaryRedirect) + return + } + if r.URL.Path == workContextStartTaskProcedure { + writeJSON(w, http.StatusOK, map[string]any{ + "token": capability("context-things.1"), "orgId": "org-1", + "expiresAt": "2099-01-01T00:00:00Z", + }) + return + } + writeJSON(w, http.StatusOK, map[string]any{"entry_id": "e1"}) + })) + t.Cleanup(gw.Close) + + mint := newHostMint(t, &hostMint{}) + tokenFile := filepath.Join(t.TempDir(), "token") + writeFile(t, tokenFile, "projected") + server := New(Manifest{ID: testSolutionID}).Credential(mintClientFor(t, mint, tokenFile)) + server.cfg.gatewayURL = gw.URL + + header := http.Header{} + header.Set("authorization", viewerBearer()) + header.Set(orgHeader, "org-1") + header.Set(sessionHeader, "session-1") + header.Set(workcontext.InstallationIDHeaderName, testInstallation) + + // The call may fail or succeed depending on which hop bounced; what + // it must never do is reach the trap. + moduleGW, err := server.gatewayFor(header).ForModule(context.Background(), "things", + Scope{ResourceKind: "things", Actions: []string{"read"}}) + if err == nil && moduleGW != nil { + type empty struct{} + _, _ = Unary[empty, empty](context.Background(), moduleGW, "/things.v1.Things/Read", &empty{}) + } + + hits := tr.received() + if len(hits) != 0 { + var leaked []string + for _, h := range hits { + for _, name := range []string{"authorization", workcontext.HeaderName, orgHeader, sessionHeader} { + if v := h.Get(name); v != "" { + leaked = append(leaked, name+"="+v) + } + } + } + t.Fatalf("a redirected credential-bearing request reached a third party %d time(s), carrying [%s]: the viewer's bearer, their capability and this workload's own credential must never follow a Location", + len(hits), strings.Join(leaked, " ")) + } + }) + } +} + +// TestEveryGatewayClientRefusesAForeignDestination is the belt to the braces of +// the test above: the redirect policy lives on the client, so the transport is +// where a client built without it is still stopped. +// +// Both of the clients a gateway hands out are checked, because the bug was that +// they were http.Client literals built in two places while only a third place +// carried the policy. +func TestEveryGatewayClientRefusesAForeignDestination(t *testing.T) { + tr := newTrap(t) + gw := newModuleGateway(t, http.StatusOK, `{"entry_id":"e1"}`) + server := New(Manifest{ID: testSolutionID}) + server.cfg.gatewayURL = gw.URL + header := http.Header{} + header.Set("authorization", viewerBearer()) + header.Set(orgHeader, "org-1") + header.Set(sessionHeader, "session-1") + header.Set(workcontext.InstallationIDHeaderName, testInstallation) + gateway := server.gatewayFor(header) + + for _, tc := range []struct { + name string + client *http.Client + }{ + {"HTTPClient", gateway.HTTPClient()}, + {"bearerClient", gateway.bearerClient()}, + } { + t.Run(tc.name, func(t *testing.T) { + request, err := http.NewRequest(http.MethodGet, tr.URL+"/stolen", nil) + if err != nil { + t.Fatal(err) + } + resp, err := tc.client.Do(request) + if err == nil { + _ = resp.Body.Close() + t.Fatalf("%s dialled a destination that is not its gateway", tc.name) + } + if !strings.Contains(err.Error(), "refusing to send a credential-bearing request") { + t.Errorf("%s refused with %v, want the refusal naming what these headers carry", tc.name, err) + } + if got := tr.received(); len(got) != 0 { + t.Errorf("%s reached the foreign destination %d time(s)", tc.name, len(got)) + } + }) + } +} + +// TestASameOriginRedirectIsNotFollowedEither isolates the redirect policy from +// the origin pin. +// +// The cross-origin test above passes if *either* mechanism holds, which is what +// defence in depth means and also what makes it a poor test of either one: with +// CheckRedirect removed, the transport's pin still refuses the foreign hop, so +// the test stays green while the policy it is named after is gone. (Found by +// mutating it, which is the only reason I know.) +// +// Here the Location is on the gateway's own origin, so the pin permits it and +// only the client's policy can stop the hop. A same-origin redirect is also the +// case that is not merely theoretical: a gateway behind an ingress that +// normalises a path answers 307 to itself, and following it would re-send the +// mint's body and re-present every credential for no reason — the runtime +// should see the 3xx and report it, not chase it. +func TestASameOriginRedirectIsNotFollowedEither(t *testing.T) { + var hops struct { + mu sync.Mutex + paths []string + } + record := func(p string) { + hops.mu.Lock() + defer hops.mu.Unlock() + hops.paths = append(hops.paths, p) + } + + gw := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + record(r.URL.Path) + if r.URL.Path == workContextStartTaskProcedure { + // Same origin, different path: the pin allows it. + http.Redirect(w, r, "/relocated"+workContextStartTaskProcedure, http.StatusTemporaryRedirect) + return + } + writeJSON(w, http.StatusOK, map[string]any{ + "token": capability("context-things.1"), "orgId": "org-1", + "expiresAt": "2099-01-01T00:00:00Z", + }) + })) + t.Cleanup(gw.Close) + + mint := newHostMint(t, &hostMint{}) + tokenFile := filepath.Join(t.TempDir(), "token") + writeFile(t, tokenFile, "projected") + server := New(Manifest{ID: testSolutionID}).Credential(mintClientFor(t, mint, tokenFile)) + server.cfg.gatewayURL = gw.URL + + header := http.Header{} + header.Set("authorization", viewerBearer()) + header.Set(orgHeader, "org-1") + header.Set(sessionHeader, "session-1") + header.Set(workcontext.InstallationIDHeaderName, testInstallation) + if _, err := server.gatewayFor(header).ForModule(context.Background(), "things", + Scope{ResourceKind: "things", Actions: []string{"read"}}); err == nil { + t.Error("ForModule reported success on a mint that answered 307: the redirect was followed and its answer taken as the capability") + } + + hops.mu.Lock() + defer hops.mu.Unlock() + for _, path := range hops.paths { + if strings.HasPrefix(path, "/relocated") { + t.Fatalf("the mint followed a same-origin 307 to %q, re-sending its body and re-presenting the viewer's bearer and this workload's credential: a 3xx is an answer to report, not a hop to take (hops: %v)", + path, hops.paths) + } + } + if len(hops.paths) != 1 { + t.Errorf("the mint made %d requests (%v), want exactly 1: the 307 must end the exchange", len(hops.paths), hops.paths) + } +} + +// TestOutboundTrustIsReReadPerConnection is the finding that went unanswered +// through two review rounds, and the argument against it was one I had already +// written down for the other direction. +// +// Inbound, peer trust is re-read per handshake so that removing a compromised +// root takes effect without a restart, on the stated grounds that judging by a +// stale anchor admits whoever should be refused — which is not symmetric with +// serving a stale leaf. Outbound the anchor was read once, into the client's +// RootCAs at boot. It applies with more force there, not less: the two +// destinations on the other side of that client are the mint and the gateway, +// and they receive the projected service-account token, the viewer's bearer and +// this workload's own credential. A root removed because it was compromised +// kept authenticating exactly the parties that are handed everything. +// +// The test replaces the bundle's contents with an unrelated anchor and asserts +// the next connection fails to verify the platform — no restart, no new client. +func TestOutboundTrustIsReReadPerConnection(t *testing.T) { + c := newCell(t) + certFile, keyFile, bundleFile, _, _ := c.workload(t, testPrincipal) + // A platform endpoint served under the cell's anchor, requiring the + // caller's certificate, which is what the real mint does. + host := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + writeJSON(w, http.StatusOK, map[string]string{"ok": "yes"}) + })) + host.TLS = &tls.Config{ + MinVersion: tls.VersionTLS13, + Certificates: []tls.Certificate{*c.identity(t, testGatewayPrincipal)}, + ClientCAs: c.roots, + ClientAuth: tls.RequireAndVerifyClientCert, + } + host.StartTLS() + t.Cleanup(host.Close) + + server := New(Manifest{ID: testSolutionID}) + server.cfg = config{identityCertFile: certFile, identityKeyFile: keyFile, trustBundleFile: bundleFile, + mintPeersFile: identitiesFile(t, testGatewayPrincipal), + gatewayPeersFile: identitiesFile(t, testGatewayPrincipal), + // The destination this test dials, so the authorization set for it is + // the mint's: the set is chosen per destination now. + mintURL: host.URL + credentialMintPath} + server.principal = testPrincipal + client, err := server.outboundClient(nil) + if err != nil { + t.Fatalf("outboundClient: %v", err) + } + + // Control: the platform is reachable while its root is in the bundle. + resp, err := client.Get(host.URL + "/platform/_credential") + if err != nil { + t.Fatalf("the platform was not reachable with its own root in the bundle: %v", err) + } + // Drained, not just closed. net/http returns a connection to the pool only + // once its body is consumed, so closing an unread body leaves the + // connection in no defined state for the CloseIdleConnections below — + // which was a flake, not a subtlety: when the connection had not yet + // become idle there was nothing to close, the next request reused it, and + // the reused connection answered without a handshake. The test then + // reported that revoking a root does nothing, intermittently. + _, _ = io.Copy(io.Discard, resp.Body) + _ = resp.Body.Close() + + // The root is removed and replaced by an unrelated one — the shape of a + // revocation: the bundle still parses, and no longer contains the issuer of + // the certificate the platform presents. + unrelated, _, _ := issueAnchor(t) + writeFile(t, bundleFile, string(unrelated)) + + // Idle connections would otherwise answer from the pool without a + // handshake, which is why per-dial reloading is only half the fix. + client.CloseIdleConnections() + + // And the second request has to actually dial, or it is testing nothing: + // a reused connection skips the handshake and would pass this whether + // trust was re-read or not. httptrace says which happened. + var reused atomic.Bool + trace := &httptrace.ClientTrace{ + GotConn: func(info httptrace.GotConnInfo) { reused.Store(info.Reused) }, + } + request, err := http.NewRequestWithContext( + httptrace.WithClientTrace(context.Background(), trace), + http.MethodGet, host.URL+"/platform/_credential", nil) + if err != nil { + t.Fatal(err) + } + if resp, err := client.Do(request); err == nil { + _ = resp.Body.Close() + if reused.Load() { + t.Fatal("the second request was served on a connection that was already open, so it never handshook and this test asserted nothing: the control's connection had not returned to the pool when CloseIdleConnections ran") + } + t.Fatal("the platform still authenticated after its root was removed from the bundle: outbound trust was snapshotted at boot, so revoking a root would need a restart — and this client carries the projected token, the viewer's bearer and this workload's credential") + } + + // And the bound exists, so a connection cannot outlive the reload + // indefinitely even without CloseIdleConnections. + transport, ok := client.Transport.(*http.Transport) + if !ok { + t.Fatalf("outbound transport is %T", client.Transport) + } + if transport.IdleConnTimeout == 0 || transport.IdleConnTimeout > outboundTrustReloadBound { + t.Errorf("idle connections live for %s: per-dial reloading bounds nothing without a bound on reuse", transport.IdleConnTimeout) + } +} diff --git a/request_handler_test.go b/request_handler_test.go index 8400245..46c40fd 100644 --- a/request_handler_test.go +++ b/request_handler_test.go @@ -27,7 +27,7 @@ func TestRequestHandlerPreservesCallerAndInput(t *testing.T) { if err != nil { return nil, err } - if r.Method != "POST" || string(body) != `{"question":"why?"}` || r.URL.Query().Get("mode") != "wiki" { + if r.Method != "POST" || string(body) != `{"question":"why?"}` || r.URL.Query().Get("mode") != "notes" { t.Error("request input lost") } if r.Context().Value(requestTestKey{}) != "request-context" { @@ -38,11 +38,11 @@ func TestRequestHandlerPreservesCallerAndInput(t *testing.T) { if err != nil { return nil, err } - resp.Body.Close() + _ = resp.Body.Close() return map[string]string{"answer": "ok"}, nil }) for _, bearer := range []string{"", "Bearer viewer"} { - req := httptest.NewRequest("POST", "/ask?mode=wiki", strings.NewReader(`{"question":"why?"}`)) + req := httptest.NewRequest("POST", "/ask?mode=notes", strings.NewReader(`{"question":"why?"}`)) req = req.WithContext(context.WithValue(req.Context(), requestTestKey{}, "request-context")) req.Header.Set("Authorization", bearer) rec := httptest.NewRecorder() diff --git a/served_test.go b/served_test.go new file mode 100644 index 0000000..d1ed0d7 --- /dev/null +++ b/served_test.go @@ -0,0 +1,169 @@ +package solution + +import ( + "encoding/json" + "io" + "io/fs" + "net/http" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +// Coverage of what a solution still *serves*, restored under the TLS harness. +// +// It lived in the registration test files this PR deleted, which made it look +// like coverage of registration; it is not. Nothing about these documents is +// registered any more — they are published at fixed paths and the host fetches +// them, which is why deleting them was the one thing the host asked me not to +// do — and the asset cache policy is what a browser does with a build. All of +// it survives the cutover, so all of it keeps its test. + +// TestServedDocumentsDeclareOneSetOfMajors: a reader checks both documents, so +// a bump of either constant must not leave one announcing the old major while +// the other announces the new one. +func TestServedDocumentsDeclareOneSetOfMajors(t *testing.T) { + mint := newHostMint(t, &hostMint{}) + solution := boot(t, New(Manifest{ID: testSolutionID, Title: testSolutionTitle}), mint) + + var capabilities map[string]any + readServed(t, solution, "/.well-known/capabilities", &capabilities) + var served map[string]any + readServed(t, solution, "/.well-known/solution.json", &served) + + if got, want := capabilities["schemaVersion"], served["schemaVersion"]; got != want { + t.Errorf("capabilities schemaVersion = %v, the manifest says %v", got, want) + } + frontend, ok := served["frontend"].(map[string]any) + if !ok { + t.Fatalf("the served manifest has no frontend block: %v", served) + } + if got, want := capabilities["contractMajor"], frontend["hostContract"]; got != want { + t.Errorf("capabilities contractMajor = %v, the manifest's frontend.hostContract says %v", got, want) + } + if got := capabilities["contract"]; got != testSolutionID { + t.Errorf("capabilities contract = %v, want the solution's own id: the default used to be a product's name, so every solution that left it unset announced that product's contract", got) + } + if got := frontend["manifestUrl"]; got != federationManifestPath { + t.Errorf("frontend.manifestUrl = %v, want the path on this backend", got) + } + if got := served["id"]; got != testSolutionID { + t.Errorf("served id = %v, want %q", got, testSolutionID) + } +} + +// TestAssetsAreServedWithTheRightCachePolicy: a content-hashed file never +// changes under its name, so it is cached for good; everything else — above all +// the federation manifest and the remote entry, whose names are fixed so the +// host can find them — must be revalidated, or a browser keeps a manifest +// naming chunks the redeployed solution no longer serves. A 404 for a hashed +// name, asked of an old replica mid-rollout, must not be cached for a year +// under the name the next build will serve. +func TestAssetsAreServedWithTheRightCachePolicy(t *testing.T) { + assets := t.TempDir() + writeFile(t, filepath.Join(assets, "mf-manifest.json"), `{"name":"widgets"}`) + writeFile(t, filepath.Join(assets, "page.3f9a1c2b.js"), "console.log(1)") + + t.Setenv("ASSETS_DIR", assets) + mint := newHostMint(t, &hostMint{}) + solution := boot(t, New(Manifest{ID: testSolutionID}), mint) + + for _, tc := range []struct { + path string + status int + cache string + }{ + {"/assets/page.3f9a1c2b.js", http.StatusOK, "public, max-age=31536000, immutable"}, + {"/assets/mf-manifest.json", http.StatusOK, "no-cache"}, + {"/assets/page.0badc0de.js", http.StatusNotFound, "no-cache"}, + } { + t.Run(tc.path, func(t *testing.T) { + resp, err := solution.client.Get(solution.base + tc.path) + if err != nil { + t.Fatalf("GET %s: %v", tc.path, err) + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode != tc.status { + t.Fatalf("status %d, want %d", resp.StatusCode, tc.status) + } + if got := resp.Header.Get("cache-control"); got != tc.cache { + t.Errorf("cache-control = %q, want %q", got, tc.cache) + } + if got := resp.Header.Get("access-control-allow-origin"); got == "" { + t.Error("the asset was served without CORS, so a host origin could not load it") + } + }) + } +} + +// TestTheAssetsOfAnEmbeddedBuildAreServedFromTheBinary: a solution shipping its +// frontend in the binary reads nothing from disk to serve it, so it runs with a +// read-only root filesystem and cannot drift from the build it was compiled +// with. +func TestTheAssetsOfAnEmbeddedBuildAreServedFromTheBinary(t *testing.T) { + // No assets directory on disk at all: a solution shipping its frontend in + // the binary reads nothing to serve it. + t.Setenv("ASSETS_DIR", filepath.Join(t.TempDir(), "does-not-exist")) + mint := newHostMint(t, &hostMint{}) + solution := boot(t, New(Manifest{ID: testSolutionID, Assets: embeddedBuild{}}), mint) + + resp, err := solution.client.Get(solution.base + "/assets/mf-manifest.json") + if err != nil { + t.Fatalf("GET the embedded manifest: %v", err) + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode != http.StatusOK { + t.Fatalf("status %d, want 200 served from the binary with no assets directory on disk", resp.StatusCode) + } + body, err := io.ReadAll(resp.Body) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(body), "embedded") { + t.Errorf("body = %q, want the embedded manifest", body) + } +} + +// readServed decodes one document this solution publishes. +func readServed(t *testing.T, solution *booted, path string, into any) { + t.Helper() + resp, err := solution.client.Get(solution.base + path) + if err != nil { + t.Fatalf("GET %s: %v", path, err) + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode != http.StatusOK { + t.Fatalf("GET %s = %d, want 200", path, resp.StatusCode) + } + if err := json.NewDecoder(resp.Body).Decode(into); err != nil { + t.Fatalf("decode %s: %v", path, err) + } +} + +// embeddedBuild stands in for a go:embed of a frontend build. +type embeddedBuild struct{} + +func (embeddedBuild) Open(name string) (fs.File, error) { + if name != "mf-manifest.json" { + return nil, os.ErrNotExist + } + return &embeddedFile{Reader: strings.NewReader(`{"name":"embedded"}`)}, nil +} + +type embeddedFile struct { + *strings.Reader +} + +func (f *embeddedFile) Stat() (fs.FileInfo, error) { return embeddedInfo{size: f.Size()}, nil } +func (f *embeddedFile) Close() error { return nil } + +type embeddedInfo struct{ size int64 } + +func (embeddedInfo) Name() string { return "mf-manifest.json" } +func (i embeddedInfo) Size() int64 { return i.size } +func (embeddedInfo) Mode() fs.FileMode { return 0o444 } +func (embeddedInfo) ModTime() time.Time { return time.Time{} } +func (embeddedInfo) IsDir() bool { return false } +func (embeddedInfo) Sys() any { return nil } diff --git a/solution.go b/solution.go index 7d8c3d6..d747917 100644 --- a/solution.go +++ b/solution.go @@ -1,21 +1,48 @@ // Package solution is a generic runtime for "solution" modules — independently // deployed extensions that plug into a host at runtime with no build-time // coupling. It owns everything every solution needs identically: env/config, -// self-registration with the host and the gateway (with a heartbeat), CORS, -// static Module Federation asset serving, the capability handshake, the -// solution manifest, and serving the solution's MCP server to agent clients. A -// solution author supplies a manifest and one or more handlers; each handler -// receives a Gateway that forwards the caller's bearer, and ServeMCP exposes -// the same experience to an MCP client as the signed-in person. +// obtaining this execution's credential, a listener that presents the +// workload's own identity, CORS, static Module Federation asset serving, the +// capability handshake, the solution manifest, the authority contract, and +// serving the solution's MCP server to agent clients. A solution author +// supplies a manifest and one or more handlers; each handler receives a Gateway +// that forwards the caller's bearer, and ServeMCP exposes the same experience to +// an MCP client as the signed-in person. +// +// # A runtime does not register itself +// +// It used to. Two self-registrations — the manifest to the host frontend, the +// dialable upstream to the gateway — were sent every 15s, each one burning a +// single-use credential the issuer audited as a mint, so a solution that had not +// changed in a week announced itself ~11.5k times a day and the issuer's audit +// log recorded every one of them. Presence came from a process being up, which +// means a process could claim it. +// +// Presence is now delivered: a signed presence document declares which solution +// runs on which host, and the host reconciles towards it. Authority is +// delivered the same way, by a signed document naming one approved build. This +// runtime's only outbound act at boot is to obtain its own credential once, from +// the projected service-account token the platform mounts for it, bound to the +// build it is; it renews that credential when it expires and otherwise says +// nothing to anyone. Health is answered, never pushed: the host probes the +// destination its own presence document names. +// +// Nothing bridges the two models. A runtime of this generation against a host +// that still expects registrations registers nothing; a runtime of the previous +// generation against a host of this one gets 404 on endpoints that no longer +// exist, which is the intended outcome of the cutover and not a condition either +// side recovers from. // // This package knows nothing about any specific host or solution: what it -// depends on beyond the standard library is the codefly SDK it resolves -// configuration through, Connect for the wire, and the official MCP SDK. +// depends on beyond the standard library is the Codefly SDK it resolves +// configuration through, Core, Connect for the wire, and the official MCP SDK. package solution import ( "bytes" "context" + "crypto/tls" + "crypto/x509" "encoding/json" "errors" "fmt" @@ -29,9 +56,11 @@ import ( "os" "path" "regexp" + "slices" "strconv" "strings" "sync" + "sync/atomic" "time" "unicode" @@ -39,13 +68,14 @@ import ( "github.com/codefly-dev/core/resources" "github.com/codefly-dev/core/solution/manifest" codefly "github.com/codefly-dev/sdk-go" + "github.com/codefly-dev/sdk-go/workcontext" "github.com/google/uuid" ) // Manifest is the small, solution-specific description the author provides. type Manifest struct { - ID string // logical id / gateway service alias, e.g. "lastlogin-go" - Title string // nav title, e.g. "Last Login · GO" + ID string // logical id / gateway service alias, e.g. "widgets-go" + Title string // nav title, e.g. "Widgets · GO" Order int // nav order ExposedModule string // MF exposed module (default "./Page") Contract string // capability contract id (default: ID) @@ -89,9 +119,11 @@ type Surface struct { // this runtime at all. The host's registry and the client's loader own that // boundary and have to enforce it themselves. Module string - // Contract is the surface contract major this module is built against - // (obin-ai/obin-word docs/SURFACE.md), so a client refuses a surface it - // cannot run rather than loading it and failing inside. + // Contract is the surface contract major this module is built against, as + // the host client's own surface documentation defines it, so a client + // refuses a surface it cannot run rather than loading it and failing + // inside. Named generically on purpose: this runtime is the generic one, + // and the comment used to point at one product's repository. Contract int // Applies is "always" or a {"tagged": [...]} selector, and Events are the // journal namespaces the surface reconciles on. Both are carried verbatim: @@ -117,6 +149,21 @@ var surfaceClients = map[string]bool{"word": true, "powerpoint": true, "excel": // author writes these in code, so every failure here is a mistake a boot should // name rather than a condition the runtime can recover from. func (m Manifest) validateSurfaces() error { + // The solution's own id first. Nothing refused an empty one, and it was + // load-bearing in a place it does not look load-bearing: the published + // contract carries it as `Solution`, which was also the test for whether a + // contract had been resolved at all — so an empty id switched the scope + // ceiling off while every other part of the boot reported success, and a + // ForModule asking for no scopes against an undeclared audience reached the + // host as a viewer mint. The ceiling is keyed on its own flag now, and this + // refusal closes the way to produce the state. + // + // It is also what the host's presence document names this solution by, and + // what every refusal in this package identifies the process as, so an empty + // one makes the logs of a misconfigured deployment anonymous. + if strings.TrimSpace(m.ID) == "" { + return fmt.Errorf("this solution declares no id: Manifest.ID is what the host's presence document names it by, what the published contract reports as its own, and what every refusal here identifies this process as. Set it to the module id the composition deploys") + } // Keyed by client and id together: two clients never see each other's // surfaces, so the same offering carries the same id in Word and in // PowerPoint, and only a collision within one client is ambiguous. The id @@ -231,48 +278,193 @@ type Server struct { mcp *mcpSurface mcpRedeclared bool cfg config - // registrationInterval is how long a registration heartbeat waits between - // beats. Zero means defaultRegistrationInterval. Per-server rather than a - // package value so a test can drive several beats without every other - // server in the process — including the heartbeat goroutines a finished - // test has not yet unwound — reading the same variable. - registrationInterval time.Duration + // declaredContract is the authority contract the author declared + // (Contract); contract is that contract resolved for the profile this + // process runs under, which is what it publishes. + declaredContract ModuleContract + contract effectiveContract + // contractResolved says the boot resolved that contract, which is the only + // thing that makes its ceilings governing. + // + // A flag, because the test for it was `s.contract.Solution != ""` — a + // field that is the solution's own id, so it doubled as the sentinel for + // "this contract came from a boot". An empty Manifest.ID therefore left the + // ceiling switched off while everything else about the boot succeeded, and + // ForModule accepted no scopes for an undeclared audience and minted as the + // viewer. New() refuses an empty id now as well, so there are two + // independent reasons this cannot recur; a value that is also a sentinel + // has one. + contractResolved bool + // identity is where the listener's workload identity comes from. Nil means + // the one the platform projects, read from the configured files; a consumer + // whose platform issues an identity another way supplies it with Identity. + identity IdentitySource + // identityConfig is the resolved, posture-checked configuration from that + // source: one resolution serving the listener and the outbound client, so + // the two cannot be different identities. + identityConfig *tls.Config + // credential is where this execution's credential comes from. Nil until + // the boot opens the platform's mint endpoint; a test or a consumer on + // another issuer supplies its own with Credential. + credential CredentialSource + // authority is the frozen set of authority-bearing values this process runs + // under, read once at boot and rechecked before every credential renewal. + authority *codefly.Authority + // terminal is closed the first time the issuer answers a renewal with a + // judgement no retry can change. It makes health fail and serve return, + // because a process holding a credential the host has stopped honouring + // cannot act for anyone and nothing it does locally changes that. + terminal chan struct{} + terminalOnce sync.Once + terminalErr atomic.Pointer[error] + // The one in-flight credential ask and what it last produced. See askOnce: + // without single-flight, a credential entering its renewal window turns + // every concurrent viewer request into its own audited mint. + credentialMu sync.Mutex + credentialAsk *credentialFlight + credentialAskErr error + credentialHeld workcontext.Credential + credentialQuietUntil time.Time + // credentialAskFailures counts consecutive failed acquisitions, which is + // what the backoff grows on and a success resets. + credentialAskFailures int + // boundListener is a listener this process was handed instead of binding + // one itself. Nil in every deployment: the port comes from the SDK and + // listen() binds it. + // + // It exists because a test harness cannot hand over a free *port* without + // a race. Binding :0, reading the number and closing the socket leaves a + // window in which anything else asking the OS for an ephemeral port can be + // given the same one — and this suite stands up httptest servers + // constantly, so it was given it, intermittently, in a different test each + // time. Narrowing the window with a retry was the previous attempt and + // could not close it. Handing over the listener removes the window + // entirely, because nothing is ever released. + boundListener net.Listener + // handshakeTimeout overrides platformHandshakeTimeout, for a test that + // would otherwise spend the real one. Unset means the constant, as + // firstMintWindow does for the credential window. + handshakeTimeout time.Duration + // attestation throttles what a failed attestation says (see attestWorkload). + attestation attestationReport + // outbound is the authenticated client this boot makes platform requests + // with (see outboundClient). Nil until the boot builds it. + outbound *http.Client + // firstMintWindow bounds how long the boot waits for a credential the + // issuer says is not available yet. Zero means firstMintWait. Per-server + // rather than a package value so a test can spend the window in + // milliseconds without every other server in the process reading the same + // variable. + firstMintWindow time.Duration + // principal is the principal this workload runs as, as the authority-bearing + // values the platform provisioned name it. It is what the published + // contract reports and what a delivered authority document grants to. + principal string } type config struct { - port, publicURL, gatewayURL string - hostRegisterURL string - gatewayRegisterURL string - moduleRegisterURL string - moduleTokenURL string - // solutionTokenURL is the gateway exchange that mints this solution's own - // registration credential (see solutionCredential). - solutionTokenURL string - selfUpstream, assetsDir string - internalToken string - // solutionSecret is the registration secret the composition provisioned for - // this solution — the plaintext twin of the digest the host declares for its - // id. Empty when the composition provisioned none. - solutionSecret string - // moduleSecrets maps a consumed module's facade prefix to the registration - // secret the composition provisioned for it. - moduleSecrets map[string]string - // registrationInterval is the beat period an operator chose, or zero for - // defaultRegistrationInterval; registrationIntervalErr is why an explicit - // one was not usable. - registrationInterval time.Duration - registrationIntervalErr error - // runtimeContext is the kind of runtime Codefly says this process runs - // under (CODEFLY__RUNTIME_CONTEXT), the explicit signal validate() uses to - // tell a deployed process from a local one. Empty when nothing injected it. + port string + gatewayURL string + // dialled is every address this process talks to, assembled at boot. + // + // It exists so nothing has to re-read the configuration to ask "is this + // one of ours?" — which is both a deferred configuration read the + // environment boundary refuses, and work repeated per error result. Its + // one consumer is withoutRuntimeAddresses, the net that stops a tool + // telling an MCP client an address this runtime dialled. + dialled []string + assetsDir string + // mintURL is the host endpoint that mints this execution's credential. The + // audience the projected token must be bound to is read through the SDK's + // authority reader and frozen (see openAuthority), not carried here: there + // used to be a mintAudience field beside this one, unread by anything, and + // its comment claimed both were "resolved, never defaulted" while mintURL + // does carry a default path. + mintURL string + // projectedTokenPath is the file the platform projects this workload's + // service-account token into. It is re-read before every renewal, because + // the projection is rotated under the running process and a token read once + // at boot stops verifying long before the process stops running. + projectedTokenPath string + // identityCertFile, identityKeyFile and trustBundleFile are the workload's + // X.509-SVID and the anchor its peers are verified against. The listener + // presents the first pair; there is no plain-HTTP listener to fall back to. + + identityCertFile, identityKeyFile, trustBundleFile string + // allowedCallersFile is the path to the set of identities this listener + // admits, one per line or comma-separated. Authentication says a caller + // holds a certificate from the cell's anchor; this says which of them may + // call, which is a different question and the one the host's own route + // admission answers for everybody else. + // + // A path, read per handshake, for the reason the trust anchor beside it is + // one. A snapshot of an admission decision is the same defect as a + // snapshotted trust anchor: removing a compromised consumed module from the + // provisioned set would leave this listener admitting it for the life of the + // process. Resolving the set through the SDK's value accessor on every + // handshake is not live, whatever it looks like — a + // configuration *value*, inline or file-carried, is fixed at process start + // (sdk-go file_carrier.go reads the file once and keeps it), so + // re-resolution returned the boot answer forever and the refusal branch + // below was unreachable in any deployment. The file is this process's to + // read, like the bundle, which is what makes the claim true instead of + // documented. + // + // The asymmetry is the one peerAnchor records: serving a stale *leaf* + // refuses callers who should be let in, judging by a stale *admission* lets + // in callers who should be refused. + allowedCallersFile string + // admissionConflict is set when both an override and the platform answered + // for an admission set. See conflictingAdmissionSources. + admissionConflict error + // platformPeersFile is the path to the set of identities this runtime + // accepts *outbound*: the mint and the gateway. It is the mirror of + // allowedCallersFile and it exists for the same reason — every workload in + // the cell holds a certificate from the same anchor, so verifying the chain + // and the hostname says a destination is some workload the platform issued + // for, not that it is the platform. A neighbouring workload holding a + // certificate valid for the gateway's hostname completed this handshake and + // was handed the projected token. + // mintPeersFile and gatewayPeersFile are the identities this runtime + // accepts at each of the two destinations it dials, one set per + // destination. + // + // One set for both was the previous shape, and it meant the gateway's + // identity was accepted at the mint's address and the mint's at the + // gateway's. The set already excluded every other workload in the cell, + // which was the finding it was added for — but "the platform" is not one + // party, and a set that spans two parties authorises each of them to stand + // in for the other. The maximum-security answer is one authorization set + // per destination, and a dial to anything that is neither is refused + // outright: this runtime has exactly two destinations. + // + // mintPeersFile is KEPT rather than deleted because it is consumed rather + // than reserved: sdk-go v0.3.0 owns the mint's transport, so the per-dial + // peer re-read reaches the gateway alone — but where the mint and the + // gateway answer at the SAME address this runtime cannot attribute a dial, + // and what it admits there is the intersection of both sets, so a gateway + // dial at a shared address is refused unless the mint's set admits it too. + // It is not a claim about the mint hop's posture; `credential.go` states + // what that hop does and does not have. + mintPeersFile, gatewayPeersFile string + // profile is the configuration profile this process runs under, which the + // published contract is keyed by. A deployed environment has a profile of + // its own (codefly-dev/core#687, fixed in core v0.7.1), so a contract that + // declares only "local" is refused in a deployment rather than read as if + // the deployment were a developer machine. + profile string + // runtimeContext is the kind of runtime Codefly says this process runs in, + // read from one explicit signal. It is what tells a loopback MCP issuer + // apart from a deployed one; see deployedRuntimeContext. runtimeContext string // environmentLoadErr is the failure, if any, of loading Codefly's injected - // carriers. Every SDK-resolved value below is empty when that load failed, + // carriers. Every SDK-resolved value above is empty when that load failed, // so validate() must say so rather than report each empty value as // something the composition forgot to provision. environmentLoadErr error // apiConsumes is the api.consumes projection Codefly injected, which the - // passthrough declaration is checked against before the boot listens. + // passthrough declaration and the published contract are both checked + // against before the boot listens. apiConsumes string // mcp says the solution declared an MCP surface (ServeMCP), which is what // makes the two values below load-bearing: validate() checks them only @@ -417,39 +609,43 @@ func hostAddress(ctx context.Context, module, service, endpoint, api string) str return address(ctx, module, service, endpoint, api) } -// resolveGateway resolves the host gateway's rest endpoint by role, independent of -// the host module's name (see hostAddress). saas-starter renamed this service -// auth-sidecar → auth-gateway (v0.0.49); when the current name resolves empty we -// retry the old one so a solution boots against either host version. +// resolveGateway resolves the host gateway's rest endpoint by role, independent +// of the host module's name (see hostAddress). +// +// One role, no fallback. It used to retry the pre-v0.0.49 `auth-sidecar` name +// when the current one did not resolve, so a solution booted against either +// host version — which is a compatibility path, and in this cutover an +// unresolved current gateway silently selecting a legacy service is worse than +// a boot that fails naming the role. A composition that still exposes only the +// old role is a composition to re-render, not a case to accommodate. func resolveGateway(ctx context.Context, module, gateway string) string { - if addr := hostAddress(ctx, module, gateway, "rest", "rest"); addr != "" { - return addr - } - if gateway == "auth-gateway" { - if addr := hostAddress(ctx, module, "auth-sidecar", "rest", "rest"); addr != "" { - return addr - } - } - return "" -} - -// resolveFrontend resolves the host frontend's http endpoint by role, like -// resolveGateway — host-module-name-agnostic. -func resolveFrontend(ctx context.Context, module, frontend string) string { - return hostAddress(ctx, module, frontend, "http", "http") + return hostAddress(ctx, module, gateway, "rest", "rest") } -// loadConfig resolves every address, port, and secret through the Codefly SDK -// so nothing is hardcoded. The host it plugs into is named by Codefly-convention +// loadConfig resolves every address, port, and path through the Codefly SDK so +// nothing is hardcoded. The host it plugs into is named by Codefly-convention // roles (overridable), and their concrete addresses are resolved from the SDK — // the same source `codefly endpoint` and the host services themselves use. -func loadConfig(ctx context.Context, id string) config { +// +// Nothing here resolves a credential. This runtime obtains one for itself, once, +// from the service-account token the platform projects for this workload (see +// credential.go); the shared cluster-internal token and the per-solution +// registration secret that preceded it are gone, along with the surfaces that +// accepted them. +func loadConfig(ctx context.Context, id string, environmentLoadErr error) config { // Empty by default: the host is resolved by service+endpoint role, not by its // workspace module name (see resolveGateway). An explicit CODEFLY_HOST_MODULE // scopes the lookup only when a composition is genuinely ambiguous. hostModule := env("CODEFLY_HOST_MODULE", "") - hostFrontend := env("CODEFLY_HOST_FRONTEND", "frontend") hostGateway := env("CODEFLY_HOST_GATEWAY", "auth-gateway") + hostFrontend := env("CODEFLY_HOST_FRONTEND", "frontend") + // PUBLIC_URL is back, with one consumer and a different argument behind it. + // It went with the manifest registration it fed, where building an absolute + // URL from it was wrong because the route a browser reaches this solution + // through is the host's to resolve. The MCP resource identifier an agent + // client binds its token to needs that public origin, and #49 derives it + // from this value when no MCP configuration declares one outright. + public := strings.TrimRight(env("PUBLIC_URL", ""), "/") // Own endpoint: the port Codefly assigned this service, not a fixed default. port := env("PORT", "") @@ -458,50 +654,68 @@ func loadConfig(ctx context.Context, id string) config { port = listenPort(self) } } - // Empty unless an operator set one: the manifest is then registered - // root-relative (see frontendManifestURL), never on this process's own - // loopback listen address. - public := strings.TrimRight(env("PUBLIC_URL", ""), "/") // Host endpoints, resolved via the SDK (no localhost:port literals). gatewayURL := strings.TrimRight(env("GATEWAY_URL", resolveGateway(ctx, hostModule, hostGateway)), "/") + // The host's own origin, resolved by role. Its only consumer is the MCP + // surface's default authorization server: the registrations that used to + // resolve it are gone. frontendURL := strings.TrimRight(resolveFrontend(ctx, hostModule, hostFrontend), "/") - // Both module-federation endpoints live on one gateway: the exchange mints the - // credential the registration presents, so pointing registration at another - // gateway while the exchange stayed on this one would mint against one host - // and register with a second that never trusts the result. - moduleRegisterURL := env("GATEWAY_MODULE_REGISTER_URL", gatewayURL+moduleRegisterPath) - - // Internal token: the namespaced workspace secret Codefly injects, resolved - // by name through the SDK rather than a bare os.Getenv the runtime never sees. - token := env("CODEFLY_INTERNAL_TOKEN", "") - if token == "" { - token, _ = codefly.For(ctx).WorkspaceSecret("internal-auth", "CODEFLY_INTERNAL_TOKEN") - } - - // The solution's own registration exchange lives on the same gateway as its - // registration, for the reason the module pair does: the token minted by one - // gateway is trusted by that gateway's host, not by another's. - gatewayRegisterURL := env("GATEWAY_REGISTER_URL", gatewayURL+solutionRegisterPath) - cfg := config{ - port: port, - publicURL: public, - gatewayURL: gatewayURL, - hostRegisterURL: env("HOST_REGISTER_URL", frontendURL+"/api/solutions/register"), - gatewayRegisterURL: gatewayRegisterURL, - moduleRegisterURL: moduleRegisterURL, - moduleTokenURL: env("GATEWAY_MODULE_REGISTRATION_TOKEN_URL", siblingURL(moduleRegisterURL, moduleRegisterPath, moduleRegistrationTokenPath)), - solutionTokenURL: env("GATEWAY_SOLUTION_REGISTRATION_TOKEN_URL", siblingURL(gatewayRegisterURL, solutionRegisterPath, solutionRegistrationTokenPath)), - selfUpstream: strings.TrimRight(env("SELF_UPSTREAM", selfUpstream(port)), "/"), - assetsDir: env("ASSETS_DIR", "../fe-remote/dist"), - internalToken: token, - solutionSecret: solutionRegistrationSecret(ctx), - moduleSecrets: parseModuleRegistrationSecrets(env(ModuleRegistrationSecretsEnvironmentVariable, "")), + port: port, + gatewayURL: gatewayURL, + assetsDir: env("ASSETS_DIR", "../fe-remote/dist"), + // No default, because there is nothing to default to. Whether the mint + // stays brokered through the gateway is not settled — the host prefers + // the workload to post directly to the issuer with its projected token + // bound to the issuer's own audience, and neither endpoint exists yet. + // Deriving a path on the resolved gateway and labelling it a guess is + // not an answer: this is the address + // this runtime POSTs the projected service-account token to, the + // strongest statement it can make about which workload it is, and a + // guessed address for that is the one thing fail-closed does not + // permit. It is refused at boot until something resolves it, which is + // how this package already treats a token-exchange URL it cannot pair. + mintURL: strings.TrimSpace(env(CredentialMintURLEnvironmentVariable, "")), + dialled: dialledAddresses(gatewayURL, strings.TrimSpace(env(CredentialMintURLEnvironmentVariable, ""))), + projectedTokenPath: workloadPath(ctx, ProjectedTokenFileEnvironmentVariable, WorkloadIdentityTokenFileKey), + identityCertFile: workloadPath(ctx, IdentityCertFileEnvironmentVariable, WorkloadIdentityCertFileKey), + identityKeyFile: workloadPath(ctx, IdentityKeyFileEnvironmentVariable, WorkloadIdentityKeyFileKey), + trustBundleFile: workloadPath(ctx, IdentityTrustBundleFileEnvironmentVariable, WorkloadIdentityTrustBundleFileKey), + allowedCallersFile: workloadPath(ctx, IdentityAllowedCallersFileEnvironmentVariable, WorkloadIdentityAllowedCallersFileKey), + mintPeersFile: workloadPath(ctx, IdentityMintPeersFileEnvironmentVariable, WorkloadIdentityMintPeersFileKey), + gatewayPeersFile: workloadPath(ctx, IdentityGatewayPeersFileEnvironmentVariable, WorkloadIdentityGatewayPeersFileKey), + profile: strings.TrimSpace(env(ContractProfileEnvironmentVariable, codefly.Environment())), runtimeContext: strings.TrimSpace(env(resources.RuntimeContextPrefix, "")), apiConsumes: env(manifest.APIConsumesEnvironmentVariable, ""), - } + // Carried here, which the refactor that changed this function's + // signature dropped: three refusals and a boot log exist to tell an + // operator "the SDK resolved nothing" apart from "loading the injected + // environment failed first", and with this left unset they all read as + // the former. Sending someone to inspect provisioning over a variable + // they broke themselves is the specific mistake validate() was written + // to stop making. + environmentLoadErr: environmentLoadErr, + } + // Two sources for one authorization fact are refused, not ranked. + // + // Every other path here takes the override first and the provisioned value + // second, which is right for a path: an operator with a deployment the + // resolver cannot see needs the escape hatch, and the worst case is this + // process reading its own material from somewhere else. Admission is not + // that. An override that silently outranks the platform's answer is how a + // caller set gets widened with nothing recording that the platform's + // decision was not the one in force — and logging which one won, which is + // what the previous revision did, reports the conflict without resolving + // it. + // + // So: either source may answer, and both answering is an error naming + // both. It is the stance the SDK already takes on the same question — a + // value delivered inline and by file carrier is "two sources for one fact + // and are refused" (file_carrier.go) — applied to the one kind of value + // where being wrong admits a caller. + cfg.admissionConflict = conflictingAdmissionSources(ctx, environmentLoadErr) // The host is the authorization server an MCP client authenticates against, // so its issuer is the host's own origin — resolved by role like every // other host endpoint, never typed. The resolved address is the one this @@ -511,261 +725,674 @@ func loadConfig(ctx context.Context, id string) config { // needs no declaration at all, being derived from the two values above it. cfg.mcpIssuerURL, cfg.mcpIssuerExplicit = resolveMCPIssuer(ctx, frontendURL) cfg.mcpPublicURL, cfg.mcpPublicExplicit = resolveMCPPublicURL(ctx, public, id) - cfg.registrationInterval, cfg.registrationIntervalErr = registrationIntervalFromEnv() return cfg } -// selfEndpointCarrierPrefix names the carrier Codefly injects with this -// service's own endpoint as the rest of the deployment reaches it — its -// in-cluster address, e.g. "http://backend..svc.cluster.local:8080". -// It sits beside the CODEFLY__ENDPOINT__ carrier for the same endpoint, which -// stays what it is: the address this process listens on ("localhost:8080" in a -// rendered cell), right for binding and wrong for anyone else to dial. -// -// PENDING codefly-dev/core v0.5.6, which adds this carrier. It is read here by -// name, in this one helper, only until core/sdk-go release an accessor for it; -// switch selfEndpoint to that accessor when they do. -const selfEndpointCarrierPrefix = "CODEFLY__SELF_ENDPOINT" - -// selfEndpoint is this service's reachable http endpoint from the carrier -// above, or "" when none was injected (a local run, or a render from before -// core v0.5.6). The key is normalised exactly as the CODEFLY__ENDPOINT__ -// carriers are. -func selfEndpoint() string { - module, service := os.Getenv(resources.ModulePrefix), os.Getenv(resources.ServicePrefix) - if module == "" || service == "" { - return "" +// conflictingAdmissionSources is the error validate() reports when any value in +// the identity group is answered by both an operator override and the +// platform's provisioning. +// +// All of them, not only the two admission sets. The override that outranks the +// platform silently was found first on the caller set, and the same argument +// reaches the rest of the group: a substituted *trust bundle* widens admission +// exactly as much as a substituted caller list, and a substituted key pair +// changes who this process is. The two sets were simply where it was noticed. +func conflictingAdmissionSources(ctx context.Context, loadErr error) error { + for _, admission := range []struct{ what, override, key string }{ + {"allowed callers", IdentityAllowedCallersFileEnvironmentVariable, WorkloadIdentityAllowedCallersFileKey}, + {"credential mint peer identity", IdentityMintPeersFileEnvironmentVariable, WorkloadIdentityMintPeersFileKey}, + {"gateway peer identity", IdentityGatewayPeersFileEnvironmentVariable, WorkloadIdentityGatewayPeersFileKey}, + {"peer trust anchor", IdentityTrustBundleFileEnvironmentVariable, WorkloadIdentityTrustBundleFileKey}, + {"workload identity certificate", IdentityCertFileEnvironmentVariable, WorkloadIdentityCertFileKey}, + {"workload identity private key", IdentityKeyFileEnvironmentVariable, WorkloadIdentityKeyFileKey}, + {"projected service-account token", ProjectedTokenFileEnvironmentVariable, WorkloadIdentityTokenFileKey}, + } { + overridden := strings.TrimSpace(env(admission.override, "")) + if overridden == "" { + continue + } + provisioned, err := codefly.For(ctx).WorkspaceConfiguration(WorkloadIdentityGroup, admission.key) + if err != nil { + // An error here is NOT evidence of a second answer, and that is + // the awkward part: the SDK returns the same "no workspace + // configuration value" whether the key was never provisioned or + // the carriers holding it never loaded. So this cannot refuse on + // the error alone — doing that refuses every legitimate override, + // which is what it did when first written. + // + // The one signal that separates them is whether the environment + // loaded at all, which is why loadConfig keeps that error. With a + // clean load, an error means genuinely absent and the override + // stands alone. With a failed load, nothing the SDK says is + // absent can be trusted to be absent, and an override put in force + // against provisioning this process could not see is the conflict + // the rule exists to refuse. + if loadErr != nil { + return fmt.Errorf("the %s is overridden by %s while loading Codefly's injected environment failed (%w), so the platform's own answer for %s/%s cannot be read: this process will not put an override in force against provisioning it cannot see, because the two may disagree and the one it ignored would be the decision nobody applied. Fix the environment load, or unset the override", + admission.what, admission.override, loadErr, WorkloadIdentityGroup, admission.key) + } + continue + } + if strings.TrimSpace(provisioned) == "" { + continue + } + return fmt.Errorf("the %s is answered twice: %s names %q and the platform provisioned %s/%s as %q. A value this process is held to with two sources is refused rather than ranked — whichever it picked, the other is a decision somebody made that is not in force. Unset the override, or remove the provisioning", + admission.what, admission.override, overridden, WorkloadIdentityGroup, admission.key, strings.TrimSpace(provisioned)) } - key := selfEndpointCarrierPrefix + "__" + resources.EndpointAsEnvironmentVariableKeyBase(&resources.EndpointInformation{ - Module: module, Service: service, Name: "http", API: "http", - }) - return strings.TrimSpace(os.Getenv(key)) + return nil } -// selfUpstream is the upstream this solution registers with the gateway: the -// address the gateway dials to proxy to it. It is the reachable self endpoint -// when Codefly injected one, and otherwise this process's own listen address — -// correct only when the gateway runs on the same machine, which is why -// validate() refuses a loopback upstream in a deployed runtime context rather -// than letting the gateway proxy the solution to itself. +// WorkloadIdentityGroup is the workspace configuration group through which the +// platform tells this workload where its own identity material lives: the +// projected service-account token it mints its credential with, the X.509-SVID +// key pair its listener presents, and the anchor its peers are verified +// against. // -// It used to be the public URL, which defaulted to that same loopback listen -// address, so every deployed solution registered "http://localhost:8080" and -// the gateway, dialling its own localhost, proxied the solution to itself. -func selfUpstream(port string) string { - if self := selfEndpoint(); self != "" { - return self - } - if port == "" { - return "" +// Paths, never material, and no exception: every value this group carries is a +// file path, and the file behind it is read by this process — the token on every +// renewal, the key pair and the anchor on every handshake, the two admission +// sets on every handshake and dial. The SDK's own value accessors would be the +// wrong tool for any of it: a file-carried configuration value is read once and +// kept for the life of the process (sdk-go file_carrier.go), which is exactly +// right for a configuration value and exactly wrong for anything the platform +// changes under a running process. +// +// The two admission sets were the exception, carried as values here and +// re-resolved through that accessor on every handshake, which looked live and +// was not: the accessor returns what the process started with, so revoking a +// caller took a restart while the code and the README both said it did not. +// Material the platform can change is a path in this group or it is a snapshot +// pretending otherwise. +const WorkloadIdentityGroup = "workload-identity" + +// The keys of WorkloadIdentityGroup. Each has an explicit environment override +// below, for an operator with a projection the resolver cannot see. +const ( + WorkloadIdentityTokenFileKey = "TOKEN_FILE" + WorkloadIdentityCertFileKey = "CERT_FILE" + WorkloadIdentityKeyFileKey = "KEY_FILE" + WorkloadIdentityTrustBundleFileKey = "TRUST_BUNDLE_FILE" + // WorkloadIdentityAllowedCallersFileKey names the *file* listing the + // identities allowed to call this solution. See allowedCallersFile for why + // it is a path and not the list: a provisioned value is fixed at process + // start, and an admission decision that cannot narrow while the process + // runs is a decision this listener would keep honouring after it was + // revoked. + WorkloadIdentityAllowedCallersFileKey = "ALLOWED_CALLERS_FILE" + // WorkloadIdentityMintPeersFileKey and + // WorkloadIdentityGatewayPeersFileKey name the *files* listing the + // identities this runtime will present its credentials to, one per + // destination. Paths, for the reason above. + // + // Two keys rather than one, because the mint and the gateway are two + // parties and a single set lets each stand in for the other at the other's + // address. + WorkloadIdentityMintPeersFileKey = "MINT_PEERS_FILE" + WorkloadIdentityGatewayPeersFileKey = "GATEWAY_PEERS_FILE" +) + +// The environment overrides for the paths above. Each names a file, and +// loadConfig logs which source answered for the two admission paths — an +// override that quietly outranked the platform's provisioning was the one way +// an operator could widen admission without it appearing anywhere. +const ( + ProjectedTokenFileEnvironmentVariable = "CODEFLY__WORKLOAD_TOKEN_FILE" + IdentityCertFileEnvironmentVariable = "CODEFLY__WORKLOAD_IDENTITY_CERT_FILE" + IdentityKeyFileEnvironmentVariable = "CODEFLY__WORKLOAD_IDENTITY_KEY_FILE" + IdentityTrustBundleFileEnvironmentVariable = "CODEFLY__WORKLOAD_IDENTITY_TRUST_BUNDLE_FILE" + // IdentityMintPeersFileEnvironmentVariable and + // IdentityGatewayPeersFileEnvironmentVariable override the files listing + // the identities this runtime accepts at each destination. + IdentityMintPeersFileEnvironmentVariable = "CODEFLY__WORKLOAD_IDENTITY_MINT_PEERS_FILE" + IdentityGatewayPeersFileEnvironmentVariable = "CODEFLY__WORKLOAD_IDENTITY_GATEWAY_PEERS_FILE" + // IdentityAllowedCallersFileEnvironmentVariable overrides the file listing + // the identities allowed to call this solution. + IdentityAllowedCallersFileEnvironmentVariable = "CODEFLY__WORKLOAD_IDENTITY_ALLOWED_CALLERS_FILE" +) + +// CredentialMintURLEnvironmentVariable is the host endpoint this runtime mints +// its execution credential at. Required: there is no default, because the +// address this runtime sends its projected service-account token to is not an +// address it may assume. +const CredentialMintURLEnvironmentVariable = "CODEFLY__CREDENTIAL_MINT_URL" + +// ContractProfileEnvironmentVariable overrides the configuration profile the +// published contract is read under. Unset, it is the Codefly environment's own +// name, which is how Core resolves a profile for an environment that declares +// none (resources.Environment.ConfigurationProfileNames). A deployed +// environment therefore reads its own profile rather than the developer +// machine's — the gap codefly-dev/core#687 named and core v0.7.1 closed. +const ContractProfileEnvironmentVariable = "CODEFLY__CONTRACT_PROFILE" + +// credentialMintPath is the path this package's own tests mount their fake mint +// on. It is NOT a default: nothing in loadConfig derives an address from it, +// because the host has not settled brokered-versus-direct and a guessed mint +// address is refused rather than assumed. It exists so the suite's fixtures +// agree with each other. +const credentialMintPath = "/platform/_credential" + +// workloadPath resolves one of the identity paths above: an explicit +// environment override first, then the workspace configuration group the +// platform populates. Empty means neither resolved, which validate() reports by +// name — there is no default, because a default would be a path that is true on +// one platform's pod spec and nowhere else. +func workloadPath(ctx context.Context, override, key string) string { + if value := strings.TrimSpace(env(override, "")); value != "" { + return value } - return "http://localhost:" + port + // The SDK's error is dropped because it carries no information: + // WorkspaceConfiguration returns the same "no workspace configuration + // value" error whether the group was never declared or the carriers it + // lives in never loaded. The one signal that does distinguish them is + // whether loading the environment failed at all, which Serve records in + // config.environmentLoadErr for validate() to report. + value, _ := codefly.For(ctx).WorkspaceConfiguration(WorkloadIdentityGroup, key) + return strings.TrimSpace(value) } -// deployedRuntimeContext reports whether Codefly says this process runs in a -// deployment rather than on a developer machine. The signal is explicit — -// CODEFLY__RUNTIME_CONTEXT, which core's GitOps render injects (e.g. -// "kubernetes") — and never the environment name: an environment called -// "local-dogfood" or "staging" says nothing about where the process runs. -// Every runtime context `codefly run` uses on a developer machine is local; -// any other declared context is a deployment, so a new deployed kind is -// covered without a change here. Nothing declared means not deployed. +// validate rejects a config the runtime cannot actually serve under. When +// neither the SDK nor an explicit env override resolves a value, loadConfig +// leaves it empty; without this check Serve would bind ":"+"" — which the kernel +// happily accepts as a random port — and would mint against a scheme-less URL +// like "/platform/_credential" that http.Client.Do rejects. Both are the exact +// silent no-op this whole SDK resolution effort exists to eliminate, so an +// unresolved config must fail loud at boot rather than come up looking healthy +// on the wrong port. // -// PENDING codefly-dev/core v0.5.6, which injects the signal into renders: -// until a cell is rendered with it, this reports false there too. -func deployedRuntimeContext(kind string) bool { - switch strings.ToLower(kind) { - case "", resources.RuntimeContextNative, resources.RuntimeContextNix, - resources.RuntimeContextContainer, resources.RuntimeContextFree: - return false +// Every refusal names the value and the provisioning path that fixes it, and +// each cause is separated from the ones it looks like: "the SDK resolved +// nothing" is a different sentence from "loading the injected environment +// failed first", because one generic message sent an operator to inspect +// endpoint resolution over a variable they had broken themselves. +func (c config) validate() error { + // First, because it is the one refusal that says a decision somebody made + // is not the one in force, and every other message below would read as + // ordinary provisioning advice next to it. + if c.admissionConflict != nil { + return c.admissionConflict } - return true + if p, err := strconv.Atoi(c.port); err != nil || p < 1 || p > 65535 { + return fmt.Errorf("unresolved listen port %q: set PORT or ensure the SDK resolves this service's http endpoint", c.port) + } + // Both of these carry credentials: the mint presents the projected token + // that attests which workload this process is, and the gateway carries the + // viewer's bearer and the capability minted for them. So each has to be + // resolved *and* has to be a destination this process can authenticate and + // cannot be read off the wire. + for _, required := range []struct{ name, value, override string }{ + {"gateway URL", c.gatewayURL, "GATEWAY_URL"}, + {"credential mint URL", c.mintURL, CredentialMintURLEnvironmentVariable}, + } { + u, err := url.Parse(required.value) + if err != nil || !u.IsAbs() || u.Host == "" { + if c.environmentLoadErr != nil { + return fmt.Errorf("unresolved %s %q, and loading Codefly's injected environment failed first: %w — nothing the SDK resolves can be trusted to be absent until that is fixed", + required.name, redactedURL(required.value), c.environmentLoadErr) + } + if required.name == "credential mint URL" { + return fmt.Errorf("no credential mint URL resolved: this runtime POSTs the projected service-account token that attests which workload it is to this address, so there is no default for it — a path assumed on the resolved gateway would be sending that attestation to an endpoint nobody published. Set %s to the endpoint this host actually serves. The fix that removes the override is the host publishing its mint as a resolvable endpoint role, which belongs to the host and the SDK rather than here", + CredentialMintURLEnvironmentVariable) + } + return fmt.Errorf("unresolved %s %q: the SDK could not resolve the host endpoint and no explicit override was set", required.name, redactedURL(required.value)) + } + // Plaintext is refused rather than warned about. A TLS listener + // protects what callers send *to* this process and nothing this + // process sends out: over http://, the mint hands this workload's + // projected service-account token to anything on the path, and a + // viewer's mint hands over their bearer and the capability minted for + // them. The listener being TLS-only made the inbound hop safe and left + // the outbound ones exactly as they were. + if u.Scheme != "https" { + // Redacted here too: a refusal is a log line like any other, and + // the URL being refused is exactly the one most likely to carry + // something that should not be written down. + return fmt.Errorf("%s %q is not https: it carries %s, so a plaintext hop hands them to anything on the path. Set %s to an https destination", + required.name, redactedURL(required.value), credentialsCarriedOn(required.name), required.override) + } + // And nothing but a destination. These three were accepted, and the + // first two were then written to the log at every boot: + // + // - userinfo (https://ops:s3cr3t@mint.cell/…) is a credential in a + // URL, which net/http also strips from the request it sends, so it + // is a secret that is logged and never used; + // - a query (…?token=s3cr3t) is the same shape, and it travels; + // - a fragment is never sent at all, so a destination carrying one + // is not the destination somebody meant. + // + // A boot refusal is the right answer rather than quietly trimming + // them: an operator who put a token in this URL believes it is doing + // something. + switch { + case u.User != nil: + return fmt.Errorf("%s carries userinfo: credentials in a URL are a secret that this runtime would log at boot and net/http would strip from the request, so it is disclosed and never used. Put the host in %s and nothing else", + required.name, required.override) + case u.RawQuery != "": + return fmt.Errorf("%s carries a query string: this address is dialled, not templated, and anything secret in a query is logged by every hop that sees the URL. Put the host and path in %s and nothing else", + required.name, required.override) + case u.Fragment != "": + return fmt.Errorf("%s carries a fragment: a fragment is never sent, so this is not the destination it appears to be. Set %s to the address the host actually serves", + required.name, required.override) + } + } + if err := resources.ValidateConfigurationProfileName(c.profile); err != nil { + return fmt.Errorf("unusable contract profile %q: %w — it is the Codefly environment's own name unless %s overrides it", + c.profile, err, ContractProfileEnvironmentVariable) + } + // The MCP surface's own configuration, checked here because that is where + // boot configuration is checked. It is a no-op for a solution that declared + // no MCP surface, which is what keeps a host whose issuer does not resolve + // from refusing the boot of a solution that serves none. + if err := c.validateMCP(); err != nil { + return err + } + c.logResolved() + return nil } -// loopbackURL reports whether raw names this machine: localhost (or a name -// under .localhost), a loopback IP, or the unspecified address. Such a URL is -// reachable only from the process's own host. -func loopbackURL(raw string) bool { - u, err := url.Parse(raw) +// logResolved says once, at boot, what this process resolved. +// +// A function of its own so what it writes can be read in a test. The redaction +// is the property, and asserting on the helper that does it left the call site +// free to pass the raw URL — which is where the secret was actually being +// written. +func (c config) logResolved() { + if c.environmentLoadErr != nil { + // Everything above resolved, so the load failure cost nothing this boot + // needs — but it is still the reason a later SDK read may come back + // empty, and it has to be said once rather than inferred from the next + // surprise. + log.Printf("codefly: the injected environment did not load cleanly; every SDK-resolved value that is present came from an override: %v", c.environmentLoadErr) + } + // Said at every boot: the operator set this address by hand, because + // nothing resolves it yet, so a 404 here is the endpoint not existing on + // this host rather than this build being refused. + // + // Redacted, always. validate() refuses userinfo and a query, so there + // should be nothing to hide — and a log line is the wrong place to depend + // on a check that runs elsewhere. + log.Printf("codefly: this execution's credential will be minted at %q, which was set explicitly through %s — no resolver produces it yet, so a 404 here means that endpoint does not exist on this host, not that this build was refused.", + redactedURL(c.mintURL), CredentialMintURLEnvironmentVariable) +} + +// redactedURL is a URL as it may appear in a log: url.Redacted() replaces a +// password with "xxxxx", and a query is dropped entirely, because a token in a +// query is the shape this package refuses at boot and a log line must not be +// the place that depends on that refusal having run. +func redactedURL(raw string) string { + parsed, err := url.Parse(raw) if err != nil { - return false + return "(unparseable)" } - host := strings.ToLower(strings.TrimSuffix(u.Hostname(), ".")) - if host == "localhost" || strings.HasSuffix(host, ".localhost") { - return true - } - ip := net.ParseIP(host) - return ip != nil && (ip.IsLoopback() || ip.IsUnspecified()) -} - -// RegistrationIntervalEnvironmentVariable sets how long a registration -// heartbeat waits between beats, as a Go duration ("30s", "2m"). -// -// It exists because a beat stopped being free. Each self-registration beat now -// runs a credential exchange whose every success is an audited mint on the -// issuer — at the 15s default, two surfaces, that is ~11.5k audited mints per -// solution per day, and the figure scales with the number of solutions. The -// right value is whatever the host's registration TTL allows, which this -// runtime cannot observe, so the default is unchanged and the lever belongs to -// whoever knows both numbers. -const RegistrationIntervalEnvironmentVariable = "CODEFLY__SOLUTION_REGISTRATION_INTERVAL" - -// registrationIntervalMinimum floors an explicit interval. Below this the beat -// costs more in audited mints than any freshness it buys, and a typo ("100" -// parsed as 100ns) would otherwise turn the heartbeat into a mint loop. -const registrationIntervalMinimum = time.Second - -// registrationIntervalFromEnv reads an explicit beat period. It returns zero -// for "unset", and an error — never a silent fallback — for a value that is set -// but unusable, because an operator who asked for a slower beat to bound their -// mint rate must not get the fast default because of a typo. -func registrationIntervalFromEnv() (time.Duration, error) { - raw := env(RegistrationIntervalEnvironmentVariable, "") - if raw == "" { - return 0, nil - } - interval, err := time.ParseDuration(raw) - if err != nil { - return 0, fmt.Errorf("unparseable %s %q: want a Go duration such as %q or %q: %w", - RegistrationIntervalEnvironmentVariable, raw, "30s", "2m", err) + // An OPAQUE url prints nothing of itself. + // + // url.Parse("ops:SECRET@host/path") is legal and gives scheme "ops" with + // everything after the colon as Opaque — so User is nil, RawQuery is + // empty, and the three strips below touch nothing. The whole secret came + // through this function unchanged, which is the one thing it exists to + // stop. There is no safe way to describe such a value, because its + // content is arbitrary by definition, so only the scheme is named: enough + // for an operator to recognise what they set and nothing more. + if parsed.Opaque != "" || parsed.Host == "" { + if parsed.Scheme != "" { + return parsed.Scheme + ":(redacted)" + } + return "(redacted)" } - if interval < registrationIntervalMinimum { - return 0, fmt.Errorf("%s %q is below the %s minimum: a beat this fast mints a registration token per surface per beat, each an audited event on the issuer", - RegistrationIntervalEnvironmentVariable, raw, registrationIntervalMinimum) + // Userinfo dropped entirely, not redacted. url.Redacted() masks the + // *password* and keeps the username — so a token carried as the username, + // which is how a great many of them are carried, came through it intact. + parsed.User = nil + parsed.RawQuery, parsed.Fragment = "", "" + return parsed.String() +} + +// credentialsCarriedOn names what a plaintext destination would disclose, so +// the refusal says why https is not a preference. +func credentialsCarriedOn(name string) string { + if name == "credential mint URL" { + return "the service-account token the platform projects for this workload" } - return interval, nil + return "the viewer's bearer and the capability minted on their behalf" } -// solutionRegistrationSecret resolves the registration secret the composition -// provisioned for this solution: an explicit environment override first, then -// the namespaced workspace secret Codefly injects — resolved by name through the -// SDK, exactly as the cluster-internal token is, so the same declaration serves -// a local `codefly run` and a deployed cell. Empty means none resolved. +// validateSources checks the material the *selected* sources need, which is not +// the same set for every boot. // -// The SDK's error is dropped because it carries no information: WorkspaceSecret -// returns the same "no workspace configuration value" error whether the group -// was never declared or the carriers it lives in never loaded. The one signal -// that does distinguish them is whether loading the environment failed at all, -// which Serve records in config.environmentLoadErr for validate() to report. -func solutionRegistrationSecret(ctx context.Context) string { - if secret := env(SolutionRegistrationSecretEnvironmentVariable, ""); secret != "" { - return trimmedSecret(secret) +// A consumer that supplied an IdentitySource has said where its identity comes +// from — a mesh CA, a SPIFFE workload API — and requiring the projected file +// paths from it as well would be requiring provisioning nothing reads. The same +// goes for a credential source and the projected token. What no source changes +// is the authority identity: the principal, the mint audience and the +// projection audience are read once and frozen on every path, because the +// published contract names the principal and the listener is held to it, and a +// hook is not a way out of either. +func (s *Server) validateSources() error { + if s.identity == nil { + for _, path := range []struct{ name, value, override, key string }{ + {"workload identity certificate", s.cfg.identityCertFile, IdentityCertFileEnvironmentVariable, WorkloadIdentityCertFileKey}, + {"workload identity private key", s.cfg.identityKeyFile, IdentityKeyFileEnvironmentVariable, WorkloadIdentityKeyFileKey}, + {"workload identity trust anchor", s.cfg.trustBundleFile, IdentityTrustBundleFileEnvironmentVariable, WorkloadIdentityTrustBundleFileKey}, + } { + if err := s.cfg.requirePath(path.name, path.value, path.override, path.key); err != nil { + return err + } + } + } + if s.credential == nil { + if err := s.cfg.requirePath("projected service-account token", s.cfg.projectedTokenPath, + ProjectedTokenFileEnvironmentVariable, WorkloadIdentityTokenFileKey); err != nil { + return err + } + } + // Who may call is required whatever the identity source is, because it is + // not a property of the source: the source says who this workload *is*, and + // this says which callers it serves. + if err := s.cfg.requirePath("allowed caller identities", s.cfg.allowedCallersFile, + IdentityAllowedCallersFileEnvironmentVariable, WorkloadIdentityAllowedCallersFileKey); err != nil { + return fmt.Errorf("%w — this listener verifies a caller's certificate against the cell's trust anchor, which every workload in the cell holds one from, so without this it authenticates callers and authorizes all of them, including the modules this solution consumes, which could then set their own %s and %s and drive mints carrying this workload's attestation", + err, orgHeader, sessionHeader) + } + // And whom this runtime may present its credentials to, for the same + // reason in the other direction. It is required on every path, including a + // supplied identity source: a source says who this workload is, not which + // destinations are the platform. + for _, peers := range []struct{ name, value, override, key string }{ + {"credential mint peer identity", s.cfg.mintPeersFile, IdentityMintPeersFileEnvironmentVariable, WorkloadIdentityMintPeersFileKey}, + {"gateway peer identity", s.cfg.gatewayPeersFile, IdentityGatewayPeersFileEnvironmentVariable, WorkloadIdentityGatewayPeersFileKey}, + } { + if err := s.cfg.requirePath(peers.name, peers.value, peers.override, peers.key); err != nil { + return fmt.Errorf("%w — this runtime presents the projected service-account token, the viewer's bearer and its own credential to these destinations, and verifying their chain and hostname says only that a destination holds a certificate this cell issued, which every workload in the cell does. Each destination has its own set, so neither party can stand in for the other at the other's address", + err) + } + } + // And both files have to be readable and name somebody *now*. The per-use + // readers refuse a handshake over an unreadable set, which is the right + // answer while serving and the wrong first symptom at boot: a path nobody + // provisioned would otherwise start a listener that refuses every caller + // and a client that refuses every dial, and report it as an admission + // failure once traffic arrived rather than as the provisioning gap it is. + for _, set := range []struct { + what string + admits func() ([]string, error) + }{ + {"allowed callers", s.admittedCallers()}, + {"credential mint peer identity", s.admittedMint()}, + {"gateway peer identity", s.admittedGateway()}, + // And, when the two share an address, the set that actually governs a + // dial there: an empty intersection is a provisioning gap an operator + // can fix, and discovering it at the first mint instead would read as + // the mint being unreachable. + {"identities admitted at the address the mint and the gateway share", s.admittedAt(dialAddress(s.cfg.mintURL))}, + } { + if _, err := set.admits(); err != nil { + return fmt.Errorf("the provisioned %s are not usable at boot: %w", set.what, err) + } } - secret, _ := codefly.For(ctx).WorkspaceSecret(SolutionRegistrationSecretGroup, SolutionRegistrationSecretKey) - return trimmedSecret(secret) + return nil } -// trimmedSecret strips the whitespace a secret store puts around a value — -// above all the trailing newline a file-mounted or `echo`-generated secret -// almost always carries. +// admittedCallers resolves the identities this listener serves, for one +// handshake, failing closed. // -// This is not cosmetic. The secret is sent as an HTTP header value, and net/http -// refuses to write a header containing a newline: the request never leaves the -// process, so the beat fails with "invalid header field value" on every attempt, -// forever, while validate() saw a non-empty secret and let the boot through. The -// registration silently never happens for a reason no amount of checking the -// provisioning would reveal, because the provisioning is in fact correct. -func trimmedSecret(secret string) string { - return strings.TrimSpace(secret) +// A set that resolves to nothing refuses the handshake, and so does a file that +// cannot be read. It is the same judgement peerAnchor makes about an unreadable +// trust bundle: the alternative is serving the set this process booted with, +// which is the staleness this resolution exists to remove, and an admission +// decision that cannot be resolved is not an admission decision to guess at. +func (s *Server) admittedCallers() func() ([]string, error) { + return resolvedIdentities(s.cfg.allowedCallersFile, fmt.Sprintf( + "whom this listener admits could not be resolved, so the handshake is refused rather than served against the set this process booted with. Provision %s/%s, or set %s", + WorkloadIdentityGroup, WorkloadIdentityAllowedCallersFileKey, IdentityAllowedCallersFileEnvironmentVariable)) } -// validate rejects a config the runtime cannot actually serve or register with. -// When neither the SDK nor an explicit env override resolves a value, loadConfig -// leaves it empty; without this check Serve would bind ":"+"" — which the kernel -// happily accepts as a random port — and POST registrations to scheme-less URLs -// like "/solutions/_register" that http.Client.Do rejects and the heartbeat then -// retries forever in silence. Both are the exact silent no-op this whole SDK -// resolution effort exists to eliminate, so an unresolved config must fail loud -// at boot rather than come up looking healthy on the wrong port. -func (c config) validate() error { - if p, err := strconv.Atoi(c.port); err != nil || p < 1 || p > 65535 { - return fmt.Errorf("unresolved listen port %q: set PORT or ensure the SDK resolves this service's http endpoint", c.port) +// admittedMint and admittedGateway are admittedCallers for each of the two +// destinations this runtime dials, one authorization set each. +func (s *Server) admittedMint() func() ([]string, error) { + return resolvedIdentities(s.cfg.mintPeersFile, fmt.Sprintf( + "which identity answers for the credential mint could not be resolved, and that destination receives the projected service-account token, so the dial is refused. Provision %s/%s, or set %s", + WorkloadIdentityGroup, WorkloadIdentityMintPeersFileKey, IdentityMintPeersFileEnvironmentVariable)) +} + +func (s *Server) admittedGateway() func() ([]string, error) { + return resolvedIdentities(s.cfg.gatewayPeersFile, fmt.Sprintf( + "which identity answers for the gateway could not be resolved, and that destination receives the viewer's bearer and the capability minted for them, so the dial is refused. Provision %s/%s, or set %s", + WorkloadIdentityGroup, WorkloadIdentityGatewayPeersFileKey, IdentityGatewayPeersFileEnvironmentVariable)) +} + +// admittedAt is the authorization set for one destination address, and an error +// for an address that is neither. +// +// A dial to a third address is refused rather than held to some union: this +// runtime talks to the mint and the gateway and to nothing else, so an address +// that is neither is a misbuilt client or a redirect that got through, and +// there is no set that should admit it. +func (s *Server) admittedAt(addr string) func() ([]string, error) { + mint, gateway := dialAddress(s.cfg.mintURL), dialAddress(s.cfg.gatewayURL) + // One address for both is the brokered shape, and it is the shape this + // runtime is actually deployed in today: the mint has lived on the + // gateway's host and port. Switching on the mint first silently made every + // gateway dial answer to the mint's set, and the gateway's set was then + // validated at boot and never read again — so a gateway set naming + // somebody else let the gateway through, and rewriting it cut nothing. + // The split was not in force in the only configuration that exists. + // + // The intersection, because a dial to a shared address cannot be + // attributed: this runtime does not know whether it is talking to the mint + // or to the gateway, so the only identity it can safely present + // credentials to is one BOTH sets admit. An empty intersection admits + // nobody, which validate() reports at boot rather than leaving to the + // first dial. + switch { + case mint == gateway && addr == mint: + return intersectedIdentities(s.admittedMint(), s.admittedGateway(), addr) + case addr == mint: + return s.admittedMint() + case addr == gateway: + return s.admittedGateway() } - required := map[string]string{ - "gateway URL": c.gatewayURL, - "host register URL": c.hostRegisterURL, - "gateway register URL": c.gatewayRegisterURL, - "module register URL": c.moduleRegisterURL, - "module token URL": c.moduleTokenURL, - "solution token URL": c.solutionTokenURL, - "self upstream": c.selfUpstream, - } - // A token URL is not resolved from the SDK at all: it is derived from its - // register URL by siblingURL, which yields "" for a base it cannot pair. - // Reported with the generic message below, that sent an operator looking at - // SDK endpoint resolution for a value their own override had broken — and - // the override is a variable whose shape only started mattering when the - // exchange began deriving from it, so a deployment that booted yesterday - // can stop booting today with no mention of the variable that changed. - unpairable := map[string]string{ - "module token URL": "GATEWAY_MODULE_REGISTER_URL must end in " + moduleRegisterPath + " for its exchange to be derived from it; set GATEWAY_MODULE_REGISTRATION_TOKEN_URL explicitly otherwise", - "solution token URL": "GATEWAY_REGISTER_URL must end in " + solutionRegisterPath + " for its exchange to be derived from it; set GATEWAY_SOLUTION_REGISTRATION_TOKEN_URL explicitly otherwise", - } - for name, raw := range required { - if u, err := url.Parse(raw); err != nil || !u.IsAbs() || u.Host == "" { - if c.environmentLoadErr != nil { - return fmt.Errorf("unresolved %s %q, and loading Codefly's injected environment failed first: %w — nothing the SDK resolves can be trusted to be absent until that is fixed", - name, raw, c.environmentLoadErr) + return func() ([]string, error) { + return nil, fmt.Errorf("refusing to present this workload's credentials at %s: this runtime dials the credential mint (%s) and the gateway (%s) and nothing else, so an address that is neither has no authorization set and is not a destination this runtime has one for", + addr, mint, gateway) + } +} + +// intersectedIdentities is the identities both sets admit, for an address the +// two destinations share. +func intersectedIdentities(first, second func() ([]string, error), addr string) func() ([]string, error) { + return func() ([]string, error) { + mint, err := first() + if err != nil { + return nil, err + } + gateway, err := second() + if err != nil { + return nil, err + } + var both []string + for _, identity := range mint { + if slices.Contains(gateway, identity) { + both = append(both, identity) } - if hint, ok := unpairable[name]; ok && raw == "" { - return fmt.Errorf("unresolved %s: %s", name, hint) + } + if len(both) == 0 { + return nil, fmt.Errorf("the credential mint and the gateway are both at %s and no identity is in both provisioned sets, so there is none this runtime may present its credentials to there: a dial to a shared address cannot be attributed to one of them, so only an identity both admit is safe. Provision the party that answers there in both sets, or give them separate addresses", + addr) + } + return both, nil + } +} + +// dialAddress is the host:port a URL is dialled at, with the scheme's default +// port made explicit so it compares equal to what the dialler is handed. +// +// Host and port together, because the two destinations may well share a host +// and differ only by port — the mint has lived on the gateway's host — and a +// comparison on the host alone would then hold both to whichever set matched +// first, which is the merge this split exists to undo. +func dialAddress(raw string) string { + parsed, err := url.Parse(raw) + if err != nil || parsed.Host == "" { + return "" + } + if parsed.Port() != "" { + return parsed.Host + } + if parsed.Scheme == "http" { + return parsed.Host + ":80" + } + return parsed.Host + ":443" +} + +// resolvedIdentities reads an admission set from its file, per call. +// +// No caching and no last-good fallback, which is the opposite of how the leaf +// this workload presents is treated: a leaf that cannot be re-read keeps +// serving, because the old leaf is this workload's own identity and refusing it +// only turns away callers who should be let in. An admission set that cannot be +// re-read has no such safe direction — the set on disk may be narrower than the +// one in memory, and that is the case this reads per handshake for. +func resolvedIdentities(path, unresolved string) func() ([]string, error) { + return func() ([]string, error) { + if path == "" { + return nil, fmt.Errorf("%s", unresolved) + } + content, err := readAdmissionFile(path) + if err != nil { + return nil, fmt.Errorf("%s: reading %q: %w", unresolved, path, err) + } + identities, err := parsedIdentities(content) + if err != nil { + return nil, fmt.Errorf("%s: %q: %w", unresolved, path, err) + } + if len(identities) == 0 { + return nil, fmt.Errorf("%s: %q names none", unresolved, path) + } + return identities, nil + } +} + +// admissionFileLimit caps an admission file. +// +// A set this runtime reads on every handshake, on every dial, and once a second +// per established connection in each direction is not a file to read without a +// bound: a 10MB one measured 9.5ms per read, which is a cost an operator can +// inflict by accident and an attacker who can write that path can inflict on +// purpose. A realistic set is a handful of SPIFFE IDs. +const admissionFileLimit = 64 << 10 + +// readAdmissionFile reads an admission set with a size bound. +func readAdmissionFile(path string) (string, error) { + file, err := os.Open(path) + if err != nil { + return "", err + } + defer func() { _ = file.Close() }() + content, err := io.ReadAll(io.LimitReader(file, admissionFileLimit+1)) + if err != nil { + return "", err + } + if len(content) > admissionFileLimit { + return "", fmt.Errorf("the file is larger than %d bytes, which is not a set of identities: this is read on every handshake, every dial, and once a second per established connection", admissionFileLimit) + } + return string(content), nil +} + +// parsedIdentities is the admitted set a file names, and an error for a file +// this runtime will not guess at. +// +// Each entry must be a *complete* line, which is the part that matters and is +// not obvious. The previous reader split on whatever it was handed, so a file +// caught mid-write — the platform rewriting it without an atomic swap, an +// operator's `>` redirect — truncated an entry and the truncation was admitted +// as an identity of its own: ".../sa/gateway-internal" read as ".../sa/gateway" +// and let in a caller the set never named. A trailing newline is the only +// evidence a line is whole, so a final fragment is refused rather than taken. +// +// Each entry is then checked to be a SPIFFE ID, because that is what it is +// compared against and an entry that cannot match anything is a provisioning +// mistake this runtime should name at boot rather than carry silently. +func parsedIdentities(raw string) ([]string, error) { + if strings.HasPrefix(raw, "\ufeff") { + return nil, fmt.Errorf("the file begins with a byte-order mark, so its first entry can never match an identity") + } + if raw != "" && !strings.HasSuffix(raw, "\n") { + return nil, fmt.Errorf("the file does not end in a newline, so its last line may be a fragment of one — a set caught mid-rewrite would otherwise admit a truncated identity as an identity of its own") + } + var identities []string + for _, line := range strings.Split(raw, "\n") { + if hash := strings.Index(line, "#"); hash >= 0 { + line = line[:hash] + } + for _, entry := range strings.Split(line, ",") { + trimmed := strings.TrimSpace(entry) + if trimmed == "" { + continue } - if name == "self upstream" { - // Not a host endpoint: it is this solution's own address, from - // its own endpoint carriers or an explicit SELF_UPSTREAM. - return fmt.Errorf("unresolved self upstream %q: SELF_UPSTREAM must be an absolute URL the gateway can dial, or unset so the address Codefly injects for this service is used", raw) + // Shape, not canonicalisation. An entry is matched by exact + // string equality against the peer's single URI SAN, so a + // percent-encoded, IDN, ported, userinfo-bearing or trailing-dot + // variant fails closed — it simply never matches anything. This + // refuses entries that could never match *anything at all*, which + // is a provisioning mistake worth naming at boot; it is not a + // claim that the entry is canonical, and describing it as + // "validated as a SPIFFE ID" overstated it. + if err := usableIdentity(trimmed); err != nil { + return nil, err } - return fmt.Errorf("unresolved %s %q: the SDK could not resolve the host endpoint and no explicit override was set", name, raw) + identities = append(identities, trimmed) } } - if deployedRuntimeContext(c.runtimeContext) { - // A loopback address registered from a deployment is reachable by no - // one: the gateway dialling "localhost" proxies the solution to - // itself, and a browser loading a manifest from "localhost" loads it - // from the viewer's own machine. Both used to boot and look healthy. - if loopbackURL(c.selfUpstream) { - return fmt.Errorf("self upstream %q is a loopback address in the deployed runtime context %q: the gateway would proxy this solution to itself. Codefly injects the reachable address as %s______HTTP__HTTP (core >= v0.5.6); set SELF_UPSTREAM only for a deployment the resolver cannot see", - c.selfUpstream, c.runtimeContext, selfEndpointCarrierPrefix) - } - if c.publicURL != "" && loopbackURL(c.publicURL) { - return fmt.Errorf("PUBLIC_URL %q is a loopback address in the deployed runtime context %q: no browser but this machine's could load the manifest from it. Unset it to register the manifest relative to this backend, or set the origin browsers actually reach", - c.publicURL, c.runtimeContext) + return identities, nil +} + +// usableIdentity refuses an entry that could never match a peer, so the refusal +// names the provisioning rather than appearing later as a caller nobody admits. +func usableIdentity(entry string) error { + for _, r := range entry { + if r < 0x20 || r == 0x7f || (r >= 0x80 && r <= 0x9f) || r == 0xfeff { + return fmt.Errorf("the entry %q contains a control character, so it can never match an identity a peer presents", entry) } } - if err := c.validateMCP(); err != nil { - return err + parsed, err := url.Parse(entry) + if err != nil { + return fmt.Errorf("the entry %q is not a URI: an admitted identity is a SPIFFE ID, compared against the one URI SAN a peer's certificate carries", entry) } - if c.registrationIntervalErr != nil { - return c.registrationIntervalErr - } - if c.solutionSecret == "" { - // An empty secret has two causes that look identical here, and sending - // the operator to provision a secret that is already provisioned is the - // worse of the two mistakes — so when the environment never loaded, say - // that instead of naming the provisioning. - if c.environmentLoadErr != nil { - return fmt.Errorf("no solution registration secret resolved, and loading Codefly's injected environment failed first: %w — the secret may well be provisioned; fix the environment load before treating this as missing provisioning", - c.environmentLoadErr) - } - // The host admits a registration only against a credential minted from - // this secret; without it every beat is a guaranteed refusal, so the - // boot fails here, naming the provisioning, rather than coming up - // looking healthy while nothing is served. - return fmt.Errorf("no solution registration secret provisioned: set %s, or provision the workspace secret %s/%s and declare that group as a workspace-configuration dependency of this backend", - SolutionRegistrationSecretEnvironmentVariable, SolutionRegistrationSecretGroup, SolutionRegistrationSecretKey) + if parsed.Scheme != "spiffe" || parsed.Host == "" || parsed.Path == "" { + return fmt.Errorf("the entry %q is not a SPIFFE ID (spiffe:///): it is compared against the one URI SAN a peer's certificate carries, so an entry of another shape admits nobody and is a provisioning mistake rather than a narrower set", entry) } return nil } +// requirePath refuses an unresolved path, naming the override and the group the +// platform provisions it through — and saying so differently when the +// environment never loaded, because sending an operator to provision something +// that is already provisioned is the worse of the two mistakes. +func (c config) requirePath(name, value, override, key string) error { + if value != "" { + return nil + } + if c.environmentLoadErr != nil { + return fmt.Errorf("no %s path resolved, and loading Codefly's injected environment failed first: %w — the projection may well be in place; fix the environment load before treating this as missing provisioning", + name, c.environmentLoadErr) + } + return fmt.Errorf("no %s path resolved: this listener presents this workload's X.509-SVID and requires its callers to present theirs, and there is no plain-HTTP listener, so this is required. Set %s, or have the platform provision %s/%s and declare that group as a workspace-configuration dependency of this backend", + name, override, WorkloadIdentityGroup, key) +} + // New starts a solution builder for the given manifest. func New(manifest Manifest) *Server { if manifest.ExposedModule == "" { manifest.ExposedModule = "./Page" } if manifest.Contract == "" { - manifest.Contract = "lastlogin" + // The solution's own id, which is what the field's documentation has + // always promised. It used to default to a product's name — a literal + // from one deployment, in a runtime that is supposed to know nothing + // about any specific solution, so every solution that left the field + // unset announced that product's capability contract to the host. + manifest.Contract = manifest.ID } - return &Server{manifest: manifest, handlers: make(map[string]RequestHandler)} + return &Server{manifest: manifest, handlers: make(map[string]RequestHandler), terminal: make(chan struct{})} } // Handle registers a solution endpoint. Chainable. @@ -782,57 +1409,135 @@ func (s *Server) HandleRequest(path string, handler RequestHandler) *Server { return s } -// Serve reads env config, self-registers, and blocks serving the solution. +// Serve resolves this process's configuration, obtains the one credential this +// execution holds, and blocks serving the solution over TLS. +// +// It registers nothing. Presence and authority are delivered to the host; this +// runtime's only outbound act is the single mint below, and the only thing it +// tells anyone about itself is what it answers when asked. func (s *Server) Serve() error { ctx := context.Background() - // Before anything the environment owns: a surface the author declared wrong - // is wrong in every environment, and saying so first keeps that mistake from - // reading as one more unresolved address. + ln, err := s.start(ctx) + if err != nil { + return err + } + return s.serve(ctx, ln) +} + +// start is the whole boot up to the listener: the declaration checks, the +// resolved configuration, the published contract, the one mint, and the +// listener that presents this workload's identity. Split from Serve so a test +// boots the real thing on an ephemeral port — every refusal below is a refusal +// of a real boot and not of a path only tests take. +func (s *Server) start(ctx context.Context) (net.Listener, error) { + // Before anything the environment owns: a surface or a contract the author + // declared wrong is wrong in every environment, and saying so first keeps + // that mistake from reading as one more unresolved address. if err := s.manifest.validateSurfaces(); err != nil { - return fmt.Errorf("solution %q: %w", s.manifest.ID, err) + return nil, fmt.Errorf("solution %q: %w", s.manifest.ID, err) } if err := s.validateMCPDeclaration(); err != nil { - return fmt.Errorf("solution %q: %w", s.manifest.ID, err) + return nil, fmt.Errorf("solution %q: %w", s.manifest.ID, err) } // The SDK owns environment resolution: load Codefly's injected carriers so - // endpoint and workspace-secret lookups resolve from them (falling back to - // the local native workspace map when not running under the runtime). - // Kept, not swallowed: when this fails every SDK-resolved endpoint and - // secret below comes back empty, and validate() would otherwise report each - // one as something the composition forgot to provision. + // endpoint and workspace-configuration lookups resolve from them (falling + // back to the local native workspace map when not running under the + // runtime). Kept, not swallowed: when this fails every SDK-resolved value + // below comes back empty, and validate() would otherwise report each one as + // something the composition forgot to provision. environmentLoadErr := codefly.LoadEnvironmentVariables() if environmentLoadErr != nil { log.Printf("codefly: load environment: %v", environmentLoadErr) } - s.cfg = loadConfig(ctx, s.manifest.ID) - s.cfg.environmentLoadErr = environmentLoadErr + s.cfg = loadConfig(ctx, s.manifest.ID, environmentLoadErr) + // Whether an MCP surface was declared, which is what makes the MCP + // configuration load-bearing: validate() checks it only then, so a + // solution serving no MCP is unaffected by a host whose issuer does not + // resolve. s.cfg.mcp = s.mcp != nil if err := s.cfg.validate(); err != nil { - return fmt.Errorf("solution %q: %w", s.manifest.ID, err) + return nil, fmt.Errorf("solution %q: %w", s.manifest.ID, err) + } + if err := s.validateSources(); err != nil { + return nil, fmt.Errorf("solution %q: %w", s.manifest.ID, err) } // After the environment is loaded: the declaration is checked against the // api.consumes projection Codefly injected. routes, err := s.validatePassthrough() if err != nil { - return fmt.Errorf("solution %q: %w", s.manifest.ID, err) + return nil, fmt.Errorf("solution %q: %w", s.manifest.ID, err) } s.passthrough = routes - ln, err := net.Listen("tcp", ":"+s.cfg.port) + // The authority-bearing values first: the principal is part of what the + // contract publishes, and a value the platform never provisioned is named + // here rather than discovered when a mint is refused. + if err := s.openAuthority(ctx); err != nil { + return nil, fmt.Errorf("solution %q: %w", s.manifest.ID, err) + } + // This workload's identity, resolved and held to every posture rule — + // including that the leaf is the principal the platform provisioned — + // before the first outbound act. Two bugs lived in the four lines this + // replaces. + // + // It was conditional: `if s.credential == nil || s.identity == nil`. With + // *both* sources supplied the client was never built, so s.outbound stayed + // nil and every gateway call fell through to the unauthenticated transport + // — system roots, no client certificate — which is the one posture this + // cutover exists to remove, reached by supplying more configuration rather + // than less. And with only Identity() supplied, the condition was true, so + // the client read the projected files validateSources had just declared + // unnecessary for that boot: the README's own example could not start. + // + // It also ran after the mint. A leaf issued for a neighbouring workload had + // therefore already been presented to the host, with this workload's + // projected token, by the time listen() refused it. + identityConfig, err := s.serverIdentity() if err != nil { - return err + return nil, fmt.Errorf("solution %q: %w", s.manifest.ID, err) } - return s.serve(ctx, ln) + outbound, err := s.outboundClient(identityConfig) + if err != nil { + return nil, fmt.Errorf("solution %q: %w", s.manifest.ID, err) + } + s.outbound = outbound + // The contract this solution publishes is resolved from the same + // declaration, under the profile this process runs in, and every value it + // requires is named in the refusal when it is absent. + contract, err := s.resolveContract() + if err != nil { + return nil, fmt.Errorf("solution %q: %w", s.manifest.ID, err) + } + s.contract = contract + s.contractResolved = true + // One mint per execution, before the listener is up. A runtime that cannot + // obtain its credential has no authority to serve anything with, and the + // model it serves under has nothing for it to retry against: its presence + // was declared by a document it did not write, and a refusal here says that + // document does not name this build. So it reports the refusal and exits + // non-zero, for the orchestrator to restart or the delivery to be fixed — + // it never loops, and it never comes up serving without one. + if err := s.openCredential(ctx); err != nil { + return nil, fmt.Errorf("solution %q: %w", s.manifest.ID, err) + } + ln, err := s.listen() + if err != nil { + return nil, fmt.Errorf("solution %q: %w", s.manifest.ID, err) + } + return ln, nil } -// serve wires the routes, starts the host and gateway registration heartbeats, -// and serves on ln until ctx is cancelled. Split from Serve so a test can boot a -// real solution on an ephemeral listener and exercise the whole registration and -// manifest path. +// serve wires the routes and serves on ln until ctx is cancelled. Split from +// Serve so a test can boot a real solution on an ephemeral listener and +// exercise the whole served surface. func (s *Server) serve(ctx context.Context, ln net.Listener) error { mux := http.NewServeMux() mux.HandleFunc("/.well-known/solution.json", withCORS(s.handleManifest)) mux.HandleFunc("/.well-known/capabilities", withCORS(s.handleCapabilities)) - mux.HandleFunc("/health", func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) }) + mux.HandleFunc(ContractPath, withCORS(s.handleContract)) + // Health is answered, never pushed. The host probes the destination its own + // presence document names; this process does not report its liveness + // anywhere, and has no way to make itself present by answering. + mux.HandleFunc(HealthPath, s.handleHealth) for path, handler := range s.handlers { mux.HandleFunc(path, withCORS(s.wrapRequest(handler))) } @@ -844,110 +1549,541 @@ func (s *Server) serve(ctx context.Context, ln net.Listener) error { return err } - manifestBody, _ := json.Marshal(s.manifestMap()) - upstreamBody, _ := json.Marshal(map[string]string{"id": s.manifest.ID, "upstream": s.cfg.selfUpstream}) - go s.heartbeat(ctx, s.cfg.hostRegisterURL, manifestBody, "host", s.newSolutionCredential()) - go s.heartbeat(ctx, s.cfg.gatewayRegisterURL, upstreamBody, "gateway", s.newSolutionCredential()) - s.registerConsumedAPIs(ctx) - - srv := &http.Server{Handler: mux} + watch, err := s.watchInboundTrust() + if err != nil { + return err + } + srv := &http.Server{ + Handler: mux, + // See inboundHandshakeTimeout and inboundIdleTimeout: with all of + // these unset, a peer that never finishes its ClientHello held a + // goroutine and a descriptor for as long as it liked. + ReadHeaderTimeout: inboundHandshakeTimeout, + IdleTimeout: inboundIdleTimeout, + // And every served connection is held to trust as it is now, not as + // it was when the handshake happened. + ConnState: watch, + } go func() { - <-ctx.Done() - srv.Close() + select { + case <-ctx.Done(): + case <-s.credentialRefusedC(): + } + _ = srv.Close() }() - log.Printf("solution %q listening on :%s (gateway=%s)", s.manifest.ID, s.cfg.port, s.cfg.gatewayURL) + log.Printf("solution %q listening on :%s (gateway=%s, profile=%s)", s.manifest.ID, s.cfg.port, redactedURL(s.cfg.gatewayURL), s.cfg.profile) if serveErr := srv.Serve(ln); !errors.Is(serveErr, http.ErrServerClosed) { return serveErr } + // A terminal credential failure is reported as the reason this process + // ended, so the orchestrator's restart is against a judgement rather than + // an unexplained exit. + if err := s.terminalErr.Load(); err != nil && ctx.Err() == nil { + return fmt.Errorf("solution %q stopped serving: %w", s.manifest.ID, *err) + } return nil } -// registerConsumedAPIs registers each of the solution's api.consumes targets -// with the gateway so it can proxy /v1//* to the consumed module. The -// targets are projected by core into CODEFLY__API_CONSUMES; the address of each -// is already injected (the backend depends on the consumed service), so it is -// resolved through the SDK exactly as the gateway and frontend are. Each target -// gets its own heartbeat, mirroring the host and gateway registrations, so a -// registration that is briefly unavailable at boot is retried. +// handleHealth answers the host's probe with what this process can actually do. // -// A solution that declares no api.consumes has an empty CODEFLY__API_CONSUMES -// and registers nothing — no behavior changes unless api.consumes is declared. -func (s *Server) registerConsumedAPIs(ctx context.Context) { - consumed, err := manifest.ParseConsumedAPIs(os.Getenv(manifest.APIConsumesEnvironmentVariable)) - if err != nil { - // A malformed projection is a build-time/CLI defect the runtime cannot - // repair. ParseConsumedAPIs returns no entries on a decode error, so this - // registers nothing at all: every consumed facade silently 404s at the - // gateway. Crashing would also take down this solution's own frontend, so - // serve on — but log loudly, because this line is the only signal that the - // entire api.consumes federation was dropped. - log.Printf("solution %q: api.consumes federation disabled: %v", s.manifest.ID, err) - } - for _, c := range consumed { - // The gateway proxies /v1//*, where As is the facade entry-point core - // derives from the producing endpoint's proto package (the last non-version - // segment) — not the module name. The runtime cannot reconstruct that - // default from the projected identity, so an entry that reached us without - // an explicit As is one the CLI should have resolved: registering a guessed - // prefix (e.g. the module name) would proxy a route the generated client - // never calls (a silent 404) and could even claim a prefix meant for a - // different facade. Skip it loudly instead of guessing. - if c.As == "" { - log.Printf("solution %q: consumed api %s/%s/%s has no facade entry-point (as); skipping gateway module registration", - s.manifest.ID, c.Module, c.Service, c.Endpoint) - continue +// It was an unconditional 200, which is only right while the one thing this +// runtime needs is still true. Once the issuer has refused this execution's +// credential for good, the process can serve no module call for any viewer: a +// 200 then invites the host to keep routing to a binding that answers 503 to +// everything, and the probe is the only channel this runtime has to say +// otherwise — it does not push liveness, so it has to answer honestly. +func (s *Server) handleHealth(w http.ResponseWriter, r *http.Request) { + if err := s.terminalErr.Load(); err != nil { + // Named, not detailed: a probe is not a place to put the issuer's text. + http.Error(w, "this execution's credential has been refused and will not be renewed; this process is ending", http.StatusServiceUnavailable) + return + } + // The probe asks the credential, rather than only reporting a refusal some + // *other* request happened to discover. + // + // Renewal is lazy by design — the client renews when a caller asks for the + // credential — so a solution that serves only ViewerBearer routes, plain + // handlers and assets never asks, and therefore never learns that the + // issuer has stopped approving this build. It would keep answering 200 and + // keep serving, with the refusal arriving only if some request eventually + // needed a capability. That is the same "serves nothing, looks alive" shape + // one layer out, and the probe is the one channel this runtime has. + // + // Asking is cheap and is not a heartbeat: the client holds one credential + // and hands the same one back until its renewal point, so this is a + // mutex and a comparison except at the renewal the credential's own expiry + // dictates. Nothing here runs on a timer this runtime chose. + if s.credential != nil { + // Through credentialWithin, so the deadline is real: passing a context + // to the SDK's client does not bound it, because what blocks is a + // mutex held across the mint. + if _, err := s.credentialWithin(r.Context(), healthCredentialTimeout); err != nil { + if errors.Is(err, ErrCredentialRefused) { + http.Error(w, "this execution's credential has been refused and will not be renewed; this process is ending", http.StatusServiceUnavailable) + return + } + // Transient: the issuer cannot answer right now. Still unhealthy — + // this process cannot act for a viewer until it can — but it is not + // a judgement, so nothing ends. + http.Error(w, "this execution's credential cannot currently be obtained", http.StatusServiceUnavailable) + return } - prefix := c.As - upstream := address(ctx, c.Module, c.Service, c.Endpoint, c.Protocol) - if upstream == "" { - // The address is injected because the backend depends on the consumed - // service; if it did not resolve, registering an empty upstream would - // only be rejected, so skip loudly instead. - log.Printf("solution %q: no upstream resolved for consumed api %q (%s/%s/%s); skipping gateway module registration", - s.manifest.ID, prefix, c.Module, c.Service, c.Endpoint) - continue + } + w.WriteHeader(http.StatusOK) +} + +const routeCredentialTimeout = 500 * time.Millisecond + +// healthCredentialTimeout bounds what a probe will wait for the credential, so +// a hung issuer makes the probe fail rather than parking it. +const healthCredentialTimeout = 2 * time.Second + +// actingForAViewer refuses to act on a viewer's behalf while this process holds +// no credential the issuer still approves. +// +// Every route that does anything for a viewer asks this first, and that was the +// hole: the *probe* asked the credential and the routes did not. The reasoning +// is written out at handleHealth — renewal is lazy, so a solution serving only +// plain handlers and ViewerBearer routes never asks and never learns the issuer +// has stopped approving this build — and it was applied to the one caller that +// reports health rather than to the callers that act. A booted server whose +// credential had been refused kept answering 200 and kept forwarding the +// viewer's bearer to the gateway, with the credential consulted zero times. +// +// A plain handler looks like it needs no credential, which is the trap. What +// authorises this process to act for a viewer at all *is* the credential: it is +// what the host approved for this build. Forwarding a viewer's bearer under a +// credential the issuer has withdrawn is acting without authority, whether or +// not a capability is minted on the way. +// +// Cheap, and not a heartbeat: the client holds one credential and hands the +// same one back until its own renewal point, so this is a mutex and a +// comparison except at the renewal the credential's expiry dictates. +func (s *Server) actingForAViewer(ctx context.Context) error { + if err := s.terminalErr.Load(); err != nil { + return fmt.Errorf("%w: %w", ErrCredentialRefused, *err) + } + if s.credential == nil { + return nil + } + _, err := s.credentialWithin(ctx, routeCredentialTimeout) + return err +} + +// credentialWithin asks for this execution's credential and never waits longer +// than d, whatever the source does. +// +// The bound has to be imposed here, because a context does not reach the thing +// that blocks. The SDK's mint client takes a sync.Mutex and holds it across the +// network mint (workcontext/mint.go, Credential), and sync.Mutex.Lock ignores +// contexts — so a caller arriving during a slow renewal waits on the lock for +// as long as the renewal takes, and the deadline it passed in bounds nothing. +// That made the two-second health deadline advisory, and it would have made +// every route's credential check park behind one slow issuer, which is a worse +// failure than the one the check was added for. +// +// The ask runs on a context detached from the caller's, deliberately: a request +// that gives up should not cancel a mint that other callers are waiting for, +// and the SDK caches the result, so the work is not wasted. It is bounded by +// its own deadline rather than by the caller's. +// +// On a timeout, the credential this process already holds decides. Refusing +// outright would turn a slow renewal — which happens while a perfectly valid +// credential is still in hand, since renewal starts inside a lead before expiry +// — into a 503 for every viewer. +func (s *Server) credentialWithin(ctx context.Context, d time.Duration) (workcontext.Credential, error) { + // Held and unexpired is the answer whenever the issuer is not answering, + // and this is the branch that was missing. + // + // The timeout path below already served on a held credential, and the + // error path refused outright — so with the credential inside its renewal + // window and the issuer answering 503, every viewer request asked, every + // ask minted, and every request got a 503 while a perfectly valid + // credential sat in hand. Forty requests, forty audited mints. That is not + // a smaller version of the heartbeat this runtime deleted, it is a larger + // one: the heartbeat minted on a timer, this minted on traffic. + // + // So a failed ask starts a backoff, and inside it nothing asks again. The + // renewal the credential's own expiry dictates still happens, because the + // backoff is shorter than the renewal lead; what cannot happen is one mint + // per request. + // The quiet window applies whether or not anything is held, which is the + // half that was missing: it was consulted only when a credential WAS in + // hand, so the one state that should ask least — nothing held, the issuer + // failing — was the state that asked on every single request. The branch + // meant to back off was the branch that hammered. + // A recorded judgement first, on every path. The quiet branch below would + // otherwise re-wrap a terminal refusal as "cannot currently be obtained", + // which is the one distinction the boot and the run must not disagree on. + if err := s.terminalErr.Load(); err != nil { + return workcontext.Credential{}, fmt.Errorf("%w: %w", ErrCredentialRefused, *err) + } + if held, quiet, last := s.credentialState(); quiet { + if held.Token() != "" { + return held, nil } - secret := s.cfg.moduleSecrets[prefix] - if secret == "" { - // The gateway admits a registration only against a token signed by - // accounts, and accounts issues one only to a caller holding the - // secret whose digest the composition declared for this prefix. - // Without it every beat would be a guaranteed 401, so skip loudly: - // this line is the signal that provisioning, not the runtime, is the - // missing half. - log.Printf("solution %q: no registration secret provisioned for consumed api %q (%s); skipping gateway module registration", - s.manifest.ID, prefix, ModuleRegistrationSecretsEnvironmentVariable) - continue + if terminalCredentialFailure(last) { + return workcontext.Credential{}, fmt.Errorf("%w: %w", ErrCredentialRefused, last) } - body, _ := json.Marshal(map[string]string{"prefix": prefix, "upstream": upstream}) - credential := &moduleCredential{ - tokenURL: s.cfg.moduleTokenURL, - internalToken: s.cfg.internalToken, - prefix: prefix, - secret: secret, + return workcontext.Credential{}, fmt.Errorf("%w: %w", ErrCredentialUnavailable, last) + } + credential, err := s.askOnce(ctx, d) + if err == nil { + return credential, nil + } + if errors.Is(err, ErrCredentialRefused) { + return credential, err + } + // Unavailable, for any reason including the ask not answering in time. A + // credential already in hand decides, because renewal begins inside a lead + // before expiry and refusing here would 503 every viewer over a slow + // issuer this process does not yet need. + if held, ok := s.heldCredential(); ok { + return held, nil + } + return credential, err +} + +// holdCredential records a credential the controller did not itself acquire — +// the boot's — so the first caller after boot sees a process that holds one. +func (s *Server) holdCredential(credential workcontext.Credential) { + s.credentialMu.Lock() + s.credentialHeld = credential + s.credentialQuietUntil = time.Time{} + s.credentialMu.Unlock() +} + +// heldCredential is the credential this process last obtained, if it has not +// expired. The zero credential never counts as held: a source that hands back +// nothing has given this process no authority, which is the case a wrapper +// keyed on the token's *value* could not see. +func (s *Server) heldCredential() (workcontext.Credential, bool) { + s.credentialMu.Lock() + defer s.credentialMu.Unlock() + if s.credentialHeld.Token() == "" { + return workcontext.Credential{}, false + } + if !time.Now().Before(s.credentialHeld.ExpiresAt()) { + return workcontext.Credential{}, false + } + return s.credentialHeld, true +} + +// credentialState is what this process holds, whether a recent ask failed +// recently enough that nobody should ask again, and that ask's error. +// +// All three under one lock, because a caller that reads them separately can act +// on a mixture of two different moments. +func (s *Server) credentialState() (held workcontext.Credential, quiet bool, last error) { + s.credentialMu.Lock() + defer s.credentialMu.Unlock() + held = s.credentialHeld + if held.Token() == "" || !time.Now().Before(held.ExpiresAt()) { + held = workcontext.Credential{} + } + last = s.credentialAskErr + if last == nil { + last = fmt.Errorf("the credential source has not answered yet") + } + return held, time.Now().Before(s.credentialQuietUntil), last +} + +// askOnce asks the source for the credential, at most one ask at a time, and +// never waits longer than d. +// +// Single-flight, because the alternative is one mint per concurrent caller the +// moment the credential enters its renewal window. The bound is imposed here +// rather than passed down, because a context does not reach what blocks: the +// SDK's client holds a sync.Mutex across the network mint and sync.Mutex.Lock +// ignores contexts (see the drafted sdk-go issue, which is the durable half of +// this). +func (s *Server) askOnce(ctx context.Context, d time.Duration) (workcontext.Credential, error) { + s.credentialMu.Lock() + inflight := s.credentialAsk + if inflight == nil { + inflight = &credentialFlight{done: make(chan struct{})} + s.credentialAsk = inflight + go s.resolveCredential(ctx, inflight) + } + s.credentialMu.Unlock() + + timer := time.NewTimer(d) + defer timer.Stop() + select { + case <-inflight.done: + case <-timer.C: + return workcontext.Credential{}, fmt.Errorf("%w: it did not answer within %s", ErrCredentialUnavailable, d) + } + + // From the flight this caller actually waited on, not from the server's + // current state. Reading the shared fields let a waiter observe a *later* + // flight's answer — or none at all, if the next flight had already started + // and cleared them — so the credential a caller acted on was not + // necessarily the one its own ask produced. A flight's result is written + // once, before its channel closes, and never again. + credential, err := inflight.credential, inflight.err + if err != nil { + if terminalCredentialFailure(err) { + // Records it, which also ends the process: the first caller to + // discover a refused build is as good a place to learn it as the + // probe, and better than the next one. + s.credentialRefused(err) + return workcontext.Credential{}, fmt.Errorf("%w: %w", ErrCredentialRefused, err) } - go s.heartbeat(ctx, s.cfg.moduleRegisterURL, body, "gateway module "+prefix, credential) + return workcontext.Credential{}, fmt.Errorf("%w: %w", ErrCredentialUnavailable, err) } + return credential, nil } -// newSolutionCredential builds one self-registration credential. Each -// heartbeat gets its own, because a solutionCredential is not safe to share: -// its refusal and mint bookkeeping is written on every beat with no locking, -// so two goroutines holding one would race on those fields. (Sharing would not -// cause two beats to present the same token — nothing is cached, so each -// authorize mints its own — which is why the fields, not the token, are the -// reason.) validate() has already refused a boot without a secret; there is no -// other credential to present. -func (s *Server) newSolutionCredential() *solutionCredential { - return &solutionCredential{ - tokenURL: s.cfg.solutionTokenURL, - internalToken: s.cfg.internalToken, - id: s.manifest.ID, - secret: s.cfg.solutionSecret, +// resolveCredential performs the one in-flight ask and records its outcome. +// +// On a context detached from the caller's, deliberately: a request that gives +// up must not cancel an ask other callers are waiting on, and the SDK caches +// the result so the work is not wasted. +func (s *Server) resolveCredential(ctx context.Context, flight *credentialFlight) { + asked, cancel := context.WithTimeout(context.WithoutCancel(ctx), platformRequestTimeout) + defer cancel() + credential, err := s.credential.Credential(asked) + if err == nil { + err = usableCredential(credential) + } + + // Recorded HERE, where the answer lands, and not only where a caller + // happens to be waiting for it. + // + // This ask runs on a detached context so a request that gives up does not + // cancel it — which means its answer can arrive after every caller has + // gone. A terminal refusal classified only by the waiting caller was + // therefore discarded whenever acquisition outlived its caller: the route + // served on the held credential, nothing called credentialRefused, and the + // process went on acting for viewers after its own source had reported + // this execution refused. The classification belongs to the answer, not to + // whoever was still listening. + if err != nil && terminalCredentialFailure(err) { + s.credentialRefused(err) + } + + // The flight's own result first, then the shared state. A waiter reads the + // former and never the latter. + flight.credential, flight.err = credential, err + + s.credentialMu.Lock() + s.credentialAskErr = err + if err != nil { + s.credentialAskFailures++ + s.credentialQuietUntil = time.Now().Add(credentialAskQuiet(s.credentialAskFailures)) + } else { + s.credentialHeld = credential + s.credentialQuietUntil = time.Time{} + s.credentialAskFailures = 0 + } + s.credentialAsk = nil + s.credentialMu.Unlock() + close(flight.done) +} + +// credentialFlight is one acquisition and the answer it produced. +type credentialFlight struct { + done chan struct{} + credential workcontext.Credential + err error +} + +// credentialAskBackoff is the FIRST quiet period after a failed ask, and +// credentialAskBackoffCeiling is the longest one. +// +// Shorter than any renewal lead at the start, so the renewal a credential's +// own expiry dictates still happens on time; long enough that a stream of +// viewer requests against an unavailable issuer is one ask rather than one per +// request. +const ( + credentialAskBackoff = time.Second + credentialAskBackoffCeiling = 30 * time.Second + credentialAskBackoffJitter = 0.2 +) + +// credentialAskQuiet is how long the next ask is held off after `failures` +// consecutive failures: doubling, capped, with jitter. +// +// It was a FLAT one second, reset to the same value after every failure. Under +// sustained traffic against a fast-failing issuer that is one audited mint +// attempt per replica per second for as long as the outage lasts, and every +// replica on the same period — so they arrive together, which is the shape +// that turns an issuer outage into an issuer stampede. Single-flight bounds +// how many asks one replica makes at once; it does nothing about the rate, and +// nothing about the fleet. +// +// Capped at 30s, which is under every renewal lead this runtime can see, so a +// credential still renews on its own schedule once the issuer answers again. A +// credential already in hand and unexpired keeps serving throughout, which is +// why waiting longer costs nothing a viewer can see. +// +// Jitter is multiplicative and one-sided-down, so a backoff never exceeds the +// ceiling and replicas that started together stop being in phase. +func credentialAskQuiet(failures int) time.Duration { + if failures < 1 { + failures = 1 + } + quiet := credentialAskBackoff + for range failures - 1 { + quiet *= 2 + if quiet >= credentialAskBackoffCeiling { + quiet = credentialAskBackoffCeiling + break + } + } + // Deterministic enough to reason about, random enough to break phase: + // subtract up to 20%. + spread := 1 - credentialAskBackoffJitter*rand.Float64() + return time.Duration(float64(quiet) * spread) +} + +// usableCredential refuses a credential that authorises nothing. +// +// A source is consumer code and may hand back the zero value — a +// half-initialised client, a stub that forgot to fail — and this runtime then +// held "a credential" with no token, no seal and no expiry, booted, answered +// /health 200, and acted for viewers under it. An expired one is the same +// shape with a timestamp. +// +// The authority wrapper could not see either: it keys on the token *changing*, +// and a token that is always "" never changes. +func usableCredential(credential workcontext.Credential) error { + if credential.Token() == "" { + return fmt.Errorf("%w: the credential source returned no credential at all (empty token), so this process holds nothing that attests which workload it is", + workcontext.ErrMintRefused) + } + // GetInstallationId, not the field: Seal() returns a POINTER now, so + // reading the field on a credential carrying no seal panics — in the one + // check written to refuse exactly that. The getter answers "" for a nil + // seal, which is the refusal this wants. + if seal := credential.Seal(); seal.GetInstallationId() == "" { + return fmt.Errorf("%w: the credential this process holds is sealed to no installation, so there is nothing for a far end to hold it to", + workcontext.ErrMintRefused) + } + // And it has to attest WHICH BUILD is asking. + // + // This check did not exist and did not need to: the previous SDK's own + // reader refused a seal whose build incarnation was zero, so a credential + // reaching here carried one by construction. The v0.3.0 migration moved + // structural validation to core, and core legitimately mints a credential + // sealing NO execution — a principal recorded as bearing none, a person at + // a terminal, which is core's own execution-missing shape. The guarantee + // was inherited, the inheritance ended, and nothing here noticed. + // + // Reproduced before fixing: a runtime booted on such a credential, held + // it, passed actingForAViewer, and served handlers and ViewerBearer + // passthroughs with a credential attesting no approved build at all. + // + // A WORKLOAD credential is the one kind that may never be execution-free. + // Its whole purpose is attesting which build is asking, so that a host can + // hold a mint to the build its presence document approved; one sealing no + // execution asserts nothing a host can check and the ceiling it is held to + // is the ceiling of nobody. + // + // Read through CORE'S OWN generated accessors on the verified capability's + // seal — Credential.Seal() returns core's protobuf type, so GetImageDigest + // and GetBuildIncarnation are core's, not a predicate of this package's. + // No decoder, no seal validator and no copy of sealOf grows here; that + // would be the second implementation this module is gated against. Both + // halves are required rather than either, because an execution is the pair + // — core's schema refuses a digest without an incarnation, so the shapes + // where "and" differs from "or" are shapes that never arrive. + // + // Terminal. A credential attesting no build is not a transient condition: + // the issuer will seal the same thing next time, and a loop around it is + // the audited-mint cost this runtime was changed to remove. + if seal := credential.Seal(); seal.GetBuildIncarnation() == 0 || seal.GetImageDigest() == "" { + return fmt.Errorf("%w: the credential this process holds seals no execution (image digest %q, build incarnation %d), so it attests no approved build and nothing a host can hold a mint to. A workload credential names the build that is asking; one that names none is a credential for a principal bearing no execution, which this process is not", + workcontext.ErrMintRefused, seal.GetImageDigest(), seal.GetBuildIncarnation()) + } + if expiry := credential.ExpiresAt(); !time.Now().Before(expiry) { + return fmt.Errorf("%w: the credential this process holds expired at %s", + workcontext.ErrMintUnavailable, expiry.UTC().Format(time.RFC3339)) } + return nil } +// routeCredentialTimeout bounds what a route acting for a viewer will wait for +// the credential. Short, because the answer is almost always a comparison +// against one already held, and a route is not the place to wait out a mint. + +// acquireCredential is the one way anything in this process obtains the +// execution credential, including a viewer's mint. +// +// Nil when no source was supplied, so attestWorkloadReporting still refuses a +// mint rather than treating an absent source as nothing to attest with. +func (s *Server) acquireCredential(ctx context.Context) (workcontext.Credential, error) { + return s.credentialWithin(ctx, routeCredentialTimeout) +} + +// credentialRefusalForAViewer is what a viewer is told when this process will +// not act for them: which of the two conditions it is, and nothing about the +// issuer, the destination or the source's own words. +func credentialRefusalForAViewer(err error) string { + if errors.Is(err, ErrCredentialRefused) { + return "this solution's authority for this execution has been refused and will not be renewed" + } + return "this solution cannot currently obtain the authority it needs to act for you" +} + +// ErrCredentialRefused and ErrCredentialUnavailable are why a route refused to +// act for a viewer: a judgement about this build, or an issuer that cannot +// answer right now. Both are 503 — this process cannot act either way — and +// they stay distinguishable because conflating them was a review blocker in +// both directions. +var ( + ErrCredentialRefused = errors.New("this execution's credential has been refused and will not be renewed, so this process will not act for a viewer") + ErrCredentialUnavailable = errors.New("this execution's credential cannot currently be obtained, so this process will not act for a viewer") +) + +// Why this runtime infers nothing from a conflict on a viewer's mint. +// +// A credential superseded before its renewal lead keeps being handed back by +// the SDK's client, which re-mints only at that lead and exposes no way to +// invalidate what it holds. Two rounds were spent building around that: a hook +// that dropped this process's belief on a 409 from the host, and then — because +// the next ask returned the same cached credential and restored the belief — a +// memory of the superseded token so the drop would stick. +// +// Both are removed, because the *trigger* was never sound. A conflict on a +// viewer's mint may mean the credential this workload presented is superseded; +// it may equally mean that viewer lacks the authority they asked for. The host +// does not distinguish the two in that answer — I wrote that down in an earlier +// round and then built on the guess anyway — and with the token remembered, one +// viewer's missing permission became a process-wide refusal to act for anybody. +// An escalating response to an ambiguous signal is worse than no response: +// before, a 409 cost that one call; after, it cost the process. +// +// So nothing is inferred. What would close it properly is a host answer that +// says which of the two it is, and a way to tell the mint client to forget what +// it cached — the second is drafted as an sdk-go issue and neither is this +// repository's to add. Until then a superseded credential is refused call by +// call by the far end, which is correct if noisy, and the renewal that replaces +// it happens on the credential's own schedule. + +// credentialRefused records a judgement the issuer will not reverse. +func (s *Server) credentialRefused(err error) { + s.terminalOnce.Do(func() { + s.terminalErr.Store(&err) + log.Printf("solution %q: this execution's credential was refused and will not be renewed, so this process is ending for the orchestrator to restart against the delivery as it stands: %v", + s.manifest.ID, err) + close(s.terminal) + }) +} + +// credentialRefusedC is the channel serve waits on. Created in New, never +// lazily: a request goroutine reporting a refusal and serve reading it would +// otherwise race on the field itself. +func (s *Server) credentialRefusedC() <-chan struct{} { return s.terminal } + +// HealthPath is where this runtime answers a health probe. It is a plain 200 on +// the listener the presence document names, which is the whole of this +// runtime's part in being observed: the host decides whether this binding is +// healthy, and nothing this process says can make it present. +const HealthPath = "/health" + // solutionManifestSchemaMajor and solutionHostContractMajor are the majors // this runtime's manifest and page contract are built against. The host checks // both before activating a remote, and treats a silent manifest as 1 — so these @@ -1075,22 +2211,23 @@ func assetCacheControl(name string) string { // manifest, relative to its own backend: the /assets/ file server in serve. const federationManifestPath = "/assets/mf-manifest.json" -// frontendManifestURL is the manifestUrl this solution registers. With an -// explicit PUBLIC_URL it is absolute on that origin — an operator who exposes -// the solution's assets directly said where. Without one it is the path on the -// solution's own backend, root-relative, and the host resolves it against the -// route by which it reaches this solution. -// -// It used to be absolute on "http://localhost:", which is this process's -// own listen address: true in a browser on the developer's machine and in no -// other browser anywhere, so every deployed solution registered a manifest the -// product could not load. The runtime cannot build a better absolute URL, -// because the origin a browser reaches this solution through is the host's — -// and naming the host's route layout here would couple every solution to it. +// frontendManifestURL is the path this solution's Module Federation manifest is +// served at, relative to this backend's own origin. The host resolves it +// against the route by which it reaches this solution — a route this runtime +// does not know and must not encode. +// +// It is a path and nothing else. It used to be an absolute URL, built from +// PUBLIC_URL or, with none set, from this process's own listen address: true in +// a browser on the developer's machine and in no other browser anywhere, so +// every deployed solution registered a manifest the product could not load. +// This runtime no longer builds it from PUBLIC_URL: the origin a browser +// reaches this solution through is named by the route in the presence document, +// which is the host's to resolve. PUBLIC_URL itself still exists, with exactly +// one consumer — the MCP resource identifier an agent client binds its token to +// (#49), which genuinely needs a public origin. Whether a runtime should derive +// the host's /solutions/ route even for that is an open follow-up; it is +// the one place this package encodes the host's route layout. func (s *Server) frontendManifestURL() string { - if s.cfg.publicURL != "" { - return strings.TrimRight(s.cfg.publicURL, "/") + federationManifestPath - } return federationManifestPath } @@ -1098,852 +2235,62 @@ func (s *Server) handleManifest(w http.ResponseWriter, _ *http.Request) { writeJSON(w, http.StatusOK, s.manifestMap()) } -func (s *Server) handleCapabilities(w http.ResponseWriter, _ *http.Request) { - // The same majors the registration manifest declares, from the same - // constants. Hardcoded here, a bump of either constant would leave this - // document announcing the old major while the manifest announced the new - // one, and a host that checks both would see one solution claiming two - // different contracts. - writeJSON(w, http.StatusOK, map[string]any{ - "schemaVersion": solutionManifestSchemaMajor, - "contract": s.manifest.Contract, - "contractMajor": solutionHostContractMajor, - "capabilities": s.manifest.Capabilities, - }) -} - -func (s *Server) wrap(handler Handler) http.HandlerFunc { - return s.wrapRequest(func(r *http.Request, gw *Gateway) (any, error) { - return handler(r.Context(), gw) - }) -} - -func (s *Server) wrapRequest(handler RequestHandler) http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - bearer := r.Header.Get("authorization") - if bearer == "" { - writeJSON(w, http.StatusUnauthorized, map[string]string{"error": "missing bearer"}) - return - } - result, err := handler(r, newGateway(s.cfg.gatewayURL, bearer, r.Header.Get(orgHeader), r.Header.Get(sessionHeader))) - if err != nil { - status, message := handlerErrorResponse(err) - writeJSON(w, status, map[string]string{"error": message}) - return - } - writeJSON(w, http.StatusOK, result) - } -} - -func (s *Server) heartbeat(ctx context.Context, url string, body []byte, label string, auth registrationAuth) { - interval := s.registrationInterval - if interval == 0 { - interval = s.cfg.registrationInterval - } - if interval == 0 { - interval = defaultRegistrationInterval - } - lastStatus := 0 - lastDetail := "" - lastErr := "" - // detailsLogged counts the distinct reasons reported for the current status. - // Reset whenever the status changes or a registration succeeds. - detailsLogged := 0 - failures := 0 - for { - status, detail, err := s.beat(ctx, url, body, auth) - if status == http.StatusUnauthorized || status == http.StatusForbidden { - // The credential we just presented was refused. Offer it for - // dropping so the next beat obtains a fresh one instead of replaying - // a token the issuer has stopped honouring for the rest of its - // lifetime. Whether that actually helps is the credential's call: - // re-obtaining one it has just obtained cannot fix a refusal that - // was never about staleness (see moduleCredential.invalidate). - // 403 counts too — a gateway that answers "forbidden" to a lapsed - // token would otherwise have it replayed for the rest of its life. - auth.invalidate() - } - switch { - case err != nil: - // A shutdown cancels the in-flight request; that error is expected, - // not a registration failure, so don't log it. - if ctx.Err() != nil { - return - } - // Transport/URL errors (connection refused, scheme-less URL) never - // surface a status. Left unlogged they made a permanently-failing - // registration indistinguishable from a working one. Throttle to one - // line per distinct error so a persistent outage doesn't spam. - if msg := err.Error(); msg != lastErr { - log.Printf("registration with %s failed for %q: %v", label, s.manifest.ID, err) - lastErr, lastStatus, lastDetail = msg, 0, "" - } - // The reason is part of what was last reported, not just the status. A - // host that keeps answering 409 while changing why — one incompatibility - // resolved, a second one now named — would otherwise say it once and go - // silent on every reason after it, which is precisely the information - // detail was plumbed through beat to carry. - // - // But the reason is text the *host* chooses, so it is not a key this - // loop can trust to settle. A refusal that quotes something per-attempt - // — the jti of the single-use token it just burned, a timestamp, a - // request id — differs on every beat by construction, and keyed on that - // alone the throttle is defeated exactly when the registration is most - // broken: one line per beat per surface, forever. So changed reasons are - // reported, but only so many per status; past that the status must - // change, or a registration must succeed, before reasons speak again. - case status != lastStatus || (detail != lastDetail && detailsLogged < maxRefusalDetailsPerStatus): - if status != lastStatus { - detailsLogged = 0 - } - switch { - case status < 300: - log.Printf("registered with %s as %q", label, s.manifest.ID) - case detail != "": - // The host says why. A refusal with reasons — an incompatible - // runtime contract, an id another publisher owns — is actionable - // only with them; the status alone sends whoever reads it to - // check provisioning. - detailsLogged++ - log.Printf("registration with %s rejected (status %d) for %q: %s", label, status, s.manifest.ID, detail) - if detailsLogged == maxRefusalDetailsPerStatus { - log.Printf("registration with %s for %q has now given %d different reasons for status %d; further reasons are suppressed until the status changes or a registration succeeds", - label, s.manifest.ID, detailsLogged, status) - } - default: - log.Printf("registration with %s rejected (status %d) for %q", label, status, s.manifest.ID) - } - lastStatus, lastDetail, lastErr = status, detail, "" - default: - lastErr = "" - } - // A beat that failed is retried further and further out. Every failure - // on the credential path — a refused registration, a refused exchange, - // a response this process cannot use — otherwise resolves to "run the - // whole thing again in `interval`", which turns one broken deployment - // into a mint (and an audited security event on the issuer) four times - // a minute per module, for as long as the solution runs. Backing off - // bounds that without ever giving up: the first success resets it. - if err != nil || status >= 300 { - failures++ - } else { - failures = 0 - detailsLogged = 0 - auth.succeeded() - } - // Which cap applies depends on what the failure cost. A refusal means - // the surface answered, so the exchange before it minted a token: that - // is the audited-event rate the long cap exists to bound. A beat that - // never got an answer at all (status 0: the exchange refused, the - // gateway was unreachable, the round trip timed out) minted nothing and - // costs the issuer nothing, and the long cap buys nothing for it while - // charging the full price — this solution stays absent from a host that - // may itself be perfectly healthy for the whole capped wait after the - // dependency comes back. Those beats retry on the short cap. - backoffCap := registrationBackoffCap - if status == 0 { - backoffCap = registrationUnreachedBackoffCap - } - select { - case <-ctx.Done(): - return - case <-time.After(jittered(backoff(interval, failures, backoffCap))): - } - } -} - -// maxRefusalDetailsPerStatus bounds how many distinct host-supplied reasons are -// logged for one unchanging status. High enough that a host working through a -// list of incompatibilities is followed to the end; low enough that a reason -// varying on every beat cannot turn the log into a per-beat stream. -const maxRefusalDetailsPerStatus = 5 - -// registrationJitter is the fraction of a wait that is randomized. Every -// heartbeat is started in the same instant by serve and shares one interval, so -// without it the two self-registration credentials run their exchange in -// lockstep — two simultaneous mints for the same solution id, every beat, for -// the life of the process. A per-id rate limit or a coarse jti on the issuer -// turns that into a registration that flaps for a reason invisible from here, -// so the beats are spread rather than aligned. -const registrationJitter = 0.2 - -// jittered spreads a wait over [d, d+registrationJitter*d]. It only ever waits -// longer, never shorter: the backoff a failing beat earned is a floor, and -// shortening it would undo the request-rate bound it exists to impose. -func jittered(d time.Duration) time.Duration { - spread := int64(float64(d) * registrationJitter) - if spread <= 0 { - return d - } - return d + time.Duration(rand.Int64N(spread)) -} - -// registrationBackoffCap bounds the retry interval of a registration the server -// answered with a refusal. Every such beat cost a mint — an audited event on the -// issuer — so this is the cap that bounds that rate: the cost of waiting is that -// a recovered gateway takes this long to see the module again, so it trades a -// bounded outage for a bounded mint rate rather than abandoning either. -const registrationBackoffCap = 2 * time.Minute - -// registrationUnreachedBackoffCap bounds the retry interval of a beat that never -// reached the registration surface. Nothing was minted and nothing was audited, -// so the only thing the wait buys is not hammering a dependency that is already -// failing fast — which a beat every 30s achieves — while the thing it costs is -// paid by a surface that did nothing wrong. Both self-registrations now mint -// through the gateway, so a gateway restart fails the *host frontend* -// registration too; on the long cap the solution stayed missing from a healthy -// host's nav for up to that cap plus jitter after the gateway was back, which is -// a self-inflicted outage rather than a bounded one. -const registrationUnreachedBackoffCap = 30 * time.Second - -// backoff returns how long to wait before the next beat after `failures` -// consecutive failed ones: the steady interval while healthy, doubling while -// broken, never past backoffCap. -// -// An interval a caller chose that is already longer than the cap is honoured -// as-is rather than shortened — a cap exists to slow retries down, so applying -// it to an interval already slower than itself would have a failing beat retry -// *sooner* than a healthy one. That mattered little while the only cap was two -// minutes; with registrationUnreachedBackoffCap at 30s it is reachable by any -// caller who chooses a slower beat. -func backoff(interval time.Duration, failures int, backoffCap time.Duration) time.Duration { - if interval >= backoffCap { - return interval - } - wait := interval - for range failures { - if wait >= backoffCap/2 { - return backoffCap - } - wait *= 2 - } - return wait -} - -// defaultRegistrationInterval is how long a registration heartbeat waits -// between beats when neither the server nor -// RegistrationIntervalEnvironmentVariable names one. -const defaultRegistrationInterval = 15 * time.Second - -// beat performs one registration POST and returns the HTTP status — with, on a -// refusal, whatever reason the server gave — or an error if the request could -// not be built or the round trip failed. -func (s *Server) beat(ctx context.Context, url string, body []byte, auth registrationAuth) (int, string, error) { - req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(body)) - if err != nil { - return 0, "", err - } - req.Header.Set("content-type", "application/json") - expiresAt, err := auth.authorize(ctx, req) - if err != nil { - return 0, "", err - } - // Never let the POST outlive the credential it carries. registrationTimeout - // allows this request 10s, which is longer than a short-lived registration - // token's entire life: without this bound a token minted with, say, 2s left - // is presented to a host that answers in 3s, arrives expired, and comes back - // as a 401 — indistinguishable at this layer from the id being owned by - // another publisher, which is exactly what invalidate() would then report. - // Bounded by the expiry instead, the same case surfaces as this credential's - // own deadline, named below, and the next beat mints a fresh token. - expired := func() bool { return false } - if !expiresAt.IsZero() { - deadlined, cancel := context.WithDeadline(ctx, expiresAt) - defer cancel() - req = req.WithContext(deadlined) - // Only the credential's deadline, not a shutdown, which the heartbeat - // recognises by the parent context and reports as no failure at all. - expired = func() bool { return deadlined.Err() != nil && ctx.Err() == nil } - } - resp, err := registrationClient.Do(req) - if err != nil { - if expired() { - return 0, "", fmt.Errorf("registration did not complete before the credential it presented expired at %s: %w (the issuer's token lifetime is shorter than this registration round trip)", - expiresAt.UTC().Format(time.RFC3339), err) - } - return 0, "", err - } - defer drainAndClose(resp) - return resp.StatusCode, refusalDetail(resp), nil -} - -// refusalDetailMaxBytes bounds how much of a refusal body is read for the log. -// A host's reasons are a short JSON object; anything larger is not a reason. -const refusalDetailMaxBytes = 4 << 10 - -// refusalDetail extracts the reason a registration was refused, when the host -// gave one: the `reasons` list of an incompatible_runtime refusal, else its -// `error` code. Empty for an accepted registration or a bodiless refusal. -func refusalDetail(resp *http.Response) string { - if resp.StatusCode < 400 { - return "" - } - var refusal struct { - Error string `json:"error"` - Reasons []string `json:"reasons"` - } - if err := json.NewDecoder(io.LimitReader(resp.Body, refusalDetailMaxBytes)).Decode(&refusal); err != nil { - return "" - } - switch { - case len(refusal.Reasons) > 0 && refusal.Error != "": - return oneLine(refusal.Error + ": " + strings.Join(refusal.Reasons, "; ")) - case len(refusal.Reasons) > 0: - return oneLine(strings.Join(refusal.Reasons, "; ")) - default: - return oneLine(refusal.Error) - } -} - -// oneLine flattens the control characters out of text the host chose, so a -// reason carrying a newline cannot forge a log line of its own. The host is -// composition-local, but its reasons quote values this runtime never saw — a -// manifest field, an id another publisher registered — so what reaches the log -// is the host's text and never the host's framing. -func oneLine(s string) string { - return strings.Map(func(r rune) rune { - if unicode.IsControl(r) { - return ' ' - } - return r - }, s) -} - -// exchangeFailureBody is what the gateway said alongside a failed exchange, -// bounded and flattened to one log line: the host's own words are the only -// clue to which of its dependencies is down. "no body" when it said nothing. -func exchangeFailureBody(resp *http.Response) string { - raw, _ := io.ReadAll(io.LimitReader(resp.Body, refusalDetailMaxBytes)) - if body := strings.TrimSpace(oneLine(string(raw))); body != "" { - return body - } - return "no body" -} - -// drainAndClose consumes what is left of a response body before closing it, so -// the connection returns to the pool instead of being dropped and re-dialled on -// the next beat. Bounded: a body larger than this is not worth reading to keep -// one connection. -func drainAndClose(resp *http.Response) { - _, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, 64<<10)) - resp.Body.Close() -} - -// --- Module registration credentials --- -// -// The gateway federates a consumed module's /v1//* only against a -// signed, prefix-bound token issued by accounts — the shared cluster-internal -// token buys no per-caller binding and is refused. A backend obtains one by -// presenting the registration secret its composition provisioned for that -// prefix; the gateway brokers the exchange, because a composed module cannot -// reach accounts' internal listener itself. - -const ( - moduleRegisterPath = "/modules/_register" - moduleRegistrationTokenPath = "/modules/_registration-token" -) - -const ( - internalTokenHeader = "X-Codefly-Internal-Token" - // moduleRegistrationHeader carries the signed, prefix-bound token - // /modules/_register requires. - moduleRegistrationHeader = "X-Codefly-Module-Registration" - // moduleSecretHeader carries this backend's own registration secret on the - // exchange that mints that token. - moduleSecretHeader = "X-Codefly-Module-Secret" -) - -// ModuleRegistrationSecretsEnvironmentVariable carries the registration secrets -// a composition provisioned into this backend, as comma-separated -// `prefix:secret` entries — the plaintext twin of the `prefix:sha256hex` -// digests the same composition declared to accounts. It is absent for a -// composition that federates nothing. -const ModuleRegistrationSecretsEnvironmentVariable = "CODEFLY__MODULE_REGISTRATION_SECRETS" - -// moduleTokenRenewal is the most lead time the credential takes before an -// expiry, so a beat never presents a token that lapses between here and the -// gateway's check. It is a ceiling, not a requirement on the issuer: a token -// whose whole life is shorter is renewed at half its lifetime instead (see -// exchange). Conflating the two rejected every token the issuer chose to make -// short-lived — a 30s credential was refused outright, with a message blaming -// this host's clock. -const moduleTokenRenewal = 30 * time.Second - -// moduleTokenMinimumLifetime is the least remaining validity an issued token -// must carry to be worth presenting at all. Below this the token would lapse -// mid-flight, so it is a boundary error rather than a credential. -const moduleTokenMinimumLifetime = 5 * time.Second - -// registrationExchangeTimeout bounds one credential exchange — the module's and -// the solution's alike, which need the identical bound rather than one of them -// reaching into the other's constant. The registration beat is on a 15s cycle, -// so a stalled gateway must surface as a failed beat rather than wedge the beat -// loop for that module. -const registrationExchangeTimeout = 10 * time.Second - -// registrationClient carries every registration request: the credential -// exchange and the three registration heartbeats. It is NOT http.DefaultClient. -// -// That transport carries Proxy: ProxyFromEnvironment, so with HTTP(S)_PROXY set -// and a NO_PROXY that does not cover the host's in-cluster names, these -// requests would be dialled to an arbitrary egress host — carrying, in headers, -// the module's plaintext registration secret, the cluster-internal token, and -// the signed token that decides where authenticated traffic for a prefix is -// forwarded. Every registration target is composition-local (each is resolved -// from the SDK's endpoint map), so none of them may be proxied. The issuing -// side of this same exchange refuses proxying for exactly this reason -// (module-saas-starter#527). -// -// It also refuses to follow redirects, for the same reason it refuses a proxy. -// net/http strips only Authorization, WWW-Authenticate and Cookie when a -// redirect crosses to another host; every other header — including the -// registration secrets and the cluster-internal token these requests carry in -// X-Codefly-* headers — is copied to the new target verbatim, and a 307/308 -// re-sends the body with them. So anything able to answer at a registration URL -// with a Location (a gateway defect, an SSRF through it, a stale Service or DNS -// record claiming that name) would be handed the plaintext credential that this -// whole publisher binding exists to protect, and the theft would look like an -// ordinary successful beat. No registration endpoint has any reason to redirect, -// so a redirect is surfaced as the response it is and never followed. -// -// It also carries a Timeout. Without one, neither the heartbeat's context (which -// has no deadline) nor the transport bounds a gateway that accepts a -// registration and never answers: the beat blocks in Do forever, so that module -// — or, for the self-registrations, this whole solution — silently stops -// registering for the lifetime of the process, with no log and no recovery. The -// exchange bounds itself with registrationExchangeTimeout; this bounds the other -// three call sites the same way. -var registrationClient = &http.Client{ - Timeout: registrationTimeout, - Transport: newRegistrationTransport(), - CheckRedirect: refuseRedirect, -} - -// refuseRedirect stops the client at the redirect itself: the 3xx is returned as -// the response, so the beat reports it like any other refusal instead of -// replaying this request's headers at whatever host the Location named. -func refuseRedirect(*http.Request, []*http.Request) error { - return http.ErrUseLastResponse -} - -// registrationTimeout bounds one registration request end to end. It matches -// registrationExchangeTimeout: a stalled gateway must surface as a failed beat, -// which the heartbeat then backs off, rather than wedge the loop. -const registrationTimeout = 10 * time.Second - -func newRegistrationTransport() *http.Transport { - transport := http.DefaultTransport.(*http.Transport).Clone() - transport.Proxy = nil - return transport -} - -// parseModuleRegistrationSecrets decodes the comma-separated `prefix:secret` -// projection. An entry that is not a `prefix:secret` pair is dropped: the -// consequence — that module cannot register — is reported where it is -// actionable, by the missing-secret log in registerConsumedAPIs. -// -// Both halves are trimmed, matching how the registrar parses the digest twin of -// this same projection. Trimming only the entry left "documents : s3cret" -// keyed under "documents " with a leading space in the secret — a pair the -// registrar accepts and this side silently mangles into a lookup miss. -func parseModuleRegistrationSecrets(raw string) map[string]string { - secrets := map[string]string{} - for _, entry := range strings.Split(raw, ",") { - prefix, secret, ok := strings.Cut(entry, ":") - if !ok { - continue - } - prefix, secret = strings.TrimSpace(prefix), strings.TrimSpace(secret) - if prefix == "" || secret == "" { - continue - } - secrets[prefix] = secret - } - return secrets -} - -// registrationAuth stamps the credential one registration POST presents, and -// drops it when the server refuses it. The host and gateway self-registrations -// present a short-lived token bound to this solution's publisher; a module -// registration presents one bound to the single prefix it may claim. Neither -// presents the shared cluster-internal token: it attests to no publisher and -// both surfaces refuse it (module-saas-starter#540). -type registrationAuth interface { - // authorize stamps the credential on req and reports when that credential - // stops being valid, or the zero time when it cannot expire mid-request. - // The caller bounds the request by that instant: a credential that lapses - // while the POST is still in flight is refused on arrival, and a refusal is - // the one answer that cannot be told apart from the causes invalidate() - // names, so it must surface as this credential's own deadline instead. - authorize(ctx context.Context, req *http.Request) (expiresAt time.Time, err error) - invalidate() - // succeeded reports that the registration this credential authorized was - // accepted. It closes a refusal episode, so the next refusal is judged on - // its own rather than against a retry an earlier one already spent. - succeeded() -} - -// moduleCredential exchanges one consumed module's registration secret for the -// short-lived token /modules/_register requires, and holds it until renewal. -// Each instance is owned by exactly one registration beat, so its cached token -// needs no locking. -type moduleCredential struct { - tokenURL string - internalToken string - prefix string - secret string - - token string - // expiresAt is when token stops being honoured — not when it is renewed. - // The registration POST is bounded by it, because a token that lapses - // in flight is refused on arrival for a reason no refusal can express. - expiresAt time.Time - // renewAt is when the exchange must run again. Zero means "no usable - // credential": either none has been obtained yet, or one was dropped. - renewAt time.Time - // mintedThisBeat records that token was obtained during the beat now in - // flight, so a refusal of it cannot be blamed on staleness. - mintedThisBeat bool - // retriedAfterRefusal records that a refusal has already been answered with - // a fresh mint, and that mint has not yet been superseded by a natural - // renewal. It caps a refusal at one re-mint. - retriedAfterRefusal bool - warnedFreshRefusal bool - warnedShortLifetime bool -} - -func (c *moduleCredential) authorize(ctx context.Context, req *http.Request) (time.Time, error) { - c.mintedThisBeat = false - // A zero renewAt means no credential yet, or one just dropped, so the - // exchange runs. It cannot mean a cached-but-unusable token: exchange - // refuses any expiry this process could not act on. - if !time.Now().Before(c.renewAt) { - // Renewing a token still held is a fresh episode: whatever refusal the - // last re-mint was answering is over, so the next one earns its own retry. - natural := c.token != "" - token, expiresAt, renewAt, err := c.exchange(ctx) - if err != nil { - return time.Time{}, err - } - if natural { - c.retriedAfterRefusal = false - } - c.token, c.expiresAt, c.renewAt, c.mintedThisBeat = token, expiresAt, renewAt, true - } - req.Header.Set(moduleRegistrationHeader, c.token) - return c.expiresAt, nil -} - -// invalidate drops the cached token so the next beat obtains a fresh one — but -// only when a fresh one could plausibly help. -// -// A registration can be refused for reasons a new token cannot repair: a gateway -// that does not trust the issuer, a prefix this module may not claim, skew on the -// gateway's clock, a perimeter check rejecting the request before the token is -// ever examined. Dropping unconditionally turned every one of those into a mint -// on every beat — an audited security event on the issuer, four times a minute -// per module, indefinitely, and silent after the first log line because the -// status never changes. Staleness is the one cause re-minting fixes, so a -// refusal buys exactly one re-mint: a token minted for this very beat is not -// stale, and neither is the replacement a previous refusal already bought. -func (c *moduleCredential) invalidate() { - if c.mintedThisBeat || c.retriedAfterRefusal { - if c.mintedThisBeat && !c.warnedFreshRefusal { - c.warnedFreshRefusal = true - // This is the signature of a cause outside the credential, so name - // it: an operator reading "rejected (status 401)" alone would go - // looking at provisioning, which is the one thing already proven - // fine — the exchange that minted this token accepted the secret. - log.Printf("registration for %q was refused while presenting a token minted for that same beat: the credential is not stale, so re-minting cannot fix it — check that the gateway trusts the issuer that signed it, that %q may be claimed by this module, and whether %s admits %s at all", - c.prefix, c.prefix, moduleRegisterPath, internalTokenHeader) - } - return - } - // Spend the retry only when a drop actually follows, so the next beat's - // fresh mint is the one attempt this refusal was owed. - c.retriedAfterRefusal = true - c.token, c.renewAt = "", time.Time{} -} - -// succeeded ends the refusal episode: a registration this credential authorized -// was accepted, so a later refusal is a new fault — a second key rotation, say — -// and earns its own re-mint rather than inheriting the spent budget of the first. -func (c *moduleCredential) succeeded() { - c.retriedAfterRefusal = false -} - -// exchange runs the credential exchange against the gateway. It presents both -// the module secret, which identifies this module to accounts, and the -// cluster-internal token, which the gateway's own perimeter check requires. -// -// It returns the token, when it expires, and when to renew it. Expiry and -// renewal are separate answers: renewal is early by a lead, while expiry is the -// instant the gateway stops honouring the token, which is what a registration -// POST must not outlive. -func (c *moduleCredential) exchange(ctx context.Context) (string, time.Time, time.Time, error) { - ctx, cancel := context.WithTimeout(ctx, registrationExchangeTimeout) - defer cancel() - - body, _ := json.Marshal(map[string]string{"prefix": c.prefix}) - req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.tokenURL, bytes.NewReader(body)) - if err != nil { - return "", time.Time{}, time.Time{}, err - } - req.Header.Set("content-type", "application/json") - // Absent rather than present-and-empty when unconfigured: an empty header - // is the harder shape to diagnose at the gateway, which sees a caller - // claiming a credential it does not have. - if c.internalToken != "" { - req.Header.Set(internalTokenHeader, c.internalToken) - } - req.Header.Set(moduleSecretHeader, c.secret) - - resp, err := registrationClient.Do(req) - if err != nil { - return "", time.Time{}, time.Time{}, err - } - defer drainAndClose(resp) - if resp.StatusCode != http.StatusOK { - return "", time.Time{}, time.Time{}, fmt.Errorf("registration token exchange for %q rejected (status %d)", c.prefix, resp.StatusCode) - } - var issued struct { - Token string `json:"token"` - ExpiresAt time.Time `json:"expiresAt"` - } - if err := json.NewDecoder(resp.Body).Decode(&issued); err != nil { - return "", time.Time{}, time.Time{}, fmt.Errorf("registration token exchange for %q returned invalid json: %w", c.prefix, err) - } - if issued.Token == "" { - return "", time.Time{}, time.Time{}, fmt.Errorf("registration token exchange for %q returned no token", c.prefix) - } - // An expiry this process cannot use is a boundary error, not a token to - // cache: treated as "already expired" it would look like success while - // re-running the exchange on every beat. The two unusable shapes have - // different causes and different fixes, so they get different messages — - // the old single message blamed the clock for a response that never carried - // an expiry at all. - now := time.Now() - lifetime := issued.ExpiresAt.Sub(now) - switch { - case issued.ExpiresAt.IsZero(): - return "", time.Time{}, time.Time{}, fmt.Errorf( - "registration token exchange for %q returned no expiry: the issuer omitted expiresAt, or spelled it differently (protobuf JSON spells it expires_at, which does not decode into this field)", - c.prefix) - case lifetime < moduleTokenMinimumLifetime: - return "", time.Time{}, time.Time{}, fmt.Errorf( - "registration token exchange for %q returned the expiry %s, already past or less than %s away (check this host's clock against the issuer's)", - c.prefix, issued.ExpiresAt.UTC().Format(time.RFC3339), moduleTokenMinimumLifetime) - } - // Renew ahead of expiry, but never by more than half the credential's own - // life: a lead longer than the lifetime would reject the token outright, and - // the issuer's chosen lifetime is not this process's to veto. - lead := moduleTokenRenewal - if half := lifetime / 2; half < lead { - lead = half - if !c.warnedShortLifetime { - c.warnedShortLifetime = true - // Honoured, but say so once: at this lifetime the token cannot be - // held across many beats, so the mint rate is the issuer's setting - // and not a defect here. - log.Printf("registration token for %q is issued with a %s lifetime, shorter than the %s renewal lead: it will be re-obtained roughly every %s", - c.prefix, lifetime.Round(time.Second), moduleTokenRenewal, lead.Round(time.Second)) - } - } - return issued.Token, issued.ExpiresAt, issued.ExpiresAt.Add(-lead), nil -} - -// --- Solution registration credentials --- -// -// The host binds a solution's registration to its publisher: the frontend -// accepts a manifest, and the gateway an upstream, only against a signed, -// solution-bound token that accounts mints to a caller presenting the -// registration secret the composition declared for that id. The shared -// cluster-internal token attests to no publisher and is refused on both -// (module-saas-starter#540). A token is minted per attempt and burned on use by -// each surface, so unlike the module credential there is nothing to cache: the -// exchange runs before every beat. - -const ( - solutionRegisterPath = "/solutions/_register" - solutionRegistrationTokenPath = "/solutions/_registration-token" -) - -const ( - // solutionRegistrationHeader carries the signed, solution-bound token both - // /solutions/_register and /api/solutions/register require. - solutionRegistrationHeader = "X-Codefly-Solution-Registration" - // solutionSecretHeader carries this solution's own registration secret on - // the exchange that mints that token. - solutionSecretHeader = "X-Codefly-Solution-Secret" -) - -// SolutionRegistrationSecretEnvironmentVariable carries the registration -// secret a composition provisioned for this solution — the plaintext twin of -// the `:sha256hex` digest the same composition declared to the host in its -// `SOLUTION_REGISTRATION_SECRETS`. It is an explicit override; the workspace -// secret below is the declared path. -const SolutionRegistrationSecretEnvironmentVariable = "CODEFLY__SOLUTION_REGISTRATION_SECRET" - -// SolutionRegistrationSecretGroup and SolutionRegistrationSecretKey name the -// workspace secret the runtime resolves through the SDK when no override is -// set: a composition declares the `solution-registration` group as a -// workspace-configuration dependency of the solution's backend and provisions -// `SECRET` in it (`codefly config generate solution-registration SECRET` -// locally; the cell's secret store when deployed). -const ( - SolutionRegistrationSecretGroup = "solution-registration" - SolutionRegistrationSecretKey = "SECRET" -) - -// solutionTokenMinimumLifetime is the least remaining validity a minted -// registration token must carry to be worth presenting. This credential is -// presented on the very next round trip rather than held between beats, so the -// floor bounds one request: whatever lifetime the issuer chose, a token that -// outlives the registration POST is usable and must not be refused here. -// -// moduleTokenMinimumLifetime is 5s because that token *is* cached across beats, -// and copying the figure over rejected every genuinely single-use credential an -// issuer might mint — a 3s token failed every beat forever, with a message -// blaming this host's clock. The two floors bound different things, so they are -// deliberately different values. -const solutionTokenMinimumLifetime = time.Second - -// solutionMintReportEvery is how many mints pass between one line reporting the -// running count. Minting one token per beat per surface is the contract's -// consequence and not a defect, but it is an audited event on the issuer, and -// the only place its rate could be observed was the issuer's own audit trail. -// If the burn-per-surface premise this design rests on ever stops holding, this -// is the number that says what abandoning the cache is costing. -const solutionMintReportEvery = 100 - -// solutionCredential mints this solution's registration token and presents it -// on one registration beat. Each self-registration heartbeat owns one, so it -// needs no locking. -type solutionCredential struct { - tokenURL string - internalToken string - id string - secret string - - // refused records that the host refused the token the beat now in flight - // presented. Nothing here is cached, so a refusal can never be staleness; - // the flag only bounds the diagnostic to one line per refusal episode. - refused bool - // minted counts the tokens this credential has obtained, so the cost of - // minting one per beat is visible here and not only on the issuer. - minted int -} - -func (c *solutionCredential) authorize(ctx context.Context, req *http.Request) (time.Time, error) { - token, expiresAt, err := c.exchange(ctx) - if err != nil { - return time.Time{}, err - } - req.Header.Set(solutionRegistrationHeader, token) - return expiresAt, nil -} - -// invalidate is what a refusal calls. There is no cached token to drop — every -// beat presents one minted for it — so the refusal cannot be about staleness, -// and the only useful thing to do is say so, once: the exchange that minted the -// refused token already proved the secret, and the beat is bounded by the -// token's own expiry so it cannot have lapsed in flight either, which leaves -// the host's side of the credential (an id another publisher owns, a manifest -// the host finds incompatible, a gateway that does not trust the issuer). -func (c *solutionCredential) invalidate() { - if c.refused { - return - } - c.refused = true - log.Printf("registration for solution %q was refused while presenting a token minted for that same beat: the exchange accepted this solution's secret, so provisioning is not the cause — check that %q is not registered by another publisher and that the host trusts the issuer that signed the token", - c.id, c.id) -} - -func (c *solutionCredential) succeeded() { - c.refused = false +func (s *Server) handleCapabilities(w http.ResponseWriter, _ *http.Request) { + // The same majors the registration manifest declares, from the same + // constants. Hardcoded here, a bump of either constant would leave this + // document announcing the old major while the manifest announced the new + // one, and a host that checks both would see one solution claiming two + // different contracts. + writeJSON(w, http.StatusOK, map[string]any{ + "schemaVersion": solutionManifestSchemaMajor, + "contract": s.manifest.Contract, + "contractMajor": solutionHostContractMajor, + "capabilities": s.manifest.Capabilities, + }) } -// exchange runs the credential exchange against the gateway. It presents both -// the solution secret, which identifies this solution to accounts, and the -// cluster-internal token, which the gateway's own perimeter check requires. -func (c *solutionCredential) exchange(ctx context.Context) (string, time.Time, error) { - ctx, cancel := context.WithTimeout(ctx, registrationExchangeTimeout) - defer cancel() - - body, _ := json.Marshal(map[string]string{"id": c.id}) - req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.tokenURL, bytes.NewReader(body)) - if err != nil { - return "", time.Time{}, err - } - req.Header.Set("content-type", "application/json") - if c.internalToken != "" { - req.Header.Set(internalTokenHeader, c.internalToken) +func (s *Server) wrapRequest(handler RequestHandler) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + bearer := r.Header.Get("authorization") + if bearer == "" { + writeJSON(w, http.StatusUnauthorized, map[string]string{"error": "missing bearer"}) + return + } + // Before the handler, because the handler is what acts. See + // actingForAViewer: a plain handler that mints nothing still receives a + // gateway carrying the viewer's bearer, and sending that anywhere under + // a credential the issuer has withdrawn is acting without authority. + if err := s.actingForAViewer(r.Context()); err != nil { + // The sanitized sentence, not the error. The error wraps whatever + // the source said — a mint URL, an issuer's own text, a dial + // failure naming an internal address — and this route gate wrote + // it straight into the viewer's response and the log, which is the + // disclosure handlerErrorResponse exists to prevent. Adding a gate + // in front of the handler put a new path around the sanitization + // rather than through it. + log.Printf("solution %q: refusing to act for a viewer: %v", s.manifest.ID, err) + writeJSON(w, http.StatusServiceUnavailable, map[string]string{"error": credentialRefusalForAViewer(err)}) + return + } + result, err := handler(r, s.gatewayFor(r.Header)) + if err != nil { + status, message := handlerErrorResponse(err) + writeJSON(w, status, map[string]string{"error": message}) + return + } + writeJSON(w, http.StatusOK, result) } - req.Header.Set(solutionSecretHeader, c.secret) +} - resp, err := registrationClient.Do(req) - if err != nil { - return "", time.Time{}, err - } - defer drainAndClose(resp) - switch resp.StatusCode { - case http.StatusOK: - case http.StatusNotFound: - // An unrouted path: a host from before the publisher-bound contract - // (module-saas-starter < v0.0.61, not supported), or a wrong URL. There - // is no other credential to fall back to — the shared cluster-internal - // token proves no publisher — so the beat fails and names the route. - return "", time.Time{}, fmt.Errorf("the solution registration exchange at %s answered 404: this host does not serve publisher-bound registration (module-saas-starter < v0.0.61 is not supported) or the URL is wrong (%s, or the %s it is derived from)", - c.tokenURL, "GATEWAY_SOLUTION_REGISTRATION_TOKEN_URL", "GATEWAY_REGISTER_URL") - case http.StatusUnauthorized, http.StatusForbidden: - // accounts answers an undeclared id and a wrong secret identically, and - // the gateway relays that; naming both here is what the reader needs. - // Only these two statuses are a verdict on the credential: they are the - // ones accounts answers after reading the secret. - return "", time.Time{}, fmt.Errorf("registration token exchange for solution %q rejected (status %d): the host declares no %s entry for this id, or the provisioned secret does not match its digest", - c.id, resp.StatusCode, "SOLUTION_REGISTRATION_SECRETS") - default: - // Anything else was never a judgement of the secret. Blaming the - // provisioning for it sent an operator to re-provision a secret that - // was correct while the gateway could not reach accounts (a 502 relayed - // as "the provisioned secret does not match its digest"). A 5xx is the - // host being unavailable; the heartbeat retries it on its short cap, so - // say that, with what the gateway said. - if resp.StatusCode >= 500 { - return "", time.Time{}, fmt.Errorf("registration token exchange for solution %q: host unavailable (status %d), retrying: %s", - c.id, resp.StatusCode, exchangeFailureBody(resp)) - } - return "", time.Time{}, fmt.Errorf("registration token exchange for solution %q failed (status %d), retrying: %s", - c.id, resp.StatusCode, exchangeFailureBody(resp)) - } - var issued struct { - Token string `json:"token"` - ExpiresAt time.Time `json:"expiresAt"` - } - if err := json.NewDecoder(resp.Body).Decode(&issued); err != nil { - return "", time.Time{}, fmt.Errorf("registration token exchange for solution %q returned invalid json: %w", c.id, err) - } - if issued.Token == "" { - return "", time.Time{}, fmt.Errorf("registration token exchange for solution %q returned no token", c.id) - } - // The token is presented once, right now: an expiry that is absent or - // already too close is a boundary error, not a credential. - switch lifetime := time.Until(issued.ExpiresAt); { - case issued.ExpiresAt.IsZero(): - return "", time.Time{}, fmt.Errorf("registration token exchange for solution %q returned no expiry: the issuer omitted expiresAt, or spelled it differently", c.id) - case lifetime < solutionTokenMinimumLifetime: - return "", time.Time{}, fmt.Errorf("registration token exchange for solution %q returned the expiry %s, already past or less than %s away (check this host's clock against the issuer's)", - c.id, issued.ExpiresAt.UTC().Format(time.RFC3339), solutionTokenMinimumLifetime) - } - if c.minted++; c.minted%solutionMintReportEvery == 0 { - // Per credential, not per solution: each self-registration surface has - // its own, so the solution's actual mint rate is this figure times the - // number of surfaces. Saying so matters — read as a whole-solution - // total it understates the audited-event rate it exists to expose. - log.Printf("solution %q: %d registration tokens minted on this credential alone (one self-registration surface, one mint per beat) — each an audited mint on the issuer. The host burns a token per surface, so none can be cached; this is what that costs", - c.id, c.minted) - } - return issued.Token, issued.ExpiresAt, nil +// drainAndClose consumes what is left of a response body before closing it, so +// the connection returns to the pool instead of being dropped and re-dialled on +// the next request. Bounded: a body larger than this is not worth reading to +// keep one connection. +func drainAndClose(resp *http.Response) { + _, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, 64<<10)) + // Discarded deliberately: the response is already read and the caller has + // what it came for, so a close failure changes nothing it could act on. + _ = resp.Body.Close() } // Gateway is a client bound to the caller's bearer. It exposes an HTTP client @@ -1959,6 +2306,17 @@ type Gateway struct { // orgID is the viewer's organization, which the bearer alone does not // carry. A Work Context is minted inside exactly one org. orgID string + // installationErr is why installationID is empty, when it is. + installationErr error + // sanitizeErrors makes this gateway hand back errors a caller may relay + // verbatim. Set on the MCP path, where a tool's error becomes content the + // client reads and nothing downstream can tell it from the author's own. + sanitizeErrors bool + // installationID is the installation the gateway stamped for this request, + // which every mint this gateway runs names. Empty when none arrived, which + // is a refusal rather than a mint under an unnamed installation. It is not + // read from a carried capability: see viewerInstallation. + installationID string // sessionID is the viewer's session, which the bearer does not carry // either. Accounts seals the selected organization into that session, so // rooting a Task in it is what makes the mint accounts journals name the @@ -1968,6 +2326,29 @@ type Gateway struct { // is given. It holds the ask rather than a capability: see delegation. delegation *delegation contexts *workContextCache + // workload is this execution's own credential, presented on the mint this + // runtime runs for a viewer so the issuer knows which module is asking and + // can hold the mint to the installation and binding that credential is + // sealed to. It is never presented to a consumed module: what a module + // verifies is the viewer's capability, and this one attests only to the + // process that asked for it. Nil on a gateway built without one, which is + // every gateway in a test that does not exercise the attestation. + workload credentialAcquirer + // id is this solution's manifest id, for the log line a failed attestation + // writes, and report throttles that line. + id string + report *attestationReport + // transport is what this gateway dials with: the boot's authenticated + // client, or nil on a gateway built outside a boot. + transport http.RoundTripper + // terminal reports a credential failure the issuer will not reverse, so the + // process can stop claiming to be healthy. Nil outside a boot. + terminal func(error) + // ceilings is the published contract's ceiling per audience, which every + // ForModule ask is held to. Non-nil exactly when a contract was resolved, + // which only a boot does; a derived gateway inherits it, so chaining + // ForModule off one cannot escape it. + ceilings map[string][]Scope } func newGateway(baseURL, bearer, orgID, sessionID string) *Gateway { @@ -1980,6 +2361,42 @@ func newGateway(baseURL, bearer, orgID, sessionID string) *Gateway { } } +// gatewayFor is the client a handler or the passthrough is given: one bound to +// the caller the headers identify, carrying this execution's credential for the +// mints it will run. +func (s *Server) gatewayFor(header http.Header) *Gateway { + gw := newGateway(s.cfg.gatewayURL, header.Get("authorization"), header.Get(orgHeader), header.Get(sessionHeader)) + // Kept with its reason, so ForModule can say why rather than only that it + // is absent. + gw.installationID, gw.installationErr = viewerInstallation(header) + gw.id, gw.report = s.manifest.ID, &s.attestation + // Left nil when no source was supplied, so a mint is refused for want of + // one rather than panicking inside the controller. + if s.credential != nil { + gw.workload = s.acquireCredential + } + gw.terminal = s.credentialRefused + // The ceiling this process published, carried onto the gateway a handler + // gets, so what it mints is held to what the contract claims. Resolved + // contracts only, read off the flag the boot sets — see contractResolved + // for why this was the solution's id and why that was wrong. + if s.contractResolved { + gw.ceilings = make(map[string][]Scope, len(s.contract.Bindings)) + for _, binding := range s.contract.Bindings { + gw.ceilings[binding.Audience] = binding.Ceiling + } + } + if s.outbound != nil { + // The transport, not the client: the client's redirect policy and + // timeout are re-applied by platformClient below, which every client + // this gateway builds goes through. Taking .Transport alone was the + // bug — it dropped exactly the CheckRedirect that kept credentials + // from being re-sent to a Location. + gw.transport = s.outbound.Transport + } + return gw +} + func (g *Gateway) BaseURL() string { return g.baseURL } // OrgID is the viewer's active organization: the x-org-id identity header the @@ -1993,11 +2410,14 @@ func (g *Gateway) OrgID() string { return g.orgID } // a gateway derived by ForModule, the viewer's Work Context — on every request. // Satisfies connect.HTTPClient. Advanced escape hatch — prefer Unary. func (g *Gateway) HTTPClient() *http.Client { - transport := bearerTransport{bearer: g.bearer, base: gatewayTransport} + transport := bearerTransport{bearer: g.bearer, base: g.roundTripper()} if g.delegation != nil { transport.acting = g } - return &http.Client{Transport: transport} + // No timeout: this client carries streams, whose bound is the method's + // declared MaxStreamDuration on the request context, and a client timeout + // would cut a conforming stream off mid-flight. + return g.platformClient(transport, 0) } // bearerClient carries the viewer's bearer and nothing else. The mint runs on @@ -2006,22 +2426,454 @@ func (g *Gateway) HTTPClient() *http.Client { // verifies every presented context, so once the first lapsed it would 401 the // very call meant to replace it. func (g *Gateway) bearerClient() *http.Client { - return &http.Client{Transport: bearerTransport{bearer: g.bearer, base: gatewayTransport}} + return g.platformClient(bearerTransport{bearer: g.bearer, base: g.roundTripper()}, platformRequestTimeout) +} + +// platformClient is every client this gateway dials with, and it exists so +// there is one place the two properties below cannot be forgotten. +// +// **A redirect is never followed.** This was a blocker, and the shape of it is +// worth keeping written down because every piece of it looked right on its own. +// The boot's outbound client did set CheckRedirect; gatewayFor copied only its +// .Transport, so every gateway client was an http.Client literal with Go's +// default policy, which follows up to ten hops. Go copies the original +// request's headers onto a redirected one and strips only Authorization, +// WWW-Authenticate and Cookie — so the viewer's capability and the installation +// headers went along, and on the mint, this workload's own credential in +// x-codefly-work-context went with them. Worse, bearerTransport sets the bearer +// on *every* round trip, so the one header Go does strip was put straight back; +// and a 307 re-sends the body. A single `307 Location: http://elsewhere/` from +// the gateway, from accounts, or from any module the gateway relays therefore +// handed over the viewer's unscoped bearer, their capability and this +// workload's credential — in cleartext, since nothing about an http:// hop +// involves the TLS configuration that protects the first one. ErrUseLastResponse +// returns the 3xx to the caller instead: no hop, nothing re-sent. +// +// **And the request may only go to the origin this gateway was built for.** +// That is the belt to the braces: a client built anywhere without the policy +// above is still refused by the transport, which is where a property this +// serious belongs. It is also stricter than "must be https" and needs no +// special case for a test, because an http:// fake gateway is simply its own +// origin: every destination this gateway has is one host, and validate() +// already refuses a gateway URL that is not https in a real boot. +func (g *Gateway) platformClient(transport http.RoundTripper, timeout time.Duration) *http.Client { + // No error sanitizing here, and the reason is worth writing down because I + // tried it. + // + // A transport wrapper cannot remove the address: http.Client.Do wraps + // whatever the transport returns in its own *url.Error carrying the URL, + // after the wrapper has run. And it would be pointless if it could — the + // author built that request from BaseURL(), so the address is already + // theirs and the error discloses nothing they did not have. + // + // What the runtime must not do is hand an author an error carrying what + // they could NOT otherwise see: the mint URL, the issuer's own words, an + // internal dial failure from a credential ask. Those come from this + // package and go through Gateway.surfaced. Whether an author then relays + // their own error text to a client is their choice, like any other string + // they return, and the runtime cannot police it without rewriting it. + return &http.Client{ + Transport: transport, + Timeout: timeout, + CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }, + } +} + +// roundTripper carries every request this gateway makes: the module reads a +// handler issues, and the mint that authenticates them. +// +// It is never http.DefaultTransport. That transport carries Proxy: +// ProxyFromEnvironment, so with HTTP(S)_PROXY set and a NO_PROXY that does not +// cover the host's in-cluster names, these requests would be dialled to an +// arbitrary egress host — carrying the viewer's bearer, the capability minted on +// their behalf, and this workload's own credential. Every target is +// composition-local, so none of it may be proxied. +// +// A gateway built by a boot carries the transport that boot configured, which +// presents this workload's identity and verifies the far end against the anchor +// the platform projected. One built outside a boot — the passthrough test seam, +// a unit test — falls back to the unauthenticated non-proxied transport, and +// that is the only place it is used: there is no path on which a *booted* +// runtime dials the platform without presenting its identity. +func (g *Gateway) roundTripper() http.RoundTripper { + var base http.RoundTripper = unauthenticatedTransport + if g.transport != nil { + base = g.transport + } + return originPinned{base: base, origin: requestOrigin(g.baseURL)} +} + +// originPinned refuses a credential-bearing request to anywhere but the one +// origin its gateway addresses. +// +// Every destination a gateway has — a module read, a transcoded call, a stream, +// the viewer's mint — is a path on the gateway it was built for, so "same +// origin as the base URL" is not a restriction on anything this runtime does. +// What it stops is a request that was *redirected* somewhere else, or built by +// a future client that forgot the policy: the credentials on these requests are +// the viewer's bearer, the capability minted for them and this workload's own, +// and the check costs a string comparison. +// +// It refuses rather than rewrites, and it refuses an unparseable or empty base +// too, since a gateway that cannot say where it dials cannot be allowed to +// carry these headers anywhere. +type originPinned struct { + base http.RoundTripper + origin string +} + +func (t originPinned) RoundTrip(r *http.Request) (*http.Response, error) { + if t.origin == "" { + return nil, fmt.Errorf("refusing to send a credential-bearing request to %q: this gateway has no resolved origin to dial, so there is nothing to hold the destination to", r.URL.Redacted()) + } + if got := requestOrigin(r.URL.String()); got != t.origin { + return nil, fmt.Errorf("refusing to send a credential-bearing request to %s: this gateway dials %s, and these headers carry the viewer's bearer, the capability minted for them and this workload's own credential — a destination that is not the gateway is a redirect or a misbuilt client, never a call this runtime makes", + got, t.origin) + } + return t.base.RoundTrip(r) +} + +// requestOrigin is scheme://host, the unit the pin compares. A URL that does +// not parse, or names no host, has no origin and is refused by the caller. +func requestOrigin(raw string) string { + u, err := url.Parse(raw) + if err != nil || u.Scheme == "" || u.Host == "" { + return "" + } + return u.Scheme + "://" + u.Host +} + +// unauthenticatedTransport is the non-proxied default for a gateway built +// outside a boot. +var unauthenticatedTransport = newPlatformTransport() + +// newPlatformTransport is the default transport with proxying dropped. +func newPlatformTransport() *http.Transport { + transport := http.DefaultTransport.(*http.Transport).Clone() + transport.Proxy = nil + return transport +} + +// outboundClient is the client this runtime makes platform requests with: the +// credential mint, and every call a handler's gateway issues. +// +// This is the other half of refusing a plaintext destination. An https URL says +// only that the scheme is https: without a trust anchor the connection is +// verified against whatever the image's system roots happen to contain, and +// without a client certificate the far end cannot tell this workload from +// anything else that reached it. So the client presents this workload's +// X.509-SVID and verifies the far end against the anchor the platform +// projected — the same pair and the same anchor the listener uses, which is the +// point: one identity, used in both directions. +// +// It refuses to follow redirects. net/http strips only Authorization, +// WWW-Authenticate and Cookie when a redirect crosses hosts; every other header +// — the projected token on a mint, a viewer's capability on a module read — is +// copied to the new target verbatim, and a 307/308 re-sends the body with them. +// So anything able to answer with a Location would be handed the credentials +// that prove who this workload is, and the theft would look like an ordinary +// successful request. +func (s *Server) outboundClient(identityConfig *tls.Config) (*http.Client, error) { + // A fresh configuration per connection, so the anchor this runtime judges + // the platform by is re-read rather than snapshotted at boot. + // + // Judging by a stale anchor admits whoever should be refused, which is not + // symmetric with serving a stale leaf — the inbound argument, and it + // applies outbound with more force, not less. The two destinations on the + // other side of this + // client are the mint and the gateway — they receive the projected + // service-account token, the viewer's bearer and this workload's own + // credential — so a root removed from the bundle because it was compromised + // kept authenticating exactly the parties that are handed everything. + // + // Reloading per connection is the whole story only if connections do not + // outlive the reload, so established connections are re-verified too (see + // below). + // + // What is *not* rebuilt per connection is this workload's own leaf source. + // Building a whole new SDK reloader per dial, and again per recheck of + // every connection, looks like "nothing is snapshotted" and is a + // regression: NewCertificateReloader loads + // eagerly and returns the error, where a long-lived reloader swallows a + // failed re-read and keeps serving the last good pair. So a cert rotated + // non-atomically — the certificate written, the key a moment behind — + // failed every new dial, and failed every recheck, which closed + // established connections whose peers were still perfectly trusted. A + // half-written copy of this workload's own key is not a judgement about + // the peer, and the two must not share a failure path. + leaf, err := s.outboundLeaf(identityConfig) + if err != nil { + return nil, err + } + trust := s.peerTrustAnchor(identityConfig) + // Per destination, not per "platform": the authorization set is chosen by + // the address being dialled, so the mint's identity is not accepted at the + // gateway's address or the other way round, and a third address has no set + // and is refused. + newConfig := func(addr string) (*tls.Config, error) { + anchor, err := trust() + if err != nil { + return nil, err + } + config := &tls.Config{ + MinVersion: tls.VersionTLS13, + RootCAs: anchor, + GetClientCertificate: leaf, + } + // Held to the frozen principal at presentation, and pointed at the one + // identity allowed to answer at this address. Both apply to a + // consumer's source and to the projected pair: one of the two paths + // carrying a check is not the check. + if err := holdPresentedCertificate(config, s.principal); err != nil { + return nil, err + } + admitOnlyPlatform(config, s.admittedAt(addr)) + return config, nil + } + // Refused at boot rather than at the first dial: a configuration this + // runtime cannot build is a boot failure, not a mint that fails later. + if _, err := newConfig(dialAddress(s.cfg.gatewayURL)); err != nil { + return nil, err + } + + transport := newPlatformTransport() + // No TLSClientConfig: every connection builds its own below, and leaving a + // snapshot here would be the bug this function exists to avoid. + transport.TLSClientConfig = nil + transport.DialTLSContext = func(ctx context.Context, network, addr string) (net.Conn, error) { + config, err := newConfig(addr) + if err != nil { + return nil, err + } + host, _, err := net.SplitHostPort(addr) + if err != nil { + return nil, err + } + config = config.Clone() + config.ServerName = host + raw, err := (&net.Dialer{Timeout: platformRequestTimeout}).DialContext(ctx, network, addr) + if err != nil { + return nil, err + } + // The handshake carries its own deadline, independent of whatever the + // caller's context says. + // + // A custom DialTLSContext takes net/http out of the handshake entirely: + // it calls customDialTLS and never reaches addTLS, so + // Transport.TLSHandshakeTimeout — inherited as 10s from + // DefaultTransport — silently does nothing, and the only bound left is + // the request context's. That is adequate for an ordinary request and + // wrong exactly where it matters: Gateway.HTTPClient carries streams + // and therefore sets no client timeout on purpose, so a dial made for + // a stream was bounded by the method's declared MaxStreamDuration (up + // to thirty minutes) or by nothing at all. A handshake is not a + // stream: it completes in milliseconds or the peer is not answering. + bound := s.handshakeTimeout + if bound == 0 { + bound = platformHandshakeTimeout + } + handshake, cancelHandshake := context.WithTimeout(ctx, bound) + defer cancelHandshake() + conn := tls.Client(raw, config) + if err := conn.HandshakeContext(handshake); err != nil { + _ = raw.Close() + return nil, err + } + // Watched on the two inputs that are a judgement about the peer — the + // anchor and the admitted set — and on nothing else. + return watchOutboundTrust(conn, host, trust, s.admittedAt(addr)), nil + } + // Pool hygiene, not the trust bound: a connection nothing sends on is + // dropped rather than held open indefinitely. What bounds the trust + // decision is the recheck above, because an inactivity timeout cannot. + transport.IdleConnTimeout = outboundTrustReloadBound + return &http.Client{ + Timeout: platformRequestTimeout, + Transport: transport, + CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }, + }, nil +} + +// outboundLeaf is the certificate this runtime presents to the platform, +// resolved once and re-read by whatever produces it. +// +// Once, because the thing that keeps a rotated leaf presented is the reloader's +// own re-read, not a new reloader: the SDK's swallows a failed or half-written +// re-read and keeps serving the last good pair (sdk-go tls.go, refresh), which +// is the property that makes a rotation invisible to a caller. Building a new +// one per dial threw exactly that away, because the constructor loads eagerly +// and reports the error instead. +func (s *Server) outboundLeaf(identityConfig *tls.Config) (func(*tls.CertificateRequestInfo) (*tls.Certificate, error), error) { + if s.identity != nil { + // A consumer's source: the identity it returned, in both directions, + // rather than projected files it never said it uses. Its own callback + // is what gets asked, so a source that rotates is followed. + client, err := clientTLSFrom(s.identity, identityConfig) + if err != nil { + return nil, fmt.Errorf("configure this workload's outbound identity from the supplied identity source: %w", err) + } + if client.GetClientCertificate != nil { + return client.GetClientCertificate, nil + } + pairs := client.Certificates + return func(*tls.CertificateRequestInfo) (*tls.Certificate, error) { + if len(pairs) == 0 { + return nil, fmt.Errorf("the identity source produces no certificate this runtime can present to the platform") + } + return &pairs[0], nil + }, nil + } + reloader, err := codefly.NewCertificateReloader(s.cfg.identityCertFile, s.cfg.identityKeyFile) + if err != nil { + return nil, fmt.Errorf("configure this workload's outbound identity from %q/%q: %w — the same pair the listener presents is what names this workload to the platform it calls", + s.cfg.identityCertFile, s.cfg.identityKeyFile, err) + } + return reloader.GetClientCertificate, nil } -// gatewayTransport carries every request a solution makes through the gateway: -// the module reads a handler issues, and the mint that authenticates them. It -// is NOT http.DefaultTransport. +// outboundTrustReloadBound is how long a connection nobody is using is kept in +// the pool. Pool hygiene, nothing more. // -// That transport carries Proxy: ProxyFromEnvironment, so with HTTP(S)_PROXY set -// and a NO_PROXY that does not cover the host's in-cluster names, these requests -// would be dialled to an arbitrary egress host — carrying, in headers, the -// viewer's bearer and the signed capability minted on their behalf. The gateway -// is composition-local (its address is resolved from the SDK's endpoint map), -// exactly like every registration target, so none of this may be proxied -// either. It shares the registration transport's constructor because it needs -// the identical thing: the default transport with proxying dropped. -var gatewayTransport = newRegistrationTransport() +// It used to be described as the bound on outbound trust staleness, and it was +// not one: an inactivity timeout bounds only a connection that goes inactive, +// so a connection carrying traffic was never dropped and the documented window +// held for a quiet process only. What bounds the trust decision is +// outboundTrustRecheckInterval below. +const outboundTrustReloadBound = 30 * time.Second + +// outboundTrustRecheckInterval bounds how long an established platform +// connection may outlive the trust that authenticated it. +// +// It is the number the reload bound above could not deliver on its own. Peer +// trust is re-read on every dial, and that was documented as a 30-second +// window on the strength of IdleConnTimeout — but an *inactivity* timeout +// bounds only a connection nobody is using. A connection carrying a request +// every 100ms never becomes idle, and one was demonstrated still answering 31 +// seconds after the server's trust root was removed, over a single handshake: +// traffic kept the connection, and the trust decision behind it, alive for as +// long as it kept arriving. A stream is the same shape by construction, held +// open for as long as its declared duration allows (up to +// MaxStreamDurationLimit). +// +// A second, not the idle bound's thirty: the recheck is a file read and a chain +// verification against a chain already in hand, a few hundred microseconds on a +// handful of platform connections, so the window is set by what the guarantee +// should be rather than by what the work costs. It is also what makes the +// busy-connection case testable in about a second instead of half a minute. +const outboundTrustRecheckInterval = time.Second + +// watchOutboundTrust re-verifies an established connection's peer against +// current trust, and closes the connection when it stops verifying. +// +// Re-verifying rather than imposing a maximum connection age, which was the +// other way to bound this. A maximum age cuts whatever is in flight when it +// expires, and a declared 30-minute stream over a peer that is still perfectly +// trusted would be cut every bound — the runtime would be ending conforming +// streams to answer a question it can answer without ending them. The chain the +// peer presented is already in hand, so the anchor and the allowed-peer set can +// be re-read and applied to it: a connection whose peer still verifies is left +// alone, and one whose peer no longer does is closed, busy or idle, stream or +// request. +// +// A closed connection surfaces as a failed request or a cut stream, which is +// the intended outcome: the alternative is continuing to present this +// workload's credentials — and the viewer's — to a destination this cell has +// stopped vouching for. +func watchOutboundTrust(conn *tls.Conn, host string, trust func() (*x509.CertPool, error), peers func() ([]string, error)) net.Conn { + watched := &recheckedConn{Conn: conn, done: make(chan struct{})} + go func() { + ticker := time.NewTicker(outboundTrustRecheckInterval) + defer ticker.Stop() + for { + select { + case <-watched.done: + return + case <-ticker.C: + if err := peerStillTrusted(conn.ConnectionState(), host, trust, peers); err != nil { + _ = watched.Close() + return + } + } + } + }() + return watched +} + +// recheckedConn is the connection the transport holds, so the recheck stops +// when net/http drops it. +type recheckedConn struct { + net.Conn + stop sync.Once + done chan struct{} +} + +func (c *recheckedConn) Close() error { + c.stop.Do(func() { close(c.done) }) + return c.Conn.Close() +} + +// peerStillTrusted holds the chain this connection was authenticated by to +// trust as it is now: the current anchor, the current allowed-peer set, and the +// current time, which is also how a peer certificate that expired mid-stream +// stops being accepted. +// It takes the anchor and the admitted set, not a whole configuration to +// rebuild: a recheck that also resolved this workload's own leaf would close a +// trusted connection over a half-written copy of this process's own key, which +// is not a statement about the peer at all. +func peerStillTrusted(state tls.ConnectionState, host string, trust func() (*x509.CertPool, error), peers func() ([]string, error)) error { + anchor, err := trust() + if err != nil { + return err + } + // The private chain, for the reason immutableChain records: the objects in + // a ConnectionState are shared, and this runs for as long as the + // connection lives. + chain, err := immutableChain(state) + if err != nil { + return err + } + if len(chain) == 0 { + return fmt.Errorf("the established platform connection presents no peer certificate to re-verify") + } + intermediates := x509.NewCertPool() + for _, cert := range chain[1:] { + intermediates.AddCert(cert) + } + if _, err := chain[0].Verify(x509.VerifyOptions{ + Roots: anchor, + Intermediates: intermediates, + DNSName: host, + KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + }); err != nil { + return fmt.Errorf("the platform destination this connection was authenticated against no longer verifies: %w", err) + } + // And the same identity check a dial makes, since the allowed-peer set is + // read per dial and a connection older than a change to it would otherwise + // keep a de-authorised destination. + admitted, err := peers() + if err != nil { + return err + } + identity, err := oneURIIdentity(chain, "platform destination") + if err != nil { + return err + } + if !slices.Contains(admitted, identity) { + return fmt.Errorf("the established connection's destination %q is no longer one this runtime presents its credentials to", identity) + } + return nil +} + +// platformHandshakeTimeout bounds one outbound TLS handshake, on its own, +// because the custom dialler means nothing else does. It is deliberately not +// derived from platformRequestTimeout: a request may legitimately be long (a +// declared stream), and a handshake never is. +const platformHandshakeTimeout = 10 * time.Second + +// platformRequestTimeout bounds one request this runtime makes on its own +// behalf, end to end, so a host that accepts a request and never answers +// surfaces as a failure rather than a goroutine parked in Do forever. +const platformRequestTimeout = 10 * time.Second // --- Work Context --- // @@ -2036,6 +2888,44 @@ var gatewayTransport = newRegistrationTransport() // reached on the same base URL the solution's module calls already use. const workContextStartTaskProcedure = "/saas.accounts.v1.WorkContextService/StartTask" +// viewerInstallation is the installation a viewer's request acts under, or an +// error naming why it cannot be read. +// +// From the header the gateway stamps, and from nothing else. +// +// Round ten asked for this to be read from the capability carrier the SDK +// names, refusing rather than degrading when one cannot be read. The second +// half is right and is what this does. The first half cannot be: INBOUND, that +// carrier is caller-controlled. A browser can put a capability in +// x-codefly-work-context, and this runtime's standing property — with its own +// test, TestBrowserSuppliedWorkContextIsNeverForwarded — is that such a +// capability authenticates nothing and is never forwarded. Reading the +// installation out of it would give a caller a say in the installation their +// own mint is attributed to, which is the defect inverted rather than fixed. +// +// It is worse than useless: FromHeaders refuses a capability whose installation +// carriers are incomplete, so a caller sending a malformed one would decide +// whether the mint happens at all. That is how the change was caught — the +// forged-capability test stopped being able to mint and hung. +// +// So the stamped header is the source, on the same footing as x-org-id and +// x-session-id: the gateway replaces all three from verified claims, and a +// value a caller sent does not survive. Absent is a refusal, not a mint under +// an unnamed installation. +// +// Where a capability IS the authority being carried — outbound, where this +// runtime attaches one — the SDK's Attach puts the seal's own installation +// beside it and the far end cross-checks the two. That is the direction the +// SDK's rule is about. +func viewerInstallation(header http.Header) (string, error) { + id := strings.TrimSpace(header.Get(workcontext.InstallationIDHeaderName)) + if id == "" { + return "", fmt.Errorf("this request names no installation: the gateway stamps it as %s from the same verified claims it stamps %s and %s from, and none arrived", + workcontext.InstallationIDHeaderName, orgHeader, sessionHeader) + } + return id, nil +} + // orgHeader carries the viewer's organization. The gateway injects it after it // authenticates the bearer, replacing anything the caller sent. const orgHeader = "x-org-id" @@ -2097,18 +2987,49 @@ type workContextScope struct { ResourceIDs []string `json:"resourceIds,omitempty"` } +// workContextScopes is the wire form of an ask's scopes, DEEP-copied. +// +// A conversion for the struct, because the two types have the same shape by +// construction and spelling the fields out meant a field added to Scope would +// silently stop travelling — but a conversion copies the slice HEADERS and +// leaves both types pointing at the caller's backing arrays. The ask is +// retained on the delegation and its JSON is the cache key, computed once, so +// a caller could hand over Scope{Actions: []string{"read"}}, take the +// delegated gateway, write "delete" into that array, and wait for the cached +// capability to expire: the next mint serialized "delete" under the key +// computed for "read", and the ceiling had been checked against "read". No +// concurrency and no second call to ForModule. +// +// So the slices are copied too. What was checked and what is minted are then +// the same bytes for the life of the delegation, which is what makes the +// published ceiling a guarantee rather than a check on a value the caller can +// still edit. func workContextScopes(scopes []Scope) []workContextScope { wire := make([]workContextScope, len(scopes)) for i, scope := range scopes { wire[i] = workContextScope{ ResourceKind: scope.ResourceKind, - Actions: scope.Actions, - ResourceIDs: scope.ResourceIDs, + Actions: append([]string(nil), scope.Actions...), + ResourceIDs: append([]string(nil), scope.ResourceIDs...), } } return wire } +// scopesOfAsk is an ask's scopes back in this package's own type, for a check +// that has to run against what will actually be minted. (askedScopes is taken: +// contract.go uses it for a declaration's union of asks.) +func scopesOfAsk(wire []workContextScope) []Scope { + scopes := make([]Scope, len(wire)) + for i, scope := range wire { + // A conversion: this direction only reads the ask, and the ceiling + // check does not retain what it is handed, so sharing the arrays is + // safe here — the copy that matters is the one on the way IN. + scopes[i] = Scope(scope) + } + return scopes +} + // ForModule returns a Gateway that acts for the viewer against one composed // module. It mints a Task Work Context owned and actored by the viewer — the // bearer is forwarded, so accounts resolves the same subject the module would @@ -2144,6 +3065,13 @@ func workContextScopes(scopes []Scope) []workContextScope { // request. Minting here as well means an ask accounts refuses fails at this // call rather than inside some later round trip. func (g *Gateway) ForModule(ctx context.Context, audience string, scopes ...Scope) (*Gateway, error) { + // The contract first, before anything about the caller: asking for + // authority this solution never published is this solution's defect, true + // of every caller, and it should not be reported only to the ones who + // happen to arrive with a complete set of identity headers. + if err := g.withinPublishedCeiling(audience, scopes); err != nil { + return nil, err + } if g.orgID == "" { // The gateway injects orgHeader from the caller's active org, so it is // present but empty for a viewer with no organization selected and for @@ -2173,8 +3101,23 @@ func (g *Gateway) ForModule(ctx context.Context, audience string, scopes ...Scop "cannot mint a work context for %q: %s is absent or empty — the gateway injects it from the verified session and stamps it empty for any caller that authenticates without one: %w", audience, sessionHeader, &ClientError{StatusCode: http.StatusForbidden, Message: "a user session is required to read composed modules"}) } + if g.installationID == "" { + if g.installationErr != nil { + return nil, g.surfaced(fmt.Errorf("cannot mint a work context for %q: %w. The host requires an installation on every mint and refuses one naming none, and an organization is not an installation — one org may hold several", + audience, g.installationErr)) + } + // Not a status the caller can act on, so not a ClientError: the + // installation comes from the capability the gateway handed us, and a + // caller cannot seal one. A viewer seeing "pick an installation" would + // be told to fix something that is not theirs, so this surfaces as the + // runtime's generic failure with an operator-readable reason. + return nil, fmt.Errorf( + "cannot mint a work context for %q: this request names no installation. The host requires one on every mint and refuses a mint naming none, and an organization is not an installation — one org may hold several. It is read from the %s header the gateway stamps from verified claims, like the org and the session beside it, so either this request did not come through the gateway or the gateway stamped none", + audience, workcontext.InstallationIDHeaderName) + } ask := startTaskRequest{ OrgID: g.orgID, + InstallationID: g.installationID, SessionID: g.sessionID, Audience: audience, AuthorityScopes: workContextScopes(scopes), @@ -2188,11 +3131,107 @@ func (g *Gateway) ForModule(ctx context.Context, audience string, scopes ...Scop acting := *g acting.delegation = &delegation{key: string(key), ask: ask} if _, err := acting.workContext(ctx); err != nil { - return nil, err + return nil, g.surfaced(err) } return &acting, nil } +// surfaced is the error a caller is handed, sanitized where the caller cannot +// be trusted to sanitize it themselves. +// +// On a handler route this returns the error unchanged: the detail is what a log +// line and an operator need, and handlerErrorResponse sanitizes at the HTTP +// boundary, which every handler response passes through. +// +// An MCP tool has no such boundary. A tool returns an error and the MCP SDK +// turns it into a CallToolResult carrying that text, *below* the middleware +// this runtime installs — so by the time anything here sees the call again the +// text is content, indistinguishable from what the author meant to say. The +// runtime's own errors name the mint URL, the gateway URL and the issuer's +// message, because that is what a boot refusal needs; relayed to an agent +// client they are a disclosure, which executed round eight reproduced. +// +// So the sanitizing happens where the error leaves this runtime's hands, +// rather than where it would be too late. The detail goes to the log. +func (g *Gateway) surfaced(err error) error { + if err == nil || !g.sanitizeErrors { + return err + } + _, message := handlerErrorResponse(err) + log.Printf("solution %q: refusing a module read for a viewer: %v", g.id, err) + return errors.New(message) +} + +// withinPublishedCeiling holds a mint to the contract this process published. +// +// The contract claims, in the document an operator and a renderer read, "the +// most authority this solution may ever ask for". Until this existed that was +// true of the *declaration* and nothing else: checkContract governs the +// Consumes entries, while ForModule is public, takes any audience and any +// scopes, and went straight to the mint. A solution whose handlers only call +// ForModule consumed nothing, so mintsAuthority was false, so it needed no +// profile and published an empty contract — and minted arbitrary authority +// behind it. A ceiling that governs the declaration and not the asks is a +// ceiling that governs nothing a reviewer cares about. +// +// Enforcing it here also closes the gap the other way: a solution that mints +// through ForModule now has to declare the audience and its ceiling, because an +// audience the contract does not name is refused. That is the same answer the +// boot gives for an undeclared consumed module, at the moment the authority is +// actually asked for. +// +// A gateway with no resolved contract carries no ceilings and is not held to +// one. That is a gateway built outside a boot — a unit test, the passthrough +// seam — reachable only through this package's unexported constructors, never +// by a deployed runtime: start() resolves the contract before it serves, so +// every gateway a handler is handed carries it. +func (g *Gateway) withinPublishedCeiling(audience string, scopes []Scope) error { + if g.ceilings == nil { + return nil + } + // An ask for nothing is refused rather than waved through. + // + // Both shapes reached the mint: ForModule with no scopes at all, and a + // Scope naming a kind with no actions. The loop below simply had nothing to + // check in either case, so the ceiling was satisfied by construction — and + // what accounts does with an empty authorityScopes is its decision, not a + // decision this runtime gets to leave to it. A capability minted for "no + // stated authority" is either useless or the issuer's defaults, and + // neither is a thing a reviewer approved in this contract. + if len(scopes) == 0 { + return fmt.Errorf("this solution asked %q for a capability with no scopes at all: name the authority the call needs, since a mint for nothing is either useless or whatever the issuer decides to grant, and the published ceiling cannot govern either: %w", + audience, &ClientError{StatusCode: http.StatusForbidden, Message: "this solution asked for a capability with no stated authority"}) + } + for _, scope := range scopes { + if len(scope.Actions) == 0 { + return fmt.Errorf("this solution asked %q for resource kind %q with no actions: an action-less scope states no authority and is inside every ceiling by construction: %w", + audience, scope.ResourceKind, &ClientError{StatusCode: http.StatusForbidden, Message: "this solution asked for a scope with no actions"}) + } + } + ceiling, declared := g.ceilings[audience] + // A declared audience with an empty ceiling is a module this solution + // publishes as minting nothing — a ViewerBearer binding carries a nil + // ceiling and still sat in this map, so ForModule minted real authority for + // the one kind of module the contract says mints none. The contract and the + // mint disagreeing is the whole defect class this check exists for. + if declared && len(ceiling) == 0 { + return fmt.Errorf("this solution asked %q for authority, and its published contract declares that module with no ceiling at all — it is called with the viewer's bearer and mints nothing, so there is no authority to ask for: %w", + audience, &ClientError{StatusCode: http.StatusForbidden, Message: "that module is called with the viewer's bearer and holds no minted authority"}) + } + if !declared { + return fmt.Errorf("this solution asked %q for authority, and its published contract names no binding for that audience: declare it — the contract is what the host derives this solution's authority from, so an audience missing from it is authority nobody approved: %w", + audience, &ClientError{StatusCode: http.StatusForbidden, Message: "this solution holds no binding for that module"}) + } + for _, scope := range scopes { + if err := withinCeiling(scope, ceiling); err != nil { + return fmt.Errorf("this solution asked %q for %s outside the ceiling its contract publishes: %w: %w", + audience, scopeText([]Scope{scope}), err, + &ClientError{StatusCode: http.StatusForbidden, Message: "this solution asked for more authority than its contract allows"}) + } + } + return nil +} + // delegation is the ask a derived gateway acts under. It holds the ask and not // the capability so that every request resolves a live one: a capability // snapshotted when the gateway was derived lapses while the handler still holds @@ -2206,9 +3245,21 @@ type delegation struct { // workContext resolves the capability this gateway acts under, minting one if // the cache holds none that will outlive the call. -func (g *Gateway) workContext(ctx context.Context) (codefly.WorkContextToken, error) { - return g.contexts.resolve(ctx, g.delegation.key, func(ctx context.Context) (codefly.WorkContextToken, time.Time, error) { +func (g *Gateway) workContext(ctx context.Context) (string, error) { + return g.contexts.resolve(ctx, g.delegation.key, func(ctx context.Context) (string, time.Time, error) { ask := g.delegation.ask + // The ceiling, against the ask that is about to be MINTED. + // + // ForModule checks before it returns, and with the scopes deep-copied + // the two can no longer disagree — so this is not what closes that + // hole. It is here because the check belongs at the boundary the + // authority actually crosses: every mint passes through this function, + // a delegation reaching it by some future path would otherwise carry + // an unchecked ask, and the cost is a comparison against a declaration + // already in memory. + if err := g.withinPublishedCeiling(ask.Audience, scopesOfAsk(ask.AuthorityScopes)); err != nil { + return "", time.Time{}, err + } ask.TaskID = uuid.NewString() return g.mint(ctx, ask) }) @@ -2238,28 +3289,56 @@ func (e *WorkContextRefusal) Unwrap() error { return &GatewayError{StatusCode: e // a solution's host is not this package's dependency. Leaving actorPrincipalId // unset is what makes the viewer both owner and actor of the Task. type startTaskRequest struct { - OrgID string `json:"orgId"` + OrgID string `json:"orgId"` + // InstallationID is the installation this mint acts under, which the host + // requires and refuses a mint naming none (core's StartInput). An + // organization is not an installation: one org may hold several, and a + // capability minted without naming one is attributable to the org and to no + // deployment inside it. + // + // For a viewer-driven mint it is the viewer's own, read from the + // x-codefly-installation-id header the gateway stamps from verified claims + // — on the same footing as the org and the session beside it, and replaced + // by the gateway whatever a caller sent. NOT from the seal of a carried + // capability: inbound that carrier is caller-controlled, and a + // caller-supplied capability authenticates nothing here. See + // viewerInstallation, which carries the whole argument. + // + // For a background or delegated call it would be the installation the + // parent capability was sealed to, read off this execution's held + // credential; this runtime has no such path today, because every mint it + // makes is driven by a viewer request and ForModule refuses one without an + // org and a session. + InstallationID string `json:"installationId"` TaskID string `json:"taskId"` SessionID string `json:"sessionId"` Audience string `json:"audience"` AuthorityScopes []workContextScope `json:"authorityScopes"` } -func (g *Gateway) mint(ctx context.Context, ask startTaskRequest) (codefly.WorkContextToken, time.Time, error) { +func (g *Gateway) mint(ctx context.Context, ask startTaskRequest) (string, time.Time, error) { body, err := json.Marshal(ask) if err != nil { - return codefly.WorkContextToken{}, time.Time{}, err + return "", time.Time{}, err } ctx, cancel := context.WithTimeout(ctx, workContextMintTimeout) defer cancel() post, err := http.NewRequestWithContext(ctx, http.MethodPost, g.baseURL+workContextStartTaskProcedure, bytes.NewReader(body)) if err != nil { - return codefly.WorkContextToken{}, time.Time{}, err + return "", time.Time{}, err } post.Header.Set("content-type", "application/json") + // The viewer's bearer says on whose behalf; this execution's credential + // says which module is asking. The work-context header is free on this + // request — the capability being minted is what the answer carries, and the + // viewer has none yet — so attesting here collides with nothing the module + // call later presents. + if err := attestWorkloadReporting(ctx, g.workload, g.report, post, g.id, g.terminal); err != nil { + return "", time.Time{}, err + } resp, err := g.bearerClient().Do(post) if err != nil { - return codefly.WorkContextToken{}, time.Time{}, fmt.Errorf("work context mint for %q: %w", ask.Audience, err) + return "", time.Time{}, fmt.Errorf("work context mint for %q: %w", ask.Audience, err) } defer drainAndClose(resp) if resp.StatusCode != http.StatusOK { @@ -2272,7 +3351,7 @@ func (g *Gateway) mint(ctx context.Context, ask startTaskRequest) (codefly.WorkC Message string `json:"message"` } _ = json.NewDecoder(io.LimitReader(resp.Body, 4<<10)).Decode(&refusal) - return codefly.WorkContextToken{}, time.Time{}, &WorkContextRefusal{ + return "", time.Time{}, &WorkContextRefusal{ Audience: ask.Audience, StatusCode: resp.StatusCode, Code: refusal.Code, Message: refusal.Message, } } @@ -2282,11 +3361,20 @@ func (g *Gateway) mint(ctx context.Context, ask startTaskRequest) (codefly.WorkC WorkContextPrincipals } if err := json.NewDecoder(resp.Body).Decode(&issued); err != nil { - return codefly.WorkContextToken{}, time.Time{}, fmt.Errorf("work context mint for %q returned invalid json: %w", ask.Audience, err) - } - token, err := codefly.ParseWorkContextToken(issued.Token) - if err != nil { - return codefly.WorkContextToken{}, time.Time{}, fmt.Errorf("work context mint for %q returned an unusable token: %w", ask.Audience, err) + return "", time.Time{}, fmt.Errorf("work context mint for %q returned invalid json: %w", ask.Audience, err) + } + // The capability travels as a string and is not parsed here. There is one + // implementation of a Work Context — core's — and this runtime is not it: + // a token this process decoded to inspect would be a second reading of a + // format whose only authoritative reader is the verifier at the far end. + // What this runtime checks is what it is responsible for: that a usable + // expiry came back, so a capability is not cached under one it can never + // reuse. The carrier refuses an unsealed capability when it is attached + // (workcontext.Attach), which is the one check that has to happen before a + // call rather than at the far end. + token := issued.Token + if strings.TrimSpace(token) == "" { + return "", time.Time{}, fmt.Errorf("work context mint for %q returned no capability", ask.Audience) } // An expiry this process cannot use is a boundary error, not a capability to // cache. Treated as "already lapsed" it would look like success while @@ -2297,11 +3385,11 @@ func (g *Gateway) mint(ctx context.Context, ask startTaskRequest) (codefly.WorkC lifetime := issued.ExpiresAt.Sub(now) switch { case issued.ExpiresAt.IsZero(): - return codefly.WorkContextToken{}, time.Time{}, fmt.Errorf( + return "", time.Time{}, fmt.Errorf( "work context mint for %q returned no expiry: the issuer omitted expiresAt, or spelled it differently (protobuf JSON spells it expires_at, which does not decode into this field)", ask.Audience) case lifetime < workContextMinimumLifetime: - return codefly.WorkContextToken{}, time.Time{}, fmt.Errorf( + return "", time.Time{}, fmt.Errorf( "work context mint for %q returned the expiry %s, already past or less than %s away (check this host's clock against the issuer's)", ask.Audience, issued.ExpiresAt.UTC().Format(time.RFC3339), workContextMinimumLifetime) } @@ -2345,7 +3433,16 @@ func (g *Gateway) WorkContextPrincipals(ctx context.Context) (WorkContextPrincip } token, err := g.workContext(ctx) if err != nil { - return WorkContextPrincipals{}, err + // Through surfaced, like ForModule's. This is ForModule's exported + // sibling and it reaches the same mint, so it produced the same + // errors — the mint URL, the status, the issuer's own message — and + // handed them back unsanitized. A tool calling it relayed the whole + // chain to an agent client, which is the disclosure ForModule's was + // fixed for; two callers of one mint, one of them sanitized. + // + // Found by driving every exported Gateway method that returns an + // error rather than by reading the one that had already been fixed. + return WorkContextPrincipals{}, g.surfaced(err) } principals, ok := g.contexts.principals(token) if !ok || principals.OwnerPrincipalID == "" || principals.CurrentActorPrincipalID == "" || principals.OrgID == "" { @@ -2359,7 +3456,7 @@ func (g *Gateway) WorkContextPrincipals(ctx context.Context) (WorkContextPrincip // shares it, so the cache lives exactly as long as the viewer's request. type workContextCache struct { mu sync.Mutex - minted map[string]issuedWorkContext + minted map[string]cachedCapability // minting holds the mint in flight for an ask, so concurrent asks for the // same one wait on it instead of each running their own. A handler that // fans out would otherwise spend one audited mint per goroutine for a @@ -2369,29 +3466,34 @@ type workContextCache struct { issuedFor map[string]WorkContextPrincipals } -func (c *workContextCache) remember(token codefly.WorkContextToken, principals WorkContextPrincipals) { +func (c *workContextCache) remember(token string, principals WorkContextPrincipals) { c.mu.Lock() defer c.mu.Unlock() - c.issuedFor[token.Encoded()] = principals + c.issuedFor[token] = principals } -func (c *workContextCache) principals(token codefly.WorkContextToken) (WorkContextPrincipals, bool) { +func (c *workContextCache) principals(token string) (WorkContextPrincipals, bool) { c.mu.Lock() defer c.mu.Unlock() - principals, ok := c.issuedFor[token.Encoded()] + principals, ok := c.issuedFor[token] return principals, ok } func newWorkContextCache() *workContextCache { return &workContextCache{ - minted: map[string]issuedWorkContext{}, + minted: map[string]cachedCapability{}, minting: map[string]*pendingMint{}, issuedFor: map[string]WorkContextPrincipals{}, } } -type issuedWorkContext struct { - token codefly.WorkContextToken +// cachedCapability is one capability this gateway holds for the life of a +// request: the carrier as it travels, and when it stops being worth +// presenting. Not a Work Context — this runtime declares none of a +// capability's fields and reads none of them (see +// work_context_boundary_test.go); it holds the string and a deadline. +type cachedCapability struct { + token string // reuseUntil is when this capability stops being worth presenting — its // expiry less the renewal lead mint already applied. reuseUntil time.Time @@ -2401,7 +3503,7 @@ type issuedWorkContext struct { // is closed and read only after, so the close/receive pair orders them. type pendingMint struct { done chan struct{} - token codefly.WorkContextToken + token string err error } @@ -2411,8 +3513,8 @@ type pendingMint struct { func (c *workContextCache) resolve( ctx context.Context, key string, - mint func(context.Context) (codefly.WorkContextToken, time.Time, error), -) (codefly.WorkContextToken, error) { + mint func(context.Context) (string, time.Time, error), +) (string, error) { c.mu.Lock() if issued, ok := c.minted[key]; ok && time.Now().Before(issued.reuseUntil) { c.mu.Unlock() @@ -2426,7 +3528,7 @@ func (c *workContextCache) resolve( // failure here would turn one refusal into one mint per waiter. return inflight.token, inflight.err case <-ctx.Done(): - return codefly.WorkContextToken{}, ctx.Err() + return "", ctx.Err() } } inflight := &pendingMint{done: make(chan struct{})} @@ -2439,7 +3541,7 @@ func (c *workContextCache) resolve( c.mu.Lock() delete(c.minting, key) if err == nil { - c.minted[key] = issuedWorkContext{token: token, reuseUntil: reuseUntil} + c.minted[key] = cachedCapability{token: token, reuseUntil: reuseUntil} } c.mu.Unlock() close(inflight.done) @@ -2449,7 +3551,7 @@ func (c *workContextCache) resolve( // Unary makes a typed Connect call to a fully-qualified procedure through the // gateway. Req and Resp are generated protobuf messages; the gateway URL, the // bearer, and the wire protocol are hidden so a handler only names a procedure -// and passes typed messages — the twin of the Python runtime's gateway.unary. +// and passes typed messages. func Unary[Req, Resp any](ctx context.Context, gw *Gateway, procedure string, req *Req) (*Resp, error) { client := connect.NewClient[Req, Resp](gw.HTTPClient(), gw.baseURL+procedure) resp, err := client.CallUnary(ctx, connect.NewRequest(req)) @@ -2471,16 +3573,59 @@ type bearerTransport struct { func (t bearerTransport) RoundTrip(r *http.Request) (*http.Response, error) { r.Header.Set("authorization", t.bearer) - if t.acting != nil { - token, err := t.acting.workContext(r.Context()) - if err != nil { - return nil, err - } - if err := codefly.AttachWorkContext(r, token); err != nil { - return nil, err - } + if t.acting == nil { + return t.base.RoundTrip(r) } - return t.base.RoundTrip(r) + token, err := t.acting.workContext(r.Context()) + if err != nil { + // Through surfaced, because this is the mint's own error arriving by a + // route I had argued was safe. + // + // I reasoned that a transport error only names an address the author + // already holds from BaseURL(), and that is true of a dial failure. It + // is not true of this: a capability that lapses mid-call re-mints HERE, + // inside the round trip, so a refusal carries the issuer's status, its + // code and its words — and http.Client.Do then wraps the whole thing + // and hands it to the author. + // + // The MCP path made that a disclosure with no boundary left to catch + // it: the SDK's typed AddTool wrapper turns a tool's returned error + // into CallToolResult{IsError: true} and returns a NIL error, so the + // middleware's error branch never runs and the text travels as + // content. Sanitizing where the error leaves this runtime is the only + // place that works, which is the same conclusion ForModule reached. + return nil, t.acting.surfaced(err) + } + if err := workcontext.Attach(r, token); err != nil { + return nil, t.acting.surfaced(err) + } + resp, err := t.base.RoundTrip(r) + // NOTHING IS INFERRED FROM A 409, and that is the whole of this runtime's + // position on it. + // + // A capability was dropped from the cache when a module answered 409 with + // an installation header. That was wrong in three successive ways: any 409 + // with any installation header; then any 409 whose header named the + // installation the capability is sealed to — and a module answering an + // ordinary business conflict IS in that installation, so a duplicate or a + // lost update still evicted a perfectly valid capability and the next call + // minted again. Needless audited mints, on the one path whose purpose is + // one mint per execution. + // + // The two cases are indistinguishable on the wire: a 409 means "the state + // you were sealed to has moved" and it means "that page already exists", + // and nothing in the response says which. Matching installation metadata + // identifies the SCOPE of a conflict; it does not establish supersession. + // So this runtime infers nothing and holds the capability until its own + // expiry. A genuinely superseded one is refused call by call by the far + // end — correct if noisy — and the renewal that replaces it happens on the + // credential's own schedule. + // + // What would make an inference sound is a machine-readable discriminator + // from the host or the verifier: follow-up 11, and nobody's to invent here. + // AGENTS.md and this PR's body both already said there was no inference; + // the code is what had not caught up. + return resp, err } func writeJSON(w http.ResponseWriter, status int, body any) { @@ -2513,9 +3658,19 @@ func setCORS(w http.ResponseWriter) { w.Header().Set("access-control-allow-methods", "GET, POST, OPTIONS") } -// siblingURL swaps the trailing `replacing` of base for path. It is how a -// derived federation endpoint follows an explicitly overridden one: both -// endpoints of the registration exchange must address the same gateway. +// Restored for the MCP surface, which arrived on main after this cutover +// branched and is the only caller of each now. None of the three is a +// registration: siblingURL swaps a path suffix, deployedRuntimeContext reads +// one explicit signal, and resolveFrontend resolves the host's own origin by +// role through the SDK rather than typing an address. They were deleted as +// collateral when the registrations that called them went. + +// siblingURL swaps the trailing `replacing` of base for path. +// +// It paired the two endpoints of a registration exchange, which had to address +// the same gateway. Those are deleted; its one caller now is the MCP metadata +// document's URL, derived from the resource identifier by swapping that +// suffix. // // It replaces a suffix rather than rebuilding from scheme+host, because a // gateway is not always mounted at the root. Rebuilding dropped everything @@ -2542,3 +3697,47 @@ func siblingURL(base, replacing, path string) string { } return prefix + path } + +// deployedRuntimeContext reports whether Codefly says this process runs in a +// deployment rather than on a developer machine. The signal is explicit — +// CODEFLY__RUNTIME_CONTEXT, which core's GitOps render injects (e.g. +// "kubernetes") — and never the environment name: an environment called +// "local-dogfood" or "staging" says nothing about where the process runs. +// Every runtime context `codefly run` uses on a developer machine is local; +// any other declared context is a deployment, so a new deployed kind is +// covered without a change here. Nothing declared means not deployed. +// +// PENDING codefly-dev/core v0.5.6, which injects the signal into renders: +// until a cell is rendered with it, this reports false there too. +func deployedRuntimeContext(kind string) bool { + switch strings.ToLower(kind) { + case "", resources.RuntimeContextNative, resources.RuntimeContextNix, + resources.RuntimeContextContainer, resources.RuntimeContextFree: + return false + } + return true +} + +// resolveFrontend resolves the host frontend's http endpoint by role, like +// resolveGateway — host-module-name-agnostic. +func resolveFrontend(ctx context.Context, module, frontend string) string { + return hostAddress(ctx, module, frontend, "http", "http") +} + +// dialledAddresses is the non-empty addresses this process talks to, for the +// one question asked of them: does a string a tool is about to hand a client +// contain one? +// The parameter and loop variable are deliberately NOT called `address`: the +// package has a function of that name which reads the environment, and the +// environment boundary keys its resolvers by bare name — so a local called +// `address` reads to that gate as a reference to the resolver. Shadowing a +// resolver's name is confusing on its own account; the gate just says so. +func dialledAddresses(dialling ...string) []string { + dialled := []string{} + for _, one := range dialling { + if strings.TrimSpace(one) != "" { + dialled = append(dialled, one) + } + } + return dialled +} diff --git a/solution_registration_test.go b/solution_registration_test.go deleted file mode 100644 index 87bfa2a..0000000 --- a/solution_registration_test.go +++ /dev/null @@ -1,999 +0,0 @@ -package solution - -import ( - "context" - "encoding/json" - "fmt" - "io" - "log" - "net" - "net/http" - "net/http/httptest" - "os" - "strconv" - "strings" - "sync" - "sync/atomic" - "testing" - "time" - - codefly "github.com/codefly-dev/sdk-go" -) - -// solutionExchange is what /solutions/_registration-token received: the id the -// solution asks a credential for, plus the two headers the gateway requires — -// its own perimeter token and the solution's registration secret. -type solutionExchange struct { - ID string `json:"id"` - Secret string `json:"-"` - Internal string `json:"-"` -} - -// solutionRegistration is one self-registration as a surface saw it: which -// surface, the credential presented, and whether the shared cluster-internal -// token rode along. -type solutionRegistration struct { - Surface string - Token string - Internal string - Status int -} - -// fakeHost stands in for the host's two solution-registration surfaces and the -// gateway exchange that credentials them, with the real refusal semantics: the -// exchange mints only against the declared secret, and each surface burns a -// token's single use — a replay is refused. -type fakeHost struct { - *httptest.Server - secret string - exchanges chan solutionExchange - registrations chan solutionRegistration - // registerStatus, when non-zero, is what both surfaces answer instead of - // 200, with registerBody as the body. - registerStatus int - registerBody string - // noExchange removes the exchange route, so it answers a plain 404. - // admitsInternal makes both surfaces accept the cluster-internal token. - // - // These are separate knobs, not one "legacy" flag: welding them together - // made the host that matters most unreachable by construction — one that - // 404s the exchange *and* refuses the internal token, i.e. a current host - // reached at a wrong exchange URL. - noExchange bool - admitsInternal bool - // exchangeGoneAfter, when non-zero, makes the exchange start answering 404 - // once it has minted that many tokens: a route that disappears under a host - // that has already proved it serves one. - exchangeGoneAfter int - // tokenLifetime is how long a minted token is valid for; zero means 5m. - tokenLifetime time.Duration - // registerDelay is how long both registration surfaces take to answer. - registerDelay time.Duration - - mu sync.Mutex - minted int - burned map[string]bool -} - -func newFakeHost(t *testing.T, secret string) *fakeHost { - t.Helper() - h := &fakeHost{ - secret: secret, - exchanges: make(chan solutionExchange, 16), - registrations: make(chan solutionRegistration, 16), - burned: map[string]bool{}, - } - h.Server = httptest.NewServer(http.HandlerFunc(h.serve)) - t.Cleanup(h.Close) - return h -} - -func (h *fakeHost) serve(w http.ResponseWriter, r *http.Request) { - switch r.URL.Path { - case solutionRegistrationTokenPath: - h.mu.Lock() - unrouted := h.noExchange || (h.exchangeGoneAfter > 0 && h.minted >= h.exchangeGoneAfter) - h.mu.Unlock() - if unrouted { - send(h.exchanges, solutionExchange{ID: "(unrouted)"}) - w.WriteHeader(http.StatusNotFound) - return - } - var exchange solutionExchange - _ = json.NewDecoder(r.Body).Decode(&exchange) - exchange.Secret = r.Header.Get(solutionSecretHeader) - exchange.Internal = r.Header.Get(internalTokenHeader) - send(h.exchanges, exchange) - if exchange.Internal != internalTokenTest || h.secret == "" || exchange.Secret != h.secret { - w.WriteHeader(http.StatusUnauthorized) - return - } - h.mu.Lock() - h.minted++ - token := fmt.Sprintf("solution-token-%d", h.minted) - h.mu.Unlock() - lifetime := h.tokenLifetime - if lifetime == 0 { - lifetime = 5 * time.Minute - } - writeJSON(w, http.StatusOK, map[string]string{ - "token": token, - // Preserve the requested lifetime: truncating to seconds can put - // a 1.5s token below the 1s admission floor before the beat starts. - "expiresAt": time.Now().Add(lifetime).UTC().Format(time.RFC3339Nano), - }) - case solutionRegisterPath, "/api/solutions/register": - if h.registerDelay > 0 { - time.Sleep(h.registerDelay) - } - surface := "gateway" - if r.URL.Path != solutionRegisterPath { - surface = "host" - } - token := r.Header.Get(solutionRegistrationHeader) - status := http.StatusOK - h.mu.Lock() - switch { - case token != "" && !h.burned[surface+":"+token]: - // A token is presented once per surface; a replay is refused. - h.burned[surface+":"+token] = true - case h.admitsInternal && r.Header.Get(internalTokenHeader) == internalTokenTest: - // Before the publisher-bound contract the shared token was the - // credential. - default: - // The cluster-internal token is not a registration credential on a - // host that binds registration to the publisher. - status = http.StatusUnauthorized - } - h.mu.Unlock() - if status == http.StatusOK && h.registerStatus != 0 { - status = h.registerStatus - } - send(h.registrations, solutionRegistration{ - Surface: surface, - Token: token, - Internal: r.Header.Get(internalTokenHeader), - Status: status, - }) - w.WriteHeader(status) - if status != http.StatusOK && h.registerBody != "" { - _, _ = w.Write([]byte(h.registerBody)) - } - default: - w.WriteHeader(http.StatusOK) - } -} - -// bootAgainst boots a real solution against h with the given secret, beating -// fast enough for a test to observe several registrations. -func bootAgainst(t *testing.T, h *fakeHost, secret string) func() { - t.Helper() - ln, err := net.Listen("tcp", "127.0.0.1:0") - if err != nil { - t.Fatal(err) - } - s := New(Manifest{ID: "lastlogin-go", Title: "Last Login"}) - s.registrationInterval = 20 * time.Millisecond - s.cfg = config{ - port: strconv.Itoa(ln.Addr().(*net.TCPAddr).Port), - publicURL: "http://127.0.0.1", - hostRegisterURL: h.URL + "/api/solutions/register", - gatewayRegisterURL: h.URL + solutionRegisterPath, - moduleRegisterURL: h.URL + moduleRegisterPath, - moduleTokenURL: h.URL + moduleRegistrationTokenPath, - solutionTokenURL: h.URL + solutionRegistrationTokenPath, - internalToken: internalTokenTest, - solutionSecret: secret, - } - ctx, cancel := context.WithCancel(context.Background()) - done := make(chan struct{}) - go func() { _ = s.serve(ctx, ln); close(done) }() - return func() { - cancel() - <-done - } -} - -func awaitRegistrations(t *testing.T, h *fakeHost, n int) []solutionRegistration { - t.Helper() - var seen []solutionRegistration - deadline := time.After(5 * time.Second) - for len(seen) < n { - select { - case r := <-h.registrations: - seen = append(seen, r) - case <-deadline: - t.Fatalf("saw %d registrations within timeout, want %d", len(seen), n) - } - } - return seen -} - -// TestServeRegistersWithSolutionCredential proves the publisher-bound handshake -// end to end on the runtime's side: with a registration secret provisioned, the -// runtime exchanges it (with the cluster-internal token as the gateway's -// perimeter check) for a signed, solution-bound token, presents that token — and -// not the cluster-internal one — on both the host and the gateway registration, -// and mints a fresh one per beat, because each surface burns a token on use. -func TestServeRegistersWithSolutionCredential(t *testing.T) { - h := newFakeHost(t, "s3cret") - stop := bootAgainst(t, h, "s3cret") - defer stop() - - select { - case exchange := <-h.exchanges: - if exchange.ID != "lastlogin-go" { - t.Errorf("exchange asked for id %q, want the solution's own id", exchange.ID) - } - if exchange.Secret != "s3cret" { - t.Errorf("exchange presented %s = %q, want the provisioned secret", solutionSecretHeader, exchange.Secret) - } - if exchange.Internal != internalTokenTest { - t.Errorf("exchange presented %s = %q, want the cluster-internal token for the gateway's perimeter check", internalTokenHeader, exchange.Internal) - } - case <-time.After(5 * time.Second): - t.Fatal("no registration token exchange within timeout") - } - - seen := awaitRegistrations(t, h, 6) - surfaces := map[string]int{} - tokens := map[string]int{} - for _, r := range seen { - surfaces[r.Surface]++ - tokens[r.Token]++ - if r.Status != http.StatusOK { - t.Errorf("%s registration presenting %q was refused (status %d): a fresh token must be accepted", r.Surface, r.Token, r.Status) - } - if !strings.HasPrefix(r.Token, "solution-token-") { - t.Errorf("%s registration presented %s = %q, want a token minted by the exchange", r.Surface, solutionRegistrationHeader, r.Token) - } - if r.Internal != "" { - t.Errorf("%s registration also presented %s = %q; the credential proves the publisher, the shared token proves nothing and must not ride along", r.Surface, internalTokenHeader, r.Internal) - } - } - if surfaces["host"] == 0 || surfaces["gateway"] == 0 { - t.Fatalf("registrations by surface = %v, want both host and gateway", surfaces) - } - for token, n := range tokens { - if n != 1 { - t.Errorf("token %q was presented %d times; a registration credential is single-use and must be minted per beat", token, n) - } - } -} - -// TestShortLivedTokenIsUsedNotRefused pins the floor at what it actually bounds. -// This credential is presented on the very next round trip, so a token the -// issuer chose to make short-lived is usable; the 5s floor copied from the -// module credential — which caches its token across beats — refused every one of -// them, failing every beat forever with a message blaming this host's clock. -func TestShortLivedTokenIsUsedNotRefused(t *testing.T) { - buf := &syncBuffer{} - log.SetOutput(buf) - defer log.SetOutput(os.Stderr) - - h := newFakeHost(t, "s3cret") - h.tokenLifetime = 2 * time.Second // under moduleTokenMinimumLifetime - stop := bootAgainst(t, h, "s3cret") - defer stop() - - for _, r := range awaitRegistrations(t, h, 2) { - if r.Status != http.StatusOK || r.Token == "" { - t.Fatalf("%s registration with a %s token: status %d, token %q — a token that outlives the round trip must be presented, not refused", - r.Surface, h.tokenLifetime, r.Status, r.Token) - } - } - if out := buf.String(); strings.Contains(out, "check this host's clock") { - t.Fatalf("a usable short-lived token was rejected as a clock problem:\n%s", out) - } -} - -// TestHeartbeatLogsAChangedRefusalReason proves the reason is part of what was -// last reported. Keyed on the status alone, a host that keeps answering 409 -// while changing why said it once and then went silent on every later reason — -// dropping exactly the information detail was plumbed through beat to carry. -func TestHeartbeatLogsAChangedRefusalReason(t *testing.T) { - buf := &syncBuffer{} - log.SetOutput(buf) - defer log.SetOutput(os.Stderr) - - var beats atomic.Int32 - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - reason := "frontend.hostContract 1 is below the host minimum 2" - if beats.Add(1) > 2 { - reason = "schemaVersion 1 is no longer accepted" - } - writeJSON(w, http.StatusConflict, map[string]any{"error": "incompatible_runtime", "reasons": []string{reason}}) - })) - defer srv.Close() - - ctx, cancel := context.WithCancel(context.Background()) - s := &Server{manifest: Manifest{ID: "lastlogin-go"}, registrationInterval: 10 * time.Millisecond} - done := make(chan struct{}) - go func() { s.heartbeat(ctx, srv.URL, []byte("{}"), "host", internalTokenAuth("t")); close(done) }() - deadline := time.Now().Add(10 * time.Second) - for !strings.Contains(buf.String(), "schemaVersion 1 is no longer accepted") { - if time.Now().After(deadline) { - cancel() - <-done - t.Fatalf("the host's changed reason never reached the log:\n%s", buf.String()) - } - time.Sleep(10 * time.Millisecond) - } - cancel() - <-done - if n := strings.Count(buf.String(), "frontend.hostContract 1 is below"); n != 1 { - t.Errorf("the unchanged reason was logged %d times, want once: a repeated reason must still be deduped", n) - } -} - -// TestRefusalDetailCannotForgeALogLine proves a reason the host chose reaches -// the log as text and never as framing: a newline in it would otherwise write a -// log line of the runtime's own shape. -func TestRefusalDetailCannotForgeALogLine(t *testing.T) { - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - writeJSON(w, http.StatusConflict, map[string]any{ - "error": "incompatible_runtime", - "reasons": []string{"nope\n2026/01/01 00:00:00 registered with host as \"impostor\""}, - }) - })) - defer srv.Close() - - resp, err := http.Post(srv.URL, "application/json", nil) - if err != nil { - t.Fatal(err) - } - defer drainAndClose(resp) - detail := refusalDetail(resp) - if strings.ContainsAny(detail, "\r\n") { - t.Fatalf("refusalDetail kept a line break, so a host reason can forge a log line: %q", detail) - } - if !strings.Contains(detail, "registered with host as") { - t.Fatalf("the host's text must still reach the log, only flattened: %q", detail) - } -} - -// TestJitteredOnlyEverWaitsLonger proves the spread cannot shorten a wait: the -// backoff a failing beat earned is the floor, and cutting into it would undo the -// request-rate bound it exists to impose. -func TestJitteredOnlyEverWaitsLonger(t *testing.T) { - const base = 15 * time.Second - distinct := map[time.Duration]bool{} - for range 200 { - got := jittered(base) - if got < base || got > base+time.Duration(float64(base)*registrationJitter) { - t.Fatalf("jittered(%s) = %s, want within [%s, +%.0f%%]", base, got, base, registrationJitter*100) - } - distinct[got] = true - } - if len(distinct) < 2 { - t.Fatal("jittered returned one value 200 times; heartbeats would stay in lockstep") - } - if got := jittered(0); got != 0 { - t.Errorf("jittered(0) = %s, want 0", got) - } -} - -// TestSolutionCredentialRefusedExchangeNamesTheCause proves a refused exchange -// surfaces as a failed beat whose error names the two indistinguishable causes -// the host relays identically: an undeclared id and a mismatched secret. -func TestSolutionCredentialRefusedExchangeNamesTheCause(t *testing.T) { - buf := &syncBuffer{} - log.SetOutput(buf) - defer log.SetOutput(os.Stderr) - - h := newFakeHost(t, "s3cret") - stop := bootAgainst(t, h, "wrong") - defer stop() - - select { - case <-h.exchanges: - case <-time.After(5 * time.Second): - t.Fatal("no exchange within timeout") - } - deadline := time.Now().Add(5 * time.Second) - for { - if out := buf.String(); strings.Contains(out, "registration with host failed") && - strings.Contains(out, "SOLUTION_REGISTRATION_SECRETS") { - break - } - if time.Now().After(deadline) { - t.Fatalf("refused exchange was not reported:\n%s", buf.String()) - } - time.Sleep(10 * time.Millisecond) - } - select { - case r := <-h.registrations: - t.Fatalf("a registration reached the host (%+v) although the exchange refused the secret", r) - default: - } -} - -// TestSolutionCredentialRefusalIsReportedOnce proves a refusal of a token minted -// for the same beat is diagnosed as what it is — not staleness, since nothing -// is cached — and diagnosed once per episode rather than on every beat. -func TestSolutionCredentialRefusalIsReportedOnce(t *testing.T) { - buf := &syncBuffer{} - log.SetOutput(buf) - defer log.SetOutput(os.Stderr) - - credential := &solutionCredential{id: "lastlogin-go"} - credential.invalidate() - credential.invalidate() - if n := strings.Count(buf.String(), "was refused while presenting a token minted for that same beat"); n != 1 { - t.Fatalf("fresh-token refusal diagnosed %d times, want once per episode:\n%s", n, buf.String()) - } - credential.succeeded() - credential.invalidate() - if n := strings.Count(buf.String(), "was refused while presenting a token minted for that same beat"); n != 2 { - t.Fatalf("a refusal after a success is a new episode and earns its own line; got %d lines", n) - } -} - -// TestHeartbeatLogsTheHostsRefusalReasons proves the reasons a host attaches to -// a refusal reach the log: an incompatible_runtime refusal is only actionable -// with them. -func TestHeartbeatLogsTheHostsRefusalReasons(t *testing.T) { - buf := &syncBuffer{} - log.SetOutput(buf) - defer log.SetOutput(os.Stderr) - - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - writeJSON(w, http.StatusConflict, map[string]any{ - "error": "incompatible_runtime", - "reasons": []string{"frontend.hostContract 2 does not satisfy host contract 1", "reactRange ^18 does not match React 19.1.0"}, - }) - })) - defer srv.Close() - - ctx, cancel := context.WithCancel(context.Background()) - s := &Server{manifest: Manifest{ID: "lastlogin-go"}, registrationInterval: 20 * time.Millisecond} - done := make(chan struct{}) - go func() { - s.heartbeat(ctx, srv.URL, []byte("{}"), "host", internalTokenAuth("")) - close(done) - }() - deadline := time.Now().Add(5 * time.Second) - for !strings.Contains(buf.String(), "rejected (status 409)") { - if time.Now().After(deadline) { - cancel() - t.Fatalf("refusal not logged:\n%s", buf.String()) - } - time.Sleep(10 * time.Millisecond) - } - cancel() - <-done - want := `rejected (status 409) for "lastlogin-go": incompatible_runtime: frontend.hostContract 2 does not satisfy host contract 1; reactRange ^18 does not match React 19.1.0` - if !strings.Contains(buf.String(), want) { - t.Fatalf("log lacks the host's reasons; want %q in:\n%s", want, buf.String()) - } -} - -// TestLoadConfigDerivesSolutionTokenURL proves the exchange follows the -// registration endpoint — the same gateway, even one mounted under a path — -// and that an override of the registration alone cannot split the pair. -func TestLoadConfigDerivesSolutionTokenURL(t *testing.T) { - t.Setenv("PORT", "1234") - t.Setenv("GATEWAY_URL", "http://gateway:8080/gw") - t.Setenv("HOST_REGISTER_URL", "http://frontend:3000/api/solutions/register") - t.Setenv("GATEWAY_REGISTER_URL", "http://gateway:8080/gw/solutions/_register") - cfg := loadConfig(context.Background(), "lastlogin-go") - if want := "http://gateway:8080/gw/solutions/_registration-token"; cfg.solutionTokenURL != want { - t.Fatalf("solutionTokenURL = %q, want %q", cfg.solutionTokenURL, want) - } - - t.Setenv("GATEWAY_SOLUTION_REGISTRATION_TOKEN_URL", "http://other:9090/solutions/_registration-token") - cfg = loadConfig(context.Background(), "lastlogin-go") - if want := "http://other:9090/solutions/_registration-token"; cfg.solutionTokenURL != want { - t.Fatalf("explicit solutionTokenURL = %q, want %q", cfg.solutionTokenURL, want) - } -} - -// TestSolutionRegistrationSecretResolvesThroughTheSDK proves the declared -// provisioning path: the secret arrives as the namespaced workspace secret -// Codefly injects for the `solution-registration` group, resolved by name -// through the SDK, and an explicit environment override wins over it. -func TestSolutionRegistrationSecretResolvesThroughTheSDK(t *testing.T) { - // The SDK snapshots injected values process-wide; registered before Setenv - // so it runs after the environment is restored and leaves no secret behind - // for the boot tests that follow. - t.Cleanup(func() { _ = codefly.LoadEnvironmentVariables() }) - t.Setenv("CODEFLY__WORKSPACE_SECRET_CONFIGURATION__SOLUTION_REGISTRATION__SECRET", "from-workspace") - if err := codefly.LoadEnvironmentVariables(); err != nil { - t.Fatal(err) - } - if got := solutionRegistrationSecret(context.Background()); got != "from-workspace" { - t.Fatalf("solutionRegistrationSecret() = %q, want the workspace secret", got) - } - t.Setenv(SolutionRegistrationSecretEnvironmentVariable, "from-override") - if got := solutionRegistrationSecret(context.Background()); got != "from-override" { - t.Fatalf("solutionRegistrationSecret() = %q, want the explicit override", got) - } -} - -// TestValidateRequiresASolutionSecret proves a boot without a registration -// secret is refused, naming the provisioning: the host admits a registration -// only against a credential minted from that secret, and there is no other -// credential to present, so coming up would serve nothing while looking -// healthy. -func TestValidateRequiresASolutionSecret(t *testing.T) { - cfg := config{ - port: "1234", - gatewayURL: "http://gateway:42152", - hostRegisterURL: "http://frontend:21931/api/solutions/register", - gatewayRegisterURL: "http://gateway:42152/solutions/_register", - moduleRegisterURL: "http://gateway:42152/modules/_register", - moduleTokenURL: "http://gateway:42152/modules/_registration-token", - solutionTokenURL: "http://gateway:42152/solutions/_registration-token", - selfUpstream: "http://backend:8080", - } - err := cfg.validate() - if err == nil || !strings.Contains(err.Error(), SolutionRegistrationSecretEnvironmentVariable) || - !strings.Contains(err.Error(), SolutionRegistrationSecretGroup+"/"+SolutionRegistrationSecretKey) { - t.Fatalf("validate() without a secret = %v, want an error naming the provisioning", err) - } - cfg.solutionSecret = "s3cret" - if err := cfg.validate(); err != nil { - t.Fatalf("validate() with a secret = %v, want nil", err) - } - cfg.solutionTokenURL = "" - if err := cfg.validate(); err == nil || !strings.Contains(err.Error(), "solution token URL") { - t.Fatalf("validate() without an exchange URL = %v, want an error naming the solution token URL", err) - } -} - -// TestServeFailsTheBeatWhenTheHostOffersNoExchange proves there is no -// downgrade: against a host that answers 404 on the exchange, the beat fails -// and names the route, and nothing — not the cluster-internal token — is -// presented to the registration surfaces. -func TestServeFailsTheBeatWhenTheHostOffersNoExchange(t *testing.T) { - buf := &syncBuffer{} - log.SetOutput(buf) - defer log.SetOutput(os.Stderr) - - h := newFakeHost(t, "s3cret") - h.noExchange, h.admitsInternal = true, true - stop := bootAgainst(t, h, "s3cret") - defer stop() - - select { - case <-h.exchanges: - case <-time.After(5 * time.Second): - t.Fatal("no exchange attempted within timeout") - } - deadline := time.Now().Add(5 * time.Second) - for { - if out := buf.String(); strings.Contains(out, "registration with host failed") && strings.Contains(out, "answered 404") { - break - } - if time.Now().After(deadline) { - t.Fatalf("the 404 was not reported as a failed beat:\n%s", buf.String()) - } - time.Sleep(10 * time.Millisecond) - } - select { - case r := <-h.registrations: - t.Fatalf("a registration reached the host (%+v) although no credential could be minted; the cluster-internal token must never stand in", r) - default: - } -} - -// TestManifestDeclaresItsContractMajors proves the manifest says which host -// contract it is built against instead of relying on the host's default for a -// silent manifest. -func TestManifestDeclaresItsContractMajors(t *testing.T) { - s := New(Manifest{ID: "lastlogin-go", Title: "Last Login"}) - m := s.manifestMap() - if got := m["schemaVersion"]; got != solutionManifestSchemaMajor { - t.Fatalf("schemaVersion = %v, want %d", got, solutionManifestSchemaMajor) - } - frontend := m["frontend"].(map[string]any) - if got := frontend["hostContract"]; got != solutionHostContractMajor { - t.Fatalf("frontend.hostContract = %v, want %d", got, solutionHostContractMajor) - } -} - -// answeringTheExchange is the least a fake gateway must do for a solution to -// register at all: mint a token on the exchange, and accept everything else. -// Tests that are about something other than the credential use it. -func answeringTheExchange(w http.ResponseWriter, r *http.Request) { - if r.URL.Path == solutionRegistrationTokenPath { - writeJSON(w, http.StatusOK, map[string]string{ - "token": "solution-token", - "expiresAt": time.Now().Add(5 * time.Minute).UTC().Format(time.RFC3339), - }) - return - } - w.WriteHeader(http.StatusOK) -} - -// TestRegistrationClientNeverFollowsARedirect proves the registration secrets -// cannot be walked off to another host by a Location header. net/http strips -// only Authorization, WWW-Authenticate and Cookie across hosts, so before this -// the X-Codefly-* headers — the plaintext registration secret and the -// cluster-internal token — were copied to whatever the redirect named, and a -// 307 re-sent the body with them. Anything able to answer at a registration URL -// could harvest the credential this publisher binding exists to protect. -func TestRegistrationClientNeverFollowsARedirect(t *testing.T) { - var sinkSaw, sinkInternal string - var sinkHits atomic.Int32 - sink := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - sinkHits.Add(1) - sinkSaw, sinkInternal = r.Header.Get(solutionSecretHeader), r.Header.Get(internalTokenHeader) - writeJSON(w, http.StatusOK, map[string]string{ - "token": "stolen-and-answered", - "expiresAt": time.Now().Add(5 * time.Minute).UTC().Format(time.RFC3339), - }) - })) - defer sink.Close() - // A different host name, so this is the cross-host case net/http claims to - // protect — and does not, for custom headers. - target := strings.Replace(sink.URL, "127.0.0.1", "localhost", 1) - redirector := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - http.Redirect(w, r, target, http.StatusTemporaryRedirect) - })) - defer redirector.Close() - - credential := &solutionCredential{ - tokenURL: redirector.URL, - internalToken: internalTokenTest, - id: "lastlogin-go", - secret: "s3cret", - } - token, _, err := credential.exchange(context.Background()) - if err == nil { - t.Fatalf("exchange followed the redirect and accepted %q; a redirected registration endpoint must fail, not be trusted", token) - } - if n := sinkHits.Load(); n != 0 { - t.Fatalf("the redirect target was contacted %d times; the request must stop at the 3xx", n) - } - if sinkSaw != "" || sinkInternal != "" { - t.Fatalf("the redirect target received the credentials: %s=%q %s=%q", solutionSecretHeader, sinkSaw, internalTokenHeader, sinkInternal) - } - if !strings.Contains(err.Error(), "307") { - t.Errorf("the 3xx must be reported as the answer it is; got %v", err) - } -} - -// TestBeatIsBoundedByTheCredentialsExpiry proves a registration cannot outlive -// the token it presents. The floor only guarantees a token is worth sending; -// the POST is allowed registrationTimeout (10s), which is longer than a -// short-lived token's whole life. Unbounded, such a token arrives expired and -// the host answers 401 — indistinguishable here from an id owned by another -// publisher, which is what invalidate() would then report, sending whoever -// reads it to check publisher ownership for what is a token-lifetime problem. -func TestBeatIsBoundedByTheCredentialsExpiry(t *testing.T) { - h := newFakeHost(t, "s3cret") - h.tokenLifetime = 1500 * time.Millisecond - h.registerDelay = 5 * time.Second // longer than the token lives - - s := &Server{manifest: Manifest{ID: "lastlogin-go"}} - credential := &solutionCredential{ - tokenURL: h.URL + solutionRegistrationTokenPath, - internalToken: internalTokenTest, - id: "lastlogin-go", - secret: "s3cret", - } - start := time.Now() - status, _, err := s.beat(context.Background(), h.URL+solutionRegisterPath, []byte("{}"), credential) - if err == nil { - t.Fatalf("beat returned status %d; a POST that outlives its credential must fail as a deadline, not be sent on to be refused", status) - } - if !strings.Contains(err.Error(), "expired at") { - t.Fatalf("the failure must name the credential's expiry, not read as a generic timeout: %v", err) - } - // Bounded by the token (1.5s), not by registrationTimeout (10s). - if elapsed := time.Since(start); elapsed > 4*time.Second { - t.Errorf("beat took %s; it must be cut short at the token's expiry, not at the request timeout", elapsed) - } -} - -// TestValidateBlamesTheEnvironmentLoadNotTheProvisioning proves a boot whose -// SDK environment never loaded says so. Every SDK-resolved value is empty in -// that case, and the message that named the provisioning sent an operator to -// provision a secret that was already provisioned. -func TestValidateBlamesTheEnvironmentLoadNotTheProvisioning(t *testing.T) { - cfg := config{ - port: "1234", - gatewayURL: "http://gateway:42152", - hostRegisterURL: "http://frontend:21931/api/solutions/register", - gatewayRegisterURL: "http://gateway:42152/solutions/_register", - moduleRegisterURL: "http://gateway:42152/modules/_register", - moduleTokenURL: "http://gateway:42152/modules/_registration-token", - solutionTokenURL: "http://gateway:42152/solutions/_registration-token", - environmentLoadErr: fmt.Errorf("carrier %q is unreadable", "/var/run/codefly/env"), - } - err := cfg.validate() - if err == nil { - t.Fatal("validate() with no secret and a failed environment load = nil, want an error") - } - if !strings.Contains(err.Error(), "carrier") { - t.Errorf("the environment-load failure must reach the boot error: %v", err) - } - if strings.Contains(err.Error(), SolutionRegistrationSecretGroup+"/"+SolutionRegistrationSecretKey) { - t.Errorf("with the environment unloaded the secret may well be provisioned; the message must not send the operator to provision it: %v", err) - } - // Unloaded environments also empty the URLs; same misattribution. - cfg.solutionSecret, cfg.moduleTokenURL = "s3cret", "" - if err := cfg.validate(); err == nil || !strings.Contains(err.Error(), "carrier") { - t.Errorf("an unresolved URL under a failed environment load must name the load: %v", err) - } -} - -// TestValidateNamesTheUnpairableRegisterOverride proves an override this -// runtime cannot derive an exchange from names itself. GATEWAY_REGISTER_URL was -// free-form until the exchange began deriving from it, so a deployment that -// booted yesterday can refuse to boot today; reported as an unresolved SDK -// endpoint, that sends the operator to endpoint resolution for a value their -// own override broke. -func TestValidateNamesTheUnpairableRegisterOverride(t *testing.T) { - t.Setenv("PORT", "8090") - t.Setenv("GATEWAY_URL", "http://gateway:42152") - t.Setenv("HOST_REGISTER_URL", "http://frontend:21931/api/solutions/register") - t.Setenv("GATEWAY_REGISTER_URL", "https://gw.example.com/api/solutions/register") - t.Setenv(SolutionRegistrationSecretEnvironmentVariable, "s3cret") - - cfg := loadConfig(context.Background(), "lastlogin-go") - if cfg.solutionTokenURL != "" { - t.Fatalf("solutionTokenURL = %q, want empty: the override cannot be paired", cfg.solutionTokenURL) - } - err := cfg.validate() - if err == nil { - t.Fatal("validate() with an unpairable register override = nil, want an error") - } - for _, want := range []string{"GATEWAY_REGISTER_URL", "GATEWAY_SOLUTION_REGISTRATION_TOKEN_URL", solutionRegisterPath} { - if !strings.Contains(err.Error(), want) { - t.Errorf("the error must name %q, so the operator sees which variable they must fix: %v", want, err) - } - } -} - -// TestCapabilitiesAndManifestDeclareTheSameMajors proves the two documents -// cannot disagree. Hardcoded in the capabilities handler, a bump of either -// constant left this document announcing the old major while the registration -// manifest announced the new one — one solution claiming two contracts. -func TestCapabilitiesAndManifestDeclareTheSameMajors(t *testing.T) { - s := New(Manifest{ID: "lastlogin-go", Title: "Last Login"}) - rec := httptest.NewRecorder() - s.handleCapabilities(rec, httptest.NewRequest(http.MethodGet, "/.well-known/capabilities", nil)) - - var capabilities map[string]any - if err := json.NewDecoder(rec.Body).Decode(&capabilities); err != nil { - t.Fatal(err) - } - manifest := s.manifestMap() - if got, want := capabilities["schemaVersion"], manifest["schemaVersion"]; got != float64(want.(int)) { - t.Errorf("capabilities schemaVersion = %v, manifest says %v", got, want) - } - frontend := manifest["frontend"].(map[string]any) - if got, want := capabilities["contractMajor"], frontend["hostContract"]; got != float64(want.(int)) { - t.Errorf("capabilities contractMajor = %v, manifest hostContract says %v", got, want) - } -} - -// TestBackoffCapDependsOnWhatTheFailureCost proves the two caps. A refusal cost -// a mint, so it is bounded by the long cap that exists to bound audited events. -// A beat that never reached the surface minted nothing, and holding it on the -// long cap left this solution missing from a host that may be perfectly healthy -// for minutes after the gateway it could not reach came back. -func TestBackoffCapDependsOnWhatTheFailureCost(t *testing.T) { - const interval = 15 * time.Second - if got := backoff(interval, 50, registrationBackoffCap); got != registrationBackoffCap { - t.Errorf("a refused beat backs off to %s, want the mint-bounding cap %s", got, registrationBackoffCap) - } - if got := backoff(interval, 50, registrationUnreachedBackoffCap); got != registrationUnreachedBackoffCap { - t.Errorf("an unreached beat backs off to %s, want the short cap %s", got, registrationUnreachedBackoffCap) - } - if registrationUnreachedBackoffCap >= registrationBackoffCap { - t.Fatalf("the unreached cap (%s) must be shorter than the refusal cap (%s): it bounds an outage nobody's mint rate pays for", - registrationUnreachedBackoffCap, registrationBackoffCap) - } -} - -// TestHeartbeatStopsQuotingAnEndlesslyChangingReason proves the reason is not a -// dedup key this loop trusts to settle. A refusal quoting something per-attempt -// — the jti of the single-use token it just burned — differs on every beat by -// construction, so keyed on it alone the throttle was defeated exactly when the -// registration was most broken: one line per beat, forever. -func TestHeartbeatStopsQuotingAnEndlesslyChangingReason(t *testing.T) { - buf := &syncBuffer{} - log.SetOutput(buf) - defer log.SetOutput(os.Stderr) - - var beats atomic.Int32 - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - writeJSON(w, http.StatusConflict, map[string]any{ - "error": "token_already_used", - "reasons": []string{fmt.Sprintf("jti %d was already burned", beats.Add(1))}, - }) - })) - defer srv.Close() - - ctx, cancel := context.WithCancel(context.Background()) - s := &Server{manifest: Manifest{ID: "lastlogin-go"}, registrationInterval: time.Millisecond} - done := make(chan struct{}) - go func() { s.heartbeat(ctx, srv.URL, []byte("{}"), "host", internalTokenAuth("t")); close(done) }() - deadline := time.Now().Add(10 * time.Second) - for beats.Load() < int32(maxRefusalDetailsPerStatus)+20 { - if time.Now().After(deadline) { - break - } - time.Sleep(5 * time.Millisecond) - } - cancel() - <-done - - if n := strings.Count(buf.String(), "rejected (status 409)"); n > maxRefusalDetailsPerStatus { - t.Fatalf("logged %d refusal lines for one unchanging status after %d beats, want at most %d:\n%s", - n, beats.Load(), maxRefusalDetailsPerStatus, buf.String()) - } - if !strings.Contains(buf.String(), "further reasons are suppressed") { - t.Errorf("suppression must be announced rather than going silent:\n%s", buf.String()) - } -} - -// TestSolutionSecretIsTrimmed proves a secret carrying the newline a file-mounted -// or `echo`-generated value almost always has is usable. net/http refuses to -// write a header containing one, so the request never left the process: the beat -// failed with "invalid header field value" forever while validate() saw a -// non-empty secret and let the boot through — a registration that silently never -// happened, with correct provisioning. -func TestSolutionSecretIsTrimmed(t *testing.T) { - t.Setenv(SolutionRegistrationSecretEnvironmentVariable, "s3cret\n") - got := solutionRegistrationSecret(context.Background()) - if got != "s3cret" { - t.Fatalf("solutionRegistrationSecret() = %q, want the value without its trailing newline", got) - } - // The point of trimming: the value must survive being written as a header. - req, err := http.NewRequest(http.MethodPost, "http://example.invalid", nil) - if err != nil { - t.Fatal(err) - } - req.Header.Set(solutionSecretHeader, got) - if err := req.Write(io.Discard); err != nil { - t.Fatalf("the resolved secret cannot be sent as a header: %v", err) - } -} - -// TestRegistrationIntervalIsConfigurable proves the beat period has a lever. A -// beat is no longer free: each self-registration beat mints a token whose every -// success is an audited event on the issuer, and the right period depends on the -// host's registration TTL, which this runtime cannot observe. -func TestRegistrationIntervalIsConfigurable(t *testing.T) { - t.Run("honoured", func(t *testing.T) { - t.Setenv(RegistrationIntervalEnvironmentVariable, "90s") - interval, err := registrationIntervalFromEnv() - if err != nil || interval != 90*time.Second { - t.Fatalf("registrationIntervalFromEnv() = %s, %v; want 90s, nil", interval, err) - } - }) - t.Run("unset means the default", func(t *testing.T) { - interval, err := registrationIntervalFromEnv() - if err != nil || interval != 0 { - t.Fatalf("registrationIntervalFromEnv() = %s, %v; want 0, nil", interval, err) - } - }) - // A typo must not silently restore the fast default for an operator who - // asked for a slow beat precisely to bound their mint rate. - for _, raw := range []string{"ninety", "100", "0s", "-5s"} { - t.Run("refuses "+raw, func(t *testing.T) { - t.Setenv(RegistrationIntervalEnvironmentVariable, raw) - if _, err := registrationIntervalFromEnv(); err == nil { - t.Fatalf("registrationIntervalFromEnv() accepted %q; an unusable interval must fail the boot, not fall back", raw) - } - cfg := config{ - port: "1234", - gatewayURL: "http://gateway:42152", - hostRegisterURL: "http://frontend:21931/api/solutions/register", - gatewayRegisterURL: "http://gateway:42152/solutions/_register", - moduleRegisterURL: "http://gateway:42152/modules/_register", - moduleTokenURL: "http://gateway:42152/modules/_registration-token", - solutionTokenURL: "http://gateway:42152/solutions/_registration-token", - solutionSecret: "s3cret", - registrationIntervalErr: fmt.Errorf("unusable"), - } - if err := cfg.validate(); err == nil { - t.Fatal("validate() accepted an unusable registration interval") - } - }) - } -} - -// TestSolutionExchangeBlamesTheCredentialOnlyWhenItWasJudged proves only a -// 401/403 from the exchange names the provisioning. A 5xx is the host being -// unavailable — reported with the gateway's own words — and before this every -// status fell through to "no SOLUTION_REGISTRATION_SECRETS entry / digest -// mismatch", sending an operator to re-provision a secret that was correct -// while the gateway could not reach accounts. -func TestSolutionExchangeBlamesTheCredentialOnlyWhenItWasJudged(t *testing.T) { - for _, tc := range []struct { - status int - body string - want []string - credential bool - }{ - {status: http.StatusUnauthorized, credential: true}, - {status: http.StatusForbidden, credential: true}, - {status: http.StatusInternalServerError, body: `{"error":"accounts unavailable"}`, - want: []string{"host unavailable (status 500), retrying", `{"error":"accounts unavailable"}`}}, - {status: http.StatusBadGateway, body: "upstream connect error\nor disconnect", - want: []string{"host unavailable (status 502), retrying", "upstream connect error or disconnect"}}, - {status: http.StatusServiceUnavailable, - want: []string{"host unavailable (status 503), retrying: no body"}}, - {status: http.StatusTooManyRequests, body: "slow down", - want: []string{"failed (status 429), retrying: slow down"}}, - } { - t.Run(strconv.Itoa(tc.status), func(t *testing.T) { - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - w.WriteHeader(tc.status) - _, _ = io.WriteString(w, tc.body) - })) - defer srv.Close() - credential := &solutionCredential{tokenURL: srv.URL, id: "lastlogin-go", secret: "s3cret"} - _, _, err := credential.exchange(context.Background()) - if err == nil { - t.Fatalf("status %d: exchange succeeded", tc.status) - } - blames := strings.Contains(err.Error(), "SOLUTION_REGISTRATION_SECRETS") - if blames != tc.credential { - t.Fatalf("status %d: blames the credential = %v, want %v: %v", tc.status, blames, tc.credential, err) - } - for _, want := range tc.want { - if !strings.Contains(err.Error(), want) { - t.Errorf("status %d: error %q does not contain %q", tc.status, err, want) - } - } - }) - } -} - -// TestSolutionRegistrationKeepsRetryingAnUnavailableHost proves a host whose -// exchange answers 5xx is retried, not given up on, and that the log says the -// host is unavailable rather than that the secret is wrong. -func TestSolutionRegistrationKeepsRetryingAnUnavailableHost(t *testing.T) { - buf := &syncBuffer{} - log.SetOutput(buf) - defer log.SetOutput(os.Stderr) - - var exchanges atomic.Int32 - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.URL.Path == solutionRegistrationTokenPath { - exchanges.Add(1) - w.WriteHeader(http.StatusBadGateway) - _, _ = io.WriteString(w, "accounts unreachable") - return - } - w.WriteHeader(http.StatusOK) - })) - defer srv.Close() - - ln, err := net.Listen("tcp", "127.0.0.1:0") - if err != nil { - t.Fatal(err) - } - s := New(Manifest{ID: "lastlogin-go", Title: "Last Login"}) - s.registrationInterval = 10 * time.Millisecond - s.cfg = config{ - port: strconv.Itoa(ln.Addr().(*net.TCPAddr).Port), - hostRegisterURL: srv.URL + "/api/solutions/register", - gatewayRegisterURL: srv.URL + solutionRegisterPath, - moduleRegisterURL: srv.URL + moduleRegisterPath, - moduleTokenURL: srv.URL + moduleRegistrationTokenPath, - solutionTokenURL: srv.URL + solutionRegistrationTokenPath, - internalToken: internalTokenTest, - solutionSecret: "s3cret", - } - ctx, cancel := context.WithCancel(context.Background()) - done := make(chan struct{}) - go func() { _ = s.serve(ctx, ln); close(done) }() - defer func() { cancel(); <-done }() - - // Two surfaces, each beating: six exchanges is at least two retries each. - waitFor(t, "exchange retries", func() bool { return exchanges.Load() >= 6 }) - out := buf.String() - if !strings.Contains(out, "host unavailable (status 502), retrying: accounts unreachable") { - t.Fatalf("an unavailable host was not reported as such:\n%s", out) - } - if strings.Contains(out, "SOLUTION_REGISTRATION_SECRETS") { - t.Fatalf("a 502 was reported as a provisioning fault:\n%s", out) - } -} diff --git a/solution_test.go b/solution_test.go index da17a0d..c483342 100644 --- a/solution_test.go +++ b/solution_test.go @@ -4,25 +4,87 @@ import ( "bytes" "context" "encoding/json" + "errors" "fmt" + corework "github.com/codefly-dev/core/workcontext" "io" "log" - "net" "net/http" "net/http/httptest" - "net/url" "os" "path/filepath" "reflect" - "strconv" "strings" "sync" "testing" "time" + "github.com/codefly-dev/core/resources" codefly "github.com/codefly-dev/sdk-go" + "github.com/codefly-dev/sdk-go/workcontext" ) +// capability is a stand-in issuer's capability, minted per seed. +// +// It is minted by **core's** authority, from core's own conformance fixture +// identities and key — the one implementation of a Work Context, which this +// runtime neither signs nor parses. The private key is public by design (core +// derives it from a seed in its own source), so this is a real sealed +// capability that a conforming verifier accepts, without a signer or a payload +// struct living here. That matters beyond convenience: a fixture issuer with +// its own encoding is how two implementations of a capability format start. +// +// Memoised by seed so a test can recompute the capability it expects to have +// been presented: every mint carries a fresh nonce, so minting the same seed +// twice would otherwise produce two different strings. +func capability(seed string) string { + capabilityMu.Lock() + defer capabilityMu.Unlock() + if issued, ok := issuedCapabilities[seed]; ok { + return issued + } + token, _, err := standInAuthority().Start(context.Background(), corework.StartInput{ + TenantID: corework.FixtureTenant, + OwnerPrincipalID: corework.FixturePrincipal, + OwnerPrincipalKind: "human", + TaskID: seed, + Audience: corework.FixtureAudience, + OrganizationID: corework.FixtureOrganization, + InstallationID: corework.FixtureInstallation, + TTL: 10 * time.Minute, + Execution: corework.Execution{ + ImageDigest: corework.FixtureImageDigest, + BuildIncarnation: corework.FixtureBuildIncarnation, + }, + }) + if err != nil { + panic("stand-in capability: " + err.Error()) + } + issuedCapabilities[seed] = token + return token +} + +var ( + capabilityMu sync.Mutex + issuedCapabilities = map[string]string{} + standInAuthorityV *corework.Authority +) + +// standInAuthority is core's minter, configured from core's fixture identities. +func standInAuthority() *corework.Authority { + if standInAuthorityV == nil { + _, key := corework.FixtureKeyPair() + standInAuthorityV = &corework.Authority{ + Issuer: corework.FixtureIssuer, + KeyID: corework.FixtureKeyID, + Key: key, + Revisions: corework.FixtureRevisions(), + Seals: corework.FixtureSeals(), + } + } + return standInAuthorityV +} + type syncBuffer struct { mu sync.Mutex buf bytes.Buffer @@ -40,134 +102,6 @@ func (b *syncBuffer) String() string { return b.buf.String() } -func TestHeartbeatSendsInternalToken(t *testing.T) { - tests := []struct { - name string - token string - wantToken string - }{ - {name: "with token", token: "local-dev-only-replace-me", wantToken: "local-dev-only-replace-me"}, - {name: "without token", token: "", wantToken: ""}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got := make(chan string, 1) - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - got <- r.Header.Get("x-codefly-internal-token") - w.WriteHeader(http.StatusOK) - })) - defer srv.Close() - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - s := &Server{cfg: config{internalToken: tt.token}} - done := make(chan struct{}) - go func() { - s.heartbeat(ctx, srv.URL, []byte("{}"), "host", internalTokenAuth(tt.token)) - close(done) - }() - - select { - case header := <-got: - if header != tt.wantToken { - t.Errorf("x-codefly-internal-token = %q, want %q", header, tt.wantToken) - } - case <-time.After(5 * time.Second): - t.Fatal("heartbeat did not POST within timeout") - } - cancel() - <-done - }) - } -} - -// TestServeRegistersOnListenPort is the solution↔host regression guard: it boots -// a real solution on an OS-assigned port against fake host and gateway registries -// and asserts the invariants that a wrong port / wrong host URL broke — the -// runtime binds and advertises the same port, registers with both host and -// gateway, and routes an unauthenticated call to a 401 rather than a 502. -func TestServeRegistersOnListenPort(t *testing.T) { - buf := &syncBuffer{} - log.SetOutput(buf) - defer log.SetOutput(os.Stderr) - - hostSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - w.WriteHeader(http.StatusOK) - })) - defer hostSrv.Close() - gatewaySrv := httptest.NewServer(http.HandlerFunc(answeringTheExchange)) - defer gatewaySrv.Close() - - assets := t.TempDir() - if err := os.WriteFile(filepath.Join(assets, "mf-manifest.json"), []byte(`{"name":"test"}`), 0o644); err != nil { - t.Fatal(err) - } - - ln, err := net.Listen("tcp", "127.0.0.1:0") - if err != nil { - t.Fatal(err) - } - port := strconv.Itoa(ln.Addr().(*net.TCPAddr).Port) - - t.Setenv("PORT", port) - t.Setenv("HOST_REGISTER_URL", hostSrv.URL) - t.Setenv("GATEWAY_REGISTER_URL", gatewaySrv.URL+solutionRegisterPath) - t.Setenv("GATEWAY_URL", gatewaySrv.URL) - t.Setenv("ASSETS_DIR", assets) - t.Setenv(SolutionRegistrationSecretEnvironmentVariable, "s3cret") - - s := New(Manifest{ID: "lastlogin-go", Title: "Last Login"}). - Handle("/audit", func(context.Context, *Gateway) (any, error) { - return map[string]string{"ok": "yes"}, nil - }) - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - s.cfg = loadConfig(ctx, s.manifest.ID) - if s.cfg.port != port { - t.Fatalf("loadConfig port = %q, want assigned listen port %q", s.cfg.port, port) - } - done := make(chan struct{}) - go func() { _ = s.serve(ctx, ln); close(done) }() - - base := "http://127.0.0.1:" + port - waitFor(t, "health", func() bool { return getStatus(t, base+"/health") == http.StatusOK }) - - // With no PUBLIC_URL the manifest is registered as a path on this backend — - // never on its own loopback listen address, which no browser but the - // developer's can reach — and that path is one this backend serves. - manifestURL := frontendManifestURL(t, base) - u, err := url.Parse(manifestURL) - if err != nil { - t.Fatalf("parse manifestUrl %q: %v", manifestURL, err) - } - if u.IsAbs() || u.Host != "" || manifestURL != federationManifestPath { - t.Errorf("manifestUrl = %q, want the root-relative path %q", manifestURL, federationManifestPath) - } - if code := getStatus(t, base+u.Path); code != http.StatusOK { - t.Errorf("GET manifestUrl = %d, want 200", code) - } - - // A solution route without a bearer is rejected as 401 (auth). A 502 would - // mean the request never reached a live upstream — the failure mode this - // registration path exists to avoid. - if code := getStatus(t, base+"/audit"); code != http.StatusUnauthorized { - t.Errorf("GET /audit without bearer = %d, want 401", code) - } - - waitFor(t, "host registration", func() bool { return strings.Contains(buf.String(), `registered with host`) }) - waitFor(t, "gateway registration", func() bool { return strings.Contains(buf.String(), `registered with gateway`) }) - - // Exactly one process owns the backend port: a second bind must fail. - if extra, err := net.Listen("tcp", "127.0.0.1:"+port); err == nil { - extra.Close() - t.Errorf("port %s bound a second time; runtime is not its sole owner", port) - } - - cancel() - <-done -} - // sampleDataGraph is a minimal but structurally complete DataGraph // (events / metrics / dashboards) a solution might declare, matching the shape // the host's @codefly/saas-plugin-manifest validates. @@ -202,7 +136,7 @@ func sampleDataGraph() map[string]any { // through JSON, so the host receives exactly what the solution declared. func TestServedManifestCarriesDashboardVerbatim(t *testing.T) { graph := sampleDataGraph() - s := &Server{manifest: Manifest{ID: "lastlogin-go", Dashboard: graph}} + s := &Server{manifest: Manifest{ID: testSolutionID, Dashboard: graph}} var got map[string]any body, err := json.Marshal(s.manifestMap()) @@ -221,20 +155,20 @@ func TestServedManifestCarriesDashboardVerbatim(t *testing.T) { // not even a null one, which would break a host that feeds a present slot into // its data-graph validator and would perturb the existing wire contract. func TestManifestOmitsAbsentDashboard(t *testing.T) { - s := &Server{manifest: Manifest{ID: "lastlogin-go"}} + s := &Server{manifest: Manifest{ID: testSolutionID}} if _, ok := s.manifestMap()["dashboard"]; ok { t.Errorf("emitted a dashboard key with no dashboard declared") } } -// wordFootnote is the surface shape the wiki declares for a Word document: the +// wordFootnote is the surface shape the notes solution declares for a Word document: the // whole set of slots a client reads, so a test can assert on all of them. func wordFootnote() Surface { return Surface{ ID: "footnote", Client: "word", Title: "Footnote", - Description: "Cite a claim from the wiki.", + Description: "Cite a claim from the notes solution.", Module: "/surfaces/word/footnote.js", Contract: 1, Applies: "always", @@ -246,7 +180,7 @@ func wordFootnote() Surface { // so every declared slot must survive the trip through JSON — including the // applies selector, which the runtime carries without interpreting. func TestServedManifestCarriesDeclaredSurfaces(t *testing.T) { - s := &Server{manifest: Manifest{ID: "wiki", Surfaces: []Surface{wordFootnote()}}} + s := &Server{manifest: Manifest{ID: "notes", Surfaces: []Surface{wordFootnote()}}} var got map[string]any body, err := json.Marshal(s.manifestMap()) @@ -260,7 +194,7 @@ func TestServedManifestCarriesDeclaredSurfaces(t *testing.T) { "id": "footnote", "client": "word", "title": "Footnote", - "description": "Cite a claim from the wiki.", + "description": "Cite a claim from the notes solution.", "module": "/surfaces/word/footnote.js", "contract": float64(1), "applies": "always", @@ -277,7 +211,7 @@ func TestServedSurfaceCarriesTaggedAppliesVerbatim(t *testing.T) { applies := map[string]any{"tagged": []any{"legal", "finance"}} surface := wordFootnote() surface.Applies = applies - s := &Server{manifest: Manifest{ID: "wiki", Surfaces: []Surface{surface}}} + s := &Server{manifest: Manifest{ID: "notes", Surfaces: []Surface{surface}}} var got map[string]any body, _ := json.Marshal(s.manifestMap()) @@ -292,7 +226,7 @@ func TestServedSurfaceCarriesTaggedAppliesVerbatim(t *testing.T) { // A solution that offers nothing inside a client must not gain the key at all. func TestManifestOmitsAbsentSurfaces(t *testing.T) { - bare := &Server{manifest: Manifest{ID: "lastlogin-go"}} + bare := &Server{manifest: Manifest{ID: testSolutionID}} if _, ok := bare.manifestMap()["surfaces"]; ok { t.Errorf("emitted a surfaces key with no surface declared") } @@ -308,7 +242,7 @@ func TestManifestDefaultsAppliesAndOmitsEmptySurfaceSlots(t *testing.T) { surface.Applies = nil surface.Events = nil surface.Description = "" - s := &Server{manifest: Manifest{ID: "wiki", Surfaces: []Surface{surface}}} + s := &Server{manifest: Manifest{ID: "notes", Surfaces: []Surface{surface}}} entry := s.manifestMap()["surfaces"].([]any)[0].(map[string]any) if got := entry["applies"]; got != "always" { @@ -380,7 +314,7 @@ func TestSurfaceValidationRejectsUnusableDeclarations(t *testing.T) { } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { - err := Manifest{ID: "wiki", Surfaces: tc.surfaces}.validateSurfaces() + err := Manifest{ID: "notes", Surfaces: tc.surfaces}.validateSurfaces() if err == nil { t.Fatalf("validateSurfaces accepted %s", tc.name) } @@ -391,21 +325,21 @@ func TestSurfaceValidationRejectsUnusableDeclarations(t *testing.T) { } } -// The same offering in two clients carries the same id — the wiki's footnote is +// The same offering in two clients carries the same id — the notes solution's footnote is // "footnote" in Word and in PowerPoint — and neither client can see the other's, // so that is not a collision. Only two surfaces of one client are ambiguous. func TestSurfaceIDsAreUniquePerClientNotPerSolution(t *testing.T) { powerpoint := wordFootnote() powerpoint.Client = "powerpoint" powerpoint.Module = "/surfaces/powerpoint/footnote.js" - if err := (Manifest{ID: "wiki", Surfaces: []Surface{wordFootnote(), powerpoint}}).validateSurfaces(); err != nil { + if err := (Manifest{ID: "notes", Surfaces: []Surface{wordFootnote(), powerpoint}}).validateSurfaces(); err != nil { t.Errorf("validateSurfaces refused one id shared across two clients: %v", err) } second := wordFootnote() second.Title = "Footnote, again" second.Module = "/surfaces/word/footnote-2.js" - err := (Manifest{ID: "wiki", Surfaces: []Surface{wordFootnote(), second}}).validateSurfaces() + err := (Manifest{ID: "notes", Surfaces: []Surface{wordFootnote(), second}}).validateSurfaces() if err == nil { t.Fatal("validateSurfaces accepted two surfaces sharing an id within one client") } @@ -421,7 +355,7 @@ func TestServeRejectsUnusableSurface(t *testing.T) { surface := wordFootnote() surface.Module = `/\evil.example/footnote.js` - s := New(Manifest{ID: "wiki", Title: "Wiki", Surfaces: []Surface{surface}}) + s := New(Manifest{ID: "notes", Title: "Notes", Surfaces: []Surface{surface}}) err := s.Serve() if err == nil { t.Fatal("Serve returned nil for an off-origin surface module; expected a boot error and no bind") @@ -431,244 +365,6 @@ func TestServeRejectsUnusableSurface(t *testing.T) { } } -// The host registration payload (the heartbeat body) must carry the declared -// data-graph verbatim, not just the GET manifest — that is the surface the host -// self-registration path reads. -func TestRegistrationPayloadCarriesDashboardVerbatim(t *testing.T) { - graph := sampleDataGraph() - - body := make(chan []byte, 1) - hostSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - b, _ := io.ReadAll(r.Body) - select { - case body <- b: - default: - } - w.WriteHeader(http.StatusOK) - })) - defer hostSrv.Close() - gatewaySrv := httptest.NewServer(http.HandlerFunc(answeringTheExchange)) - defer gatewaySrv.Close() - - ln, err := net.Listen("tcp", "127.0.0.1:0") - if err != nil { - t.Fatal(err) - } - - s := New(Manifest{ID: "lastlogin-go", Title: "Last Login", Dashboard: graph}) - s.cfg = config{ - port: strconv.Itoa(ln.Addr().(*net.TCPAddr).Port), - publicURL: "http://127.0.0.1", - hostRegisterURL: hostSrv.URL, - gatewayRegisterURL: gatewaySrv.URL + solutionRegisterPath, - solutionTokenURL: gatewaySrv.URL + solutionRegistrationTokenPath, - solutionSecret: "s3cret", - } - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - done := make(chan struct{}) - go func() { _ = s.serve(ctx, ln); close(done) }() - - var raw []byte - select { - case raw = <-body: - case <-time.After(5 * time.Second): - t.Fatal("host did not receive a registration within timeout") - } - cancel() - <-done - - var payload map[string]any - if err := json.Unmarshal(raw, &payload); err != nil { - t.Fatalf("unmarshal registration body: %v", err) - } - if !reflect.DeepEqual(payload["dashboard"], graph) { - t.Errorf("registration dashboard = %#v, want %#v", payload["dashboard"], graph) - } -} - -func waitFor(t *testing.T, what string, cond func() bool) { - t.Helper() - deadline := time.Now().Add(5 * time.Second) - for !cond() { - if time.Now().After(deadline) { - t.Fatalf("timed out waiting for %s", what) - } - time.Sleep(10 * time.Millisecond) - } -} - -func getStatus(t *testing.T, target string) int { - t.Helper() - resp, err := http.Get(target) - if err != nil { - return 0 - } - resp.Body.Close() - return resp.StatusCode -} - -func frontendManifestURL(t *testing.T, base string) string { - t.Helper() - resp, err := http.Get(base + "/.well-known/solution.json") - if err != nil { - t.Fatalf("GET solution.json: %v", err) - } - defer resp.Body.Close() - var manifest struct { - Frontend struct { - ManifestURL string `json:"manifestUrl"` - } `json:"frontend"` - } - if err := json.NewDecoder(resp.Body).Decode(&manifest); err != nil { - t.Fatalf("decode solution.json: %v", err) - } - if manifest.Frontend.ManifestURL == "" { - t.Fatal("solution.json has no frontend.manifestUrl") - } - return manifest.Frontend.ManifestURL -} - -// TestConfigValidate guards the boot-time floor: an unresolved value (empty port, -// or a scheme-less register URL as loadConfig produces when the SDK resolves -// nothing and no override is set) must be rejected, not silently accepted into a -// ":"+"" bind and heartbeats to relative URLs. -func TestConfigValidate(t *testing.T) { - valid := config{ - port: "8090", - gatewayURL: "http://gateway:42152", - hostRegisterURL: "http://frontend:21931/api/solutions/register", - gatewayRegisterURL: "http://gateway:42152/solutions/_register", - moduleRegisterURL: "http://gateway:42152/modules/_register", - moduleTokenURL: "http://gateway:42152/modules/_registration-token", - solutionTokenURL: "http://gateway:42152/solutions/_registration-token", - solutionSecret: "s3cret", - selfUpstream: "http://backend:8080", - } - if err := valid.validate(); err != nil { - t.Fatalf("valid config rejected: %v", err) - } - - cases := []struct { - name string - mutate func(*config) - }{ - {"empty port", func(c *config) { c.port = "" }}, - {"non-numeric port", func(c *config) { c.port = "http" }}, - {"port out of range", func(c *config) { c.port = "70000" }}, - {"empty gateway URL", func(c *config) { c.gatewayURL = "" }}, - {"relative host register URL", func(c *config) { c.hostRegisterURL = "/api/solutions/register" }}, - {"relative gateway register URL", func(c *config) { c.gatewayRegisterURL = "/solutions/_register" }}, - {"relative module register URL", func(c *config) { c.moduleRegisterURL = "/modules/_register" }}, - {"relative module token URL", func(c *config) { c.moduleTokenURL = "/modules/_registration-token" }}, - {"relative solution token URL", func(c *config) { c.solutionTokenURL = "/solutions/_registration-token" }}, - {"no solution registration secret", func(c *config) { c.solutionSecret = "" }}, - } - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - c := valid - tc.mutate(&c) - if err := c.validate(); err == nil { - t.Errorf("validate accepted an unresolved config (%s)", tc.name) - } - }) - } -} - -// TestServeRejectsUnresolvedConfig proves Serve refuses to boot on an unresolved -// config instead of binding a random port and registering into the void. A -// relative HOST_REGISTER_URL stands in for the empty frontendURL loadConfig -// yields when the SDK cannot resolve the host frontend endpoint. -func TestServeRejectsUnresolvedConfig(t *testing.T) { - t.Setenv("PORT", "8090") - t.Setenv("GATEWAY_URL", "http://127.0.0.1:1") - t.Setenv("GATEWAY_REGISTER_URL", "http://127.0.0.1:1/solutions/_register") - t.Setenv("HOST_REGISTER_URL", "/api/solutions/register") - - s := New(Manifest{ID: "lastlogin-go", Title: "Last Login"}) - err := s.Serve() - if err == nil { - t.Fatal("Serve returned nil for an unresolved config; expected a boot error and no bind") - } - if !strings.Contains(err.Error(), "host register URL") { - t.Errorf("boot error should name the unresolved field, got: %v", err) - } -} - -// TestHeartbeatLogsTransportError guards finding #4: a round trip that never -// yields an HTTP status (scheme-less URL, connection refused) must be logged, not -// swallowed — a permanently-failing registration was previously indistinguishable -// from a working one. -func TestHeartbeatLogsTransportError(t *testing.T) { - buf := &syncBuffer{} - log.SetOutput(buf) - defer log.SetOutput(os.Stderr) - - ctx, cancel := context.WithCancel(context.Background()) - s := &Server{cfg: config{internalToken: "t"}} - done := make(chan struct{}) - go func() { - s.heartbeat(ctx, "/solutions/_register", []byte("{}"), "gateway", internalTokenAuth("t")) - close(done) - }() - - deadline := time.Now().Add(5 * time.Second) - for !strings.Contains(buf.String(), "failed") { - if time.Now().After(deadline) { - cancel() - <-done - t.Fatalf("no transport failure logged within timeout, got: %q", buf.String()) - } - time.Sleep(10 * time.Millisecond) - } - cancel() - <-done - - if strings.Contains(buf.String(), "registered with") { - t.Errorf("logged success on a transport failure: %q", buf.String()) - } -} - -func TestHeartbeatLogsRejection(t *testing.T) { - buf := &syncBuffer{} - log.SetOutput(buf) - defer log.SetOutput(os.Stderr) - - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - w.WriteHeader(http.StatusUnauthorized) - })) - defer srv.Close() - - ctx, cancel := context.WithCancel(context.Background()) - s := &Server{cfg: config{internalToken: "wrong-token"}} - done := make(chan struct{}) - go func() { - s.heartbeat(ctx, srv.URL, []byte("{}"), "host", internalTokenAuth("wrong-token")) - close(done) - }() - - deadline := time.Now().Add(5 * time.Second) - for !strings.Contains(buf.String(), "rejected") { - if time.Now().After(deadline) { - cancel() - <-done - t.Fatalf("no rejection logged within timeout, got: %q", buf.String()) - } - time.Sleep(10 * time.Millisecond) - } - cancel() - <-done - - out := buf.String() - if strings.Contains(out, "registered with") { - t.Errorf("logged success on 401 response: %q", out) - } - if !strings.Contains(out, "401") { - t.Errorf("expected rejection log naming the status, got: %q", out) - } -} - // setEndpoint injects a single Codefly endpoint into the SDK's env-var snapshot // for the duration of the test, then restores the snapshot on cleanup. func setEndpoint(t *testing.T, key, addr string) { @@ -685,74 +381,82 @@ func setEndpoint(t *testing.T, key, addr string) { }) } -// TestLoadConfigResolvesRenamedGateway proves the gateway default follows the -// saas-starter auth-sidecar → auth-gateway rename (v0.0.49): loadConfig resolves -// the gateway URL from the SDK against either service name, with no explicit -// CODEFLY_HOST_GATEWAY or GATEWAY_URL override — so a solution boots against both -// the renamed host and older ones. -func TestLoadConfigResolvesRenamedGateway(t *testing.T) { - const addr = "http://gateway:42152" - cases := []struct { - name string - service string - }{ - {"auth-gateway (v0.0.49+)", "auth-gateway"}, - {"auth-sidecar (pre-v0.0.49 fallback)", "auth-sidecar"}, - } - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - key := "CODEFLY__ENDPOINT__SAAS__" + - strings.ToUpper(strings.ReplaceAll(tc.service, "-", "_")) + "__REST__REST" - setEndpoint(t, key, addr) +// TestOnlyTheCurrentGatewayRoleResolves reverses an acceptance test this PR +// deleted the behaviour of. +// +// Gateway resolution used to fall back to the pre-v0.0.49 `auth-sidecar` role +// when the current one did not resolve, and a test asserted a solution booted +// against either host version. That is a compatibility path: in this cutover an +// unresolved current gateway silently selecting a legacy service is worse than a +// boot that fails naming the role, because the legacy service is the one that +// served the registration endpoints being deleted. +func TestOnlyTheCurrentGatewayRoleResolves(t *testing.T) { + const addr = "https://gateway:42152" + t.Run("the current role resolves", func(t *testing.T) { + setEndpoint(t, "CODEFLY__ENDPOINT__SAAS__AUTH_GATEWAY__REST__REST", addr) + cfg := loadConfig(context.Background(), testSolutionID, nil) + if cfg.gatewayURL != addr { + t.Fatalf("gatewayURL = %q, want %q resolved from the current role without an override", cfg.gatewayURL, addr) + } + // And the mint URL is NOT derived from it. A resolved gateway says + // where the gateway is, not where the host mints a credential: that + // endpoint is unsettled and undeployed, and the address this runtime + // POSTs its projected service-account token to is not one to assume. + if cfg.mintURL != "" { + t.Errorf("mintURL = %q, want empty: a resolved gateway must not produce a mint address, because the attestation this runtime sends there would be going somewhere nobody published", cfg.mintURL) + } + }) - cfg := loadConfig(context.Background(), "lastlogin-go") - if cfg.gatewayURL != addr { - t.Fatalf("gatewayURL = %q, want %q resolved from %s without an override", cfg.gatewayURL, addr, tc.service) - } - // The gateway module-register URL defaults to the resolved gateway plus - // the /modules/_register path, with no explicit override. - if want := addr + "/modules/_register"; cfg.moduleRegisterURL != want { - t.Errorf("moduleRegisterURL = %q, want %q derived from the resolved gateway", cfg.moduleRegisterURL, want) - } - }) - } + t.Run("the legacy role resolves to nothing", func(t *testing.T) { + // Hermetic: no workspace on disk, so only the injected carriers drive + // resolution and the only role present is the legacy one. + t.Chdir(t.TempDir()) + setEndpoint(t, "CODEFLY__ENDPOINT__SAAS__AUTH_SIDECAR__REST__REST", addr) + if got := resolveGateway(context.Background(), "", "auth-gateway"); got != "" { + t.Fatalf("resolveGateway = %q, want %q: a host exposing only the pre-rename role is one to re-render, and the boot must fail naming the role rather than selecting a legacy service", got, "") + } + }) } -// TestLoadConfigResolvesHostByRole proves the host gateway/frontend resolve by -// service role alone, independent of the host module's workspace name: loadConfig -// resolves both URLs with no CODEFLY_HOST_MODULE override whether the host module -// is the current saas (auth-gateway service), the pre-rename saas-starter +// TestLoadConfigResolvesHostByRole proves the host gateway resolves by service +// role alone, independent of the host module's workspace name: loadConfig +// resolves it with no CODEFLY_HOST_MODULE override whether the host module is +// the current host module (auth-gateway service), a pre-rename one // (auth-sidecar), or any other name a solution composes it under (codefly-dev/core#382). func TestLoadConfigResolvesHostByRole(t *testing.T) { - const ( - gatewayAddr = "http://gateway:42152" - frontendAddr = "http://frontend:42153" - ) + const gatewayAddr = "https://gateway:42152" cases := []struct { name string module string gateway string }{ - {"saas host (post-rename)", "SAAS", "AUTH_GATEWAY"}, - {"saas-starter host (pre-rename)", "SAAS_STARTER", "AUTH_SIDECAR"}, - {"arbitrary host module name", "SOME_OTHER_HOST", "AUTH_GATEWAY"}, + {"a host module named saas", "SAAS", "AUTH_GATEWAY"}, + {"a host module under a pre-rename name", "SAAS_STARTER", "AUTH_GATEWAY"}, + {"any other name a solution composes it under", "SOME_OTHER_HOST", "AUTH_GATEWAY"}, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { setEndpoint(t, "CODEFLY__ENDPOINT__"+tc.module+"__"+tc.gateway+"__REST__REST", gatewayAddr) - setEndpoint(t, "CODEFLY__ENDPOINT__"+tc.module+"__FRONTEND__HTTP__HTTP", frontendAddr) - cfg := loadConfig(context.Background(), "lastlogin-go") + cfg := loadConfig(context.Background(), testSolutionID, nil) if cfg.gatewayURL != gatewayAddr { t.Errorf("gatewayURL = %q, want %q resolved without a CODEFLY_HOST_MODULE override", cfg.gatewayURL, gatewayAddr) } - if want := frontendAddr + "/api/solutions/register"; cfg.hostRegisterURL != want { - t.Errorf("hostRegisterURL = %q, want %q resolved without a CODEFLY_HOST_MODULE override", cfg.hostRegisterURL, want) - } }) } } +// identitiesFile writes an admission set to a file and returns its path, which +// is the form both admission sets are provisioned in: they are read per +// handshake and per dial, so they have to be something this process can re-read +// rather than a value fixed when it started. +func identitiesFile(t *testing.T, identities ...string) string { + t.Helper() + path := filepath.Join(t.TempDir(), "identities") + writeFile(t, path, strings.Join(identities, "\n")+"\n") + return path +} + // writeFile writes content to path, creating parent directories, for building // on-disk workspace fixtures. func writeFile(t *testing.T, path, content string) { @@ -776,15 +480,15 @@ func TestResolveGatewayAmbiguousModuleFailsLoud(t *testing.T) { t.Chdir(t.TempDir()) t.Run("single owning module resolves", func(t *testing.T) { - setEndpoint(t, "CODEFLY__ENDPOINT__SAAS__AUTH_GATEWAY__REST__REST", "http://gateway:42152") - if got := resolveGateway(context.Background(), "", "auth-gateway"); got != "http://gateway:42152" { + setEndpoint(t, "CODEFLY__ENDPOINT__SAAS__AUTH_GATEWAY__REST__REST", "https://gateway:42152") + if got := resolveGateway(context.Background(), "", "auth-gateway"); got != "https://gateway:42152" { t.Fatalf("resolveGateway = %q, want the single module's address", got) } }) t.Run("two modules owning the same role are ambiguous", func(t *testing.T) { - setEndpoint(t, "CODEFLY__ENDPOINT__SAAS__AUTH_GATEWAY__REST__REST", "http://gateway-a:42152") - setEndpoint(t, "CODEFLY__ENDPOINT__SAAS_STARTER__AUTH_GATEWAY__REST__REST", "http://gateway-b:42152") + setEndpoint(t, "CODEFLY__ENDPOINT__SAAS__AUTH_GATEWAY__REST__REST", "https://gateway-a:42152") + setEndpoint(t, "CODEFLY__ENDPOINT__SAAS_STARTER__AUTH_GATEWAY__REST__REST", "https://gateway-b:42152") if got := resolveGateway(context.Background(), "", "auth-gateway"); got != "" { t.Fatalf("resolveGateway = %q, want %q so validate() fails loud on the ambiguous host", got, "") } @@ -810,953 +514,46 @@ modules: name: platform services: - name: auth-gateway - - name: frontend `) writeFile(t, filepath.Join(root, "modules", "platform", "services", "auth-gateway", "service.codefly.yaml"), `kind: service name: auth-gateway version: 0.0.0 agent: - kind: codefly:service - name: go - version: 0.0.1 - publisher: codefly.dev -endpoints: - - name: rest -`) - writeFile(t, filepath.Join(root, "modules", "platform", "services", "frontend", "service.codefly.yaml"), `kind: service -name: frontend -version: 0.0.0 -agent: - kind: codefly:service - name: go - version: 0.0.1 - publisher: codefly.dev -endpoints: - - name: http -`) - - // Force the SDK's local-native resolution (no injected carriers), and restore - // the shared env snapshot afterwards so later tests see a clean state. - t.Setenv("CODEFLY__ENVIRONMENT", "") - if err := codefly.LoadEnvironmentVariables(); err != nil { - t.Fatal(err) - } - t.Cleanup(func() { _ = codefly.LoadEnvironmentVariables() }) - t.Chdir(root) - - // Resolving by role alone must match resolving with the module named, and - // both must actually resolve (non-empty) from the workspace map. - gotGW := resolveGateway(ctx, "", "auth-gateway") - wantGW := resolveGateway(ctx, "platform", "auth-gateway") - if wantGW == "" { - t.Fatal("resolveGateway with explicit module resolved empty; the workspace fixture did not expose the gateway endpoint") - } - if gotGW != wantGW { - t.Errorf("resolveGateway(module=\"\") = %q, want %q discovered from the workspace", gotGW, wantGW) - } - - gotFE := resolveFrontend(ctx, "", "frontend") - wantFE := resolveFrontend(ctx, "platform", "frontend") - if wantFE == "" { - t.Fatal("resolveFrontend with explicit module resolved empty; the workspace fixture did not expose the frontend endpoint") - } - if gotFE != wantFE { - t.Errorf("resolveFrontend(module=\"\") = %q, want %q discovered from the workspace", gotFE, wantFE) - } -} - -// moduleRegistration is the wire payload the gateway's /modules/_register -// accepts: a bare single-segment prefix (the gateway builds /v1//* from -// it) and the resolved upstream. Registration and Internal are not part of the -// body — they are the captured credential headers, so a test can assert the -// registration presents the signed, prefix-bound token and not the shared -// cluster-internal one. -type moduleRegistration struct { - Prefix string `json:"prefix"` - Upstream string `json:"upstream"` - Registration string `json:"-"` - Internal string `json:"-"` -} - -// moduleExchange is what the credential exchange (/modules/_registration-token) -// received: the prefix a module asks for, plus the two credentials the gateway -// requires — its own perimeter token and the module's registration secret. -type moduleExchange struct { - Prefix string `json:"prefix"` - Secret string `json:"-"` - Internal string `json:"-"` -} - -// fakeGateway stands in for the host gateway on the two endpoints module -// federation uses. It mirrors the real refusal semantics: the exchange mints a -// token only for a prefix whose declared secret the caller presents, and answers -// 401 otherwise (accounts' refusal, relayed). -type fakeGateway struct { - *httptest.Server - registrations chan moduleRegistration - exchanges chan moduleExchange - - // declared maps a prefix to the secret the composition provisioned for it — - // the plaintext twin of the digest accounts holds. - declared map[string]string - // tokenTTL is how long a minted token is claimed to be valid. - tokenTTL time.Duration - // registerStatus, when non-zero, is what /modules/_register answers instead - // of 200. - registerStatus int - - mu sync.Mutex - // registerRefusals, while positive, makes /modules/_register answer 401 and - // counts down — a gateway that has stopped honouring the token it was handed - // and accepts the next one. - registerRefusals int - minted int -} - -func newFakeGateway(t *testing.T, gw *fakeGateway) *fakeGateway { - t.Helper() - gw.registrations = make(chan moduleRegistration, 8) - gw.exchanges = make(chan moduleExchange, 8) - if gw.tokenTTL == 0 { - gw.tokenTTL = time.Hour - } - gw.Server = httptest.NewServer(http.HandlerFunc(gw.serve)) - t.Cleanup(gw.Close) - return gw -} - -func (g *fakeGateway) serve(w http.ResponseWriter, r *http.Request) { - switch r.URL.Path { - case moduleRegistrationTokenPath: - var exchange moduleExchange - _ = json.NewDecoder(r.Body).Decode(&exchange) - exchange.Secret = r.Header.Get(moduleSecretHeader) - exchange.Internal = r.Header.Get(internalTokenHeader) - send(g.exchanges, exchange) - if g.declared[exchange.Prefix] == "" || g.declared[exchange.Prefix] != exchange.Secret { - w.WriteHeader(http.StatusUnauthorized) - return - } - g.mu.Lock() - g.minted++ - token := fmt.Sprintf("minted-%s-%d", exchange.Prefix, g.minted) - g.mu.Unlock() - writeJSON(w, http.StatusOK, map[string]string{ - "token": token, - "expiresAt": time.Now().Add(g.tokenTTL).UTC().Format(time.RFC3339), - }) - case moduleRegisterPath: - var registration moduleRegistration - _ = json.NewDecoder(r.Body).Decode(®istration) - registration.Registration = r.Header.Get(moduleRegistrationHeader) - registration.Internal = r.Header.Get(internalTokenHeader) - send(g.registrations, registration) - g.mu.Lock() - refusing := g.registerRefusals > 0 - if refusing { - g.registerRefusals-- - } - g.mu.Unlock() - if refusing { - w.WriteHeader(http.StatusUnauthorized) - return - } - if g.registerStatus != 0 { - w.WriteHeader(g.registerStatus) - return - } - w.WriteHeader(http.StatusOK) - default: - w.WriteHeader(http.StatusOK) - } -} - -func (g *fakeGateway) mintCount() int { - g.mu.Lock() - defer g.mu.Unlock() - return g.minted -} - -// send records an observation without ever blocking the fake gateway's handler: -// a heartbeat re-POSTs forever, so a full channel must not wedge the server. -func send[T any](ch chan T, value T) { - select { - case ch <- value: - default: - } -} - -// bootWithModuleRegistry boots a real solution on ln against gw. The host and -// gateway self-registrations point at the same fake so their heartbeats don't -// spew transport errors into the test log. -func bootWithModuleRegistry(t *testing.T, ln net.Listener, gw *fakeGateway) func() { - t.Helper() - s := New(Manifest{ID: "lastlogin-go", Title: "Last Login"}) - s.cfg = config{ - port: strconv.Itoa(ln.Addr().(*net.TCPAddr).Port), - publicURL: "http://127.0.0.1", - hostRegisterURL: gw.URL + "/host", - gatewayRegisterURL: gw.URL + "/gateway", - moduleRegisterURL: gw.URL + moduleRegisterPath, - moduleTokenURL: gw.URL + moduleRegistrationTokenPath, - internalToken: internalTokenTest, - moduleSecrets: parseModuleRegistrationSecrets(os.Getenv(ModuleRegistrationSecretsEnvironmentVariable)), - } - - ctx, cancel := context.WithCancel(context.Background()) - done := make(chan struct{}) - go func() { _ = s.serve(ctx, ln); close(done) }() - return func() { - cancel() - <-done - } -} - -// internalTokenTest is the cluster-internal token the fake gateway's perimeter -// check expects on the credential exchange. -const internalTokenTest = "internal-token-xyz" - -// consumesDocuments is the api.consumes projection core surfaces to a running -// backend for the wiki→documents federation. The literal key is -// manifest.APIConsumesEnvironmentVariable (a wire contract). -const consumesDocuments = `[{"id":"docstore.documents","module":"docstore","service":"documents","endpoint":"rest","protocol":"rest","as":"documents"}]` - -// TestServeRegistersConsumedAPIUpstreams proves the api.consumes federation end -// to end on the runtime's side: for each consumed target core surfaces in -// CODEFLY__API_CONSUMES, the runtime exchanges the registration secret its -// composition provisioned for a signed, prefix-bound token, then registers -// /v1/ → the SDK-resolved upstream with that token. The shared -// cluster-internal token authenticates only the exchange — the gateway rejects -// it on /modules/_register, so it must not be what the registration presents. -func TestServeRegistersConsumedAPIUpstreams(t *testing.T) { - const upstream = "http://docstore-upstream:9100" - // The consumed endpoint's address is injected because the backend depends on - // the consumed service; resolve it via the SDK snapshot. - setEndpoint(t, "CODEFLY__ENDPOINT__DOCSTORE__DOCUMENTS__REST__REST", upstream) - t.Setenv("CODEFLY__API_CONSUMES", consumesDocuments) - t.Setenv(ModuleRegistrationSecretsEnvironmentVariable, "documents:s3cret") - - gw := newFakeGateway(t, &fakeGateway{declared: map[string]string{"documents": "s3cret"}}) - - ln, err := net.Listen("tcp", "127.0.0.1:0") - if err != nil { - t.Fatal(err) - } - defer bootWithModuleRegistry(t, ln, gw)() - - select { - case exchange := <-gw.exchanges: - if exchange.Prefix != "documents" { - t.Errorf("exchanged for prefix %q, want %q", exchange.Prefix, "documents") - } - if exchange.Secret != "s3cret" { - t.Errorf("exchange presented secret %q, want the provisioned %q", exchange.Secret, "s3cret") - } - if exchange.Internal != internalTokenTest { - t.Errorf("exchange presented internal token %q, want %q — the gateway's perimeter check requires it", exchange.Internal, internalTokenTest) - } - case <-time.After(5 * time.Second): - t.Fatal("gateway did not receive a registration-token exchange within timeout") - } - - select { - case reg := <-gw.registrations: - if reg.Prefix != "documents" { - t.Errorf("registered prefix = %q, want %q (the facade as core projected)", reg.Prefix, "documents") - } - if reg.Upstream != upstream { - t.Errorf("registered upstream = %q, want %q resolved via the SDK", reg.Upstream, upstream) - } - if reg.Registration != "minted-documents-1" { - t.Errorf("registered with %s = %q, want the minted token", moduleRegistrationHeader, reg.Registration) - } - if reg.Internal != "" { - t.Errorf("registration carried the shared internal token %q; the gateway refuses it and it must not leak to this path", reg.Internal) - } - case <-time.After(5 * time.Second): - t.Fatal("gateway did not receive a module registration within timeout") - } -} - -// TestServeSkipsConsumedAPIWithoutSecret proves the provisioning gap fails loud -// rather than hammering the gateway: with no secret for the prefix there is no -// credential to exchange, so every registration would be a guaranteed 401. The -// runtime neither exchanges nor registers, and says which variable is missing. -func TestServeSkipsConsumedAPIWithoutSecret(t *testing.T) { - buf := &syncBuffer{} - log.SetOutput(buf) - defer log.SetOutput(os.Stderr) - - setEndpoint(t, "CODEFLY__ENDPOINT__DOCSTORE__DOCUMENTS__REST__REST", "http://docstore-upstream:9100") - t.Setenv("CODEFLY__API_CONSUMES", consumesDocuments) - t.Setenv(ModuleRegistrationSecretsEnvironmentVariable, "") - - gw := newFakeGateway(t, &fakeGateway{}) - - ln, err := net.Listen("tcp", "127.0.0.1:0") - if err != nil { - t.Fatal(err) - } - defer bootWithModuleRegistry(t, ln, gw)() - - select { - case exchange := <-gw.exchanges: - t.Fatalf("exchanged a registration token for %q with no provisioned secret", exchange.Prefix) - case reg := <-gw.registrations: - t.Fatalf("registered module %q → %q with no provisioned secret", reg.Prefix, reg.Upstream) - case <-time.After(500 * time.Millisecond): - } - - if out := buf.String(); !strings.Contains(out, ModuleRegistrationSecretsEnvironmentVariable) { - t.Errorf("missing secret was not reported against %s, got: %q", ModuleRegistrationSecretsEnvironmentVariable, out) - } -} - -// TestServeIsolatesFailedModuleRegistration proves a per-module failure stays -// per-module: the composition provisions a secret for one consumed module and a -// stale one for another, so the first federates while the second's exchange is -// refused. The refused module must not take down the backend or the sibling -// registration — it just retries on its own heartbeat. -func TestServeIsolatesFailedModuleRegistration(t *testing.T) { - setEndpoint(t, "CODEFLY__ENDPOINT__DOCSTORE__DOCUMENTS__REST__REST", "http://docstore-upstream:9100") - setEndpoint(t, "CODEFLY__ENDPOINT__BILLING__INVOICES__REST__REST", "http://billing-upstream:9200") - t.Setenv("CODEFLY__API_CONSUMES", `[`+ - `{"id":"billing.invoices","module":"billing","service":"invoices","endpoint":"rest","protocol":"rest","as":"billing"},`+ - `{"id":"docstore.documents","module":"docstore","service":"documents","endpoint":"rest","protocol":"rest","as":"documents"}]`) - t.Setenv(ModuleRegistrationSecretsEnvironmentVariable, "documents:s3cret,billing:stale") - - gw := newFakeGateway(t, &fakeGateway{declared: map[string]string{ - "documents": "s3cret", - "billing": "rotated", - }}) - - ln, err := net.Listen("tcp", "127.0.0.1:0") - if err != nil { - t.Fatal(err) - } - defer bootWithModuleRegistry(t, ln, gw)() - - select { - case reg := <-gw.registrations: - if reg.Prefix != "documents" { - t.Errorf("registered prefix = %q, want only %q to reach registration", reg.Prefix, "documents") - } - case <-time.After(5 * time.Second): - t.Fatal("the module with a valid secret did not register within timeout") - } - - // The backend is still serving: the refused module's failure did not take it - // down with it. - resp, err := http.Get("http://127.0.0.1:" + strconv.Itoa(ln.Addr().(*net.TCPAddr).Port) + "/health") - if err != nil { - t.Fatalf("backend stopped serving after a module registration failure: %v", err) - } - resp.Body.Close() - if resp.StatusCode != http.StatusOK { - t.Errorf("health = %d, want 200", resp.StatusCode) - } -} - -// TestServeSkipsConsumedAPIWithoutFacade proves the runtime never guesses a -// facade prefix. Core derives an omitted `as` from the producing endpoint's -// proto package — a value the runtime cannot reconstruct from the projected -// identity — so an entry that arrives with no `as` is skipped rather than -// registered under a fabricated prefix (e.g. the module name), which would proxy -// a route the generated client never calls and could steal another facade's -// prefix. The consumed endpoint resolves fine; only the missing `as` suppresses -// registration. -func TestServeSkipsConsumedAPIWithoutFacade(t *testing.T) { - setEndpoint(t, "CODEFLY__ENDPOINT__DOCSTORE__DOCUMENTS__REST__REST", "http://docstore-upstream:9100") - t.Setenv("CODEFLY__API_CONSUMES", - `[{"id":"docstore.documents","module":"docstore","service":"documents","endpoint":"rest","protocol":"rest"}]`) - t.Setenv(ModuleRegistrationSecretsEnvironmentVariable, "documents:s3cret") - - gw := newFakeGateway(t, &fakeGateway{declared: map[string]string{"documents": "s3cret"}}) - - ln, err := net.Listen("tcp", "127.0.0.1:0") - if err != nil { - t.Fatal(err) - } - defer bootWithModuleRegistry(t, ln, gw)() - - select { - case reg := <-gw.registrations: - t.Fatalf("registered module %q → %q for a consumed api with no facade entry-point (as); the runtime must not invent a prefix", reg.Prefix, reg.Upstream) - case <-time.After(500 * time.Millisecond): - // No registration, as expected. - } -} - -// TestServeRegistersNoModulesWithoutConsumes proves the no-op: a solution that -// declares no api.consumes (empty CODEFLY__API_CONSUMES) registers no module -// upstream at all, so nothing changes for the solutions that consume nothing. -func TestServeRegistersNoModulesWithoutConsumes(t *testing.T) { - t.Setenv("CODEFLY__API_CONSUMES", "") - - gw := newFakeGateway(t, &fakeGateway{}) - - ln, err := net.Listen("tcp", "127.0.0.1:0") - if err != nil { - t.Fatal(err) - } - defer bootWithModuleRegistry(t, ln, gw)() - - select { - case reg := <-gw.registrations: - t.Fatalf("registered module %q → %q for a solution that declares no api.consumes", reg.Prefix, reg.Upstream) - case <-time.After(500 * time.Millisecond): - // No registration, as expected. - } -} - -// TestModuleCredentialReusesTokenUntilRenewal proves the beat does not re-mint -// every 15 seconds: a registration token lives 5 minutes, and each mint is an -// audited security event on accounts, so a token still comfortably inside its -// lifetime is reused. -func TestModuleCredentialReusesTokenUntilRenewal(t *testing.T) { - gw := newFakeGateway(t, &fakeGateway{declared: map[string]string{"documents": "s3cret"}}) - credential := &moduleCredential{ - tokenURL: gw.URL + moduleRegistrationTokenPath, - internalToken: internalTokenTest, - prefix: "documents", - secret: "s3cret", - } - - for beat := range 3 { - req, err := http.NewRequest(http.MethodPost, gw.URL+moduleRegisterPath, nil) - if err != nil { - t.Fatal(err) - } - if _, err := credential.authorize(context.Background(), req); err != nil { - t.Fatalf("beat %d: authorize: %v", beat, err) - } - if got := req.Header.Get(moduleRegistrationHeader); got != "minted-documents-1" { - t.Errorf("beat %d presented %q, want the first minted token reused", beat, got) - } - } - if got := gw.mintCount(); got != 1 { - t.Errorf("minted %d tokens across 3 beats, want 1 — a live token must be reused", got) - } -} - -// TestModuleCredentialReExchangesAfterRejection proves recovery from a refused -// registration: the gateway answering 401 means the token it was handed may no -// longer be honoured (a restarted gateway, a rotated key), so replaying it for -// the rest of its lifetime would strand the module. A refusal of a token held -// from an earlier beat forces a fresh exchange on the next one. -func TestModuleCredentialReExchangesAfterRejection(t *testing.T) { - gw := newFakeGateway(t, &fakeGateway{declared: map[string]string{"documents": "s3cret"}}) - credential := &moduleCredential{ - tokenURL: gw.URL + moduleRegistrationTokenPath, - internalToken: internalTokenTest, - prefix: "documents", - secret: "s3cret", - } - - req, err := http.NewRequest(http.MethodPost, gw.URL+moduleRegisterPath, nil) - if err != nil { - t.Fatal(err) - } - if _, err := credential.authorize(context.Background(), req); err != nil { - t.Fatal(err) - } - // A later beat presents the held token; that beat's refusal is the one that - // can be blamed on staleness, so it drops the credential. - if _, err := credential.authorize(context.Background(), req); err != nil { - t.Fatal(err) - } - credential.invalidate() - if _, err := credential.authorize(context.Background(), req); err != nil { - t.Fatal(err) - } - - if got := req.Header.Get(moduleRegistrationHeader); got != "minted-documents-2" { - t.Errorf("presented %q after invalidation, want a freshly minted token", got) - } - if got := gw.mintCount(); got != 2 { - t.Errorf("minted %d tokens, want 2 — a refused credential must be re-exchanged", got) - } -} - -// TestModuleCredentialKeepsATokenMintedForThisBeat proves the one refusal a -// fresh mint cannot fix is not answered with another mint. A gateway that -// refuses a token minted moments ago is refusing it for a reason that has -// nothing to do with staleness — it does not trust the issuer, the prefix is not -// this module's to claim, its own clock is skewed, or a perimeter check rejected -// the request before the token was ever read. Dropping the credential there put -// a mint on every single beat: an audited security event on the issuer four -// times a minute per module, indefinitely, and silent after the first log line -// because the status never changes. -func TestModuleCredentialKeepsATokenMintedForThisBeat(t *testing.T) { - log.SetOutput(io.Discard) - defer log.SetOutput(os.Stderr) - - gw := newFakeGateway(t, &fakeGateway{declared: map[string]string{"documents": "s3cret"}}) - credential := &moduleCredential{ - tokenURL: gw.URL + moduleRegistrationTokenPath, - internalToken: internalTokenTest, - prefix: "documents", - secret: "s3cret", - } - req, err := http.NewRequest(http.MethodPost, gw.URL+moduleRegisterPath, nil) - if err != nil { - t.Fatal(err) - } - if _, err := credential.authorize(context.Background(), req); err != nil { - t.Fatal(err) - } - credential.invalidate() // refused the token this beat just minted - if credential.token == "" { - t.Fatal("dropped a token minted for this very beat; re-minting cannot fix a refusal that was never about staleness") - } - if _, err := credential.authorize(context.Background(), req); err != nil { - t.Fatal(err) - } - if got := gw.mintCount(); got != 1 { - t.Errorf("minted %d tokens, want 1 — a refusal of a just-minted token must not buy another mint", got) - } -} - -// TestHeartbeatReExchangesOnRejectedRegistration proves the invalidation is -// actually wired to the beat loop, not just available on the credential: a -// gateway that answers 401 once — having stopped honouring the token it was -// handed — makes the next beat present a freshly exchanged one. -func TestHeartbeatReExchangesOnRejectedRegistration(t *testing.T) { - log.SetOutput(io.Discard) - defer log.SetOutput(os.Stderr) - - // Refuse the first two registrations. The first presents a token minted for - // that same beat, which re-minting cannot fix; the second presents that - // token held over from the earlier beat, and that is the refusal staleness - // explains — so it must be answered with a fresh exchange. - gw := newFakeGateway(t, &fakeGateway{ - declared: map[string]string{"documents": "s3cret"}, - registerRefusals: 2, - }) - credential := &moduleCredential{ - tokenURL: gw.URL + moduleRegistrationTokenPath, - internalToken: internalTokenTest, - prefix: "documents", - secret: "s3cret", - } - - ctx, cancel := context.WithCancel(context.Background()) - s := &Server{manifest: Manifest{ID: "lastlogin-go"}, registrationInterval: time.Millisecond} - done := make(chan struct{}) - go func() { - s.heartbeat(ctx, gw.URL+moduleRegisterPath, []byte(`{"prefix":"documents"}`), "gateway module documents", credential) - close(done) - }() - defer func() { cancel(); <-done }() - - first, deadline := "", time.After(10*time.Second) - for { - select { - case reg := <-gw.registrations: - switch { - case reg.Registration == "": - t.Fatal("registered with no token") - case first == "": - first = reg.Registration - case reg.Registration != first: - return // recovered: a fresh credential replaced the stale one - } - case <-deadline: - t.Fatalf("kept replaying %q: a token refused after being held across beats must be re-exchanged", first) - } - } -} - -// TestHeartbeatDoesNotMintPerBeatOnAPersistentRefusal is the regression guard -// for the mint storm. A gateway that refuses every registration does so for a -// reason no new token can repair, and the old loop answered each refusal with a -// fresh exchange: at the production 15s beat that is four mints a minute per -// module, forever, each one an audited security event on the issuer — and -// silent after the first log line, because the status never changes. -// -// The refusal is now worth exactly one re-mint, and failing beats back off, so -// the mint count stays flat however long the gateway stays broken. -func TestHeartbeatDoesNotMintPerBeatOnAPersistentRefusal(t *testing.T) { - log.SetOutput(io.Discard) - defer log.SetOutput(os.Stderr) - - gw := newFakeGateway(t, &fakeGateway{ - declared: map[string]string{"documents": "s3cret"}, - registerStatus: http.StatusUnauthorized, - }) - credential := &moduleCredential{ - tokenURL: gw.URL + moduleRegistrationTokenPath, - internalToken: internalTokenTest, - prefix: "documents", - secret: "s3cret", - } - - ctx, cancel := context.WithCancel(context.Background()) - s := &Server{manifest: Manifest{ID: "lastlogin-go"}, registrationInterval: time.Millisecond} - done := make(chan struct{}) - go func() { - s.heartbeat(ctx, gw.URL+moduleRegisterPath, []byte(`{"prefix":"documents"}`), "gateway module documents", credential) - close(done) - }() - defer func() { cancel(); <-done }() - - const beats = 5 - for range beats { - select { - case <-gw.registrations: - case <-time.After(10 * time.Second): - t.Fatalf("gateway saw fewer than %d registration attempts", beats) - } - } - // One mint to obtain the credential, at most one more to answer the refusal. - if got := gw.mintCount(); got > 2 { - t.Errorf("minted %d tokens across %d refused beats, want at most 2 — a refusal that a new token cannot fix must not re-mint on every beat", got, beats) - } -} - -// TestModuleCredentialRetriesAgainAfterRecovering proves a spent retry budget is -// restored by a successful registration, so a second, independent refusal — a -// gateway rotating its key twice inside one credential lifetime — is answered -// with a fresh token instead of inheriting the first refusal's spent attempt and -// stranding the module until the token renews on its own. -func TestModuleCredentialRetriesAgainAfterRecovering(t *testing.T) { - gw := newFakeGateway(t, &fakeGateway{declared: map[string]string{"documents": "s3cret"}}) - credential := &moduleCredential{ - tokenURL: gw.URL + moduleRegistrationTokenPath, - internalToken: internalTokenTest, - prefix: "documents", - secret: "s3cret", - } - req, err := http.NewRequest(http.MethodPost, gw.URL+moduleRegisterPath, nil) - if err != nil { - t.Fatal(err) - } - authorize := func() { - t.Helper() - if _, err := credential.authorize(context.Background(), req); err != nil { - t.Fatal(err) - } - } - authorize() // beat 1: mint - authorize() // beat 2: replay the held token - credential.invalidate() // refused: spends the retry, drops - authorize() // beat 3: mint the replacement - credential.succeeded() // ... which the gateway accepts - - authorize() // beat 4: replay - credential.invalidate() // a second, independent refusal - if credential.token != "" { - t.Fatal("kept a token refused after an intervening success: a new fault must earn its own re-mint") - } - authorize() - if got := gw.mintCount(); got != 3 { - t.Errorf("minted %d tokens, want 3 — one per refusal episode plus the original", got) - } -} - -// TestHeartbeatDropsACredentialRefusedWith403 proves 403 recovers like 401. A -// gateway answering "forbidden" to a lapsed token would otherwise have that -// token replayed until it renewed naturally, stranding the module for most of -// the credential's lifetime. -func TestHeartbeatDropsACredentialRefusedWith403(t *testing.T) { - credential := &moduleCredential{prefix: "documents", token: "held", renewAt: time.Now().Add(time.Hour)} - // A token held from an earlier beat: this beat did not mint it. - credential.invalidate() - if credential.token != "" { - t.Fatal("a refused credential held from an earlier beat must be dropped") - } - - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - w.WriteHeader(http.StatusForbidden) - })) - defer srv.Close() - if status := statusOfOneBeat(t, srv.URL); status != http.StatusForbidden { - t.Fatalf("beat status = %d, want 403", status) - } -} - -// statusOfOneBeat runs a single beat against target and returns its status. -func statusOfOneBeat(t *testing.T, target string) int { - t.Helper() - s := &Server{manifest: Manifest{ID: "lastlogin-go"}} - status, _, err := s.beat(context.Background(), target, []byte(`{}`), internalTokenAuth("tok")) - if err != nil { - t.Fatal(err) - } - return status -} - -// TestExchangeRejectsAnUnusableExpiry proves the credential refuses an expiry -// it cannot act on instead of caching it as "already expired". Treated as -// expired it would re-run the exchange on every beat while every registration -// still returned 200 — an unlogged mint, and an audited security event on the -// issuer, four times a minute for as long as the solution runs. -func TestExchangeRejectsAnUnusableExpiry(t *testing.T) { - for _, tc := range []struct { - name string - body map[string]string - }{ - // A response whose expiry field never arrives (an unset timestamp on the - // issuer, or a rename to the protobuf JSON spelling) decodes to the zero - // time. - {name: "absent", body: map[string]string{"token": "t"}}, - {name: "epoch zero", body: map[string]string{"token": "t", "expiresAt": "1970-01-01T00:00:00Z"}}, - // Stands in for a host clock skewed past the credential's own lifetime. - {name: "already past", body: map[string]string{ - "token": "t", "expiresAt": time.Now().Add(-time.Minute).UTC().Format(time.RFC3339)}}, - // Below the minimum usable lifetime: it would lapse in flight. - {name: "lapses in flight", body: map[string]string{ - "token": "t", "expiresAt": time.Now().Add(2 * time.Second).UTC().Format(time.RFC3339)}}, - } { - t.Run(tc.name, func(t *testing.T) { - exchanges := 0 - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - exchanges++ - writeJSON(w, http.StatusOK, tc.body) - })) - defer srv.Close() - - credential := &moduleCredential{ - tokenURL: srv.URL, internalToken: internalTokenTest, - prefix: "documents", secret: "s3cret", - } - req, err := http.NewRequest(http.MethodPost, srv.URL, nil) - if err != nil { - t.Fatal(err) - } - if _, err := credential.authorize(context.Background(), req); err == nil { - t.Fatal("accepted a token the runtime cannot hold; a beat loop would re-mint forever") - } - // A second beat must not have cached anything either. - _, _ = credential.authorize(context.Background(), req) - if exchanges != 2 { - t.Errorf("exchanged %d times across 2 beats, want 2 attempts and 0 cached", exchanges) - } - if credential.token != "" { - t.Errorf("cached a token with an unusable expiry: %q", credential.token) - } - }) - } -} - -// TestRegistrationRequestsAreBounded is the regression guard for a wedged -// heartbeat. The beat's context carries no deadline, so without a client timeout -// a gateway that accepts a registration and never answers blocked the beat in -// Do forever: no further beats, no log, and no recovery short of a restart — -// for that module, or for this whole solution when it happened on one of the two -// self-registrations. Asserted on the client rather than by holding a real -// request open, so the guard costs no wall clock. -func TestRegistrationRequestsAreBounded(t *testing.T) { - if registrationClient.Timeout != registrationTimeout { - t.Errorf("registrationClient.Timeout = %s, want %s: a gateway that never answers must surface as a failed beat, not wedge the loop", - registrationClient.Timeout, registrationTimeout) - } - if registrationTimeout <= 0 { - t.Error("registrationTimeout must be positive") - } -} - -// TestExchangeOmitsAnUnconfiguredInternalToken proves the exchange sends no -// header at all rather than an empty one, matching internalTokenAuth. A caller -// claiming a credential it does not hold is the harder shape to diagnose at the -// gateway. -func TestExchangeOmitsAnUnconfiguredInternalToken(t *testing.T) { - seen := make(chan http.Header, 1) - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - seen <- r.Header.Clone() - writeJSON(w, http.StatusOK, map[string]string{ - "token": "t", "expiresAt": time.Now().Add(time.Hour).UTC().Format(time.RFC3339)}) - })) - defer srv.Close() - - credential := &moduleCredential{tokenURL: srv.URL, prefix: "documents", secret: "s3cret"} - req, err := http.NewRequest(http.MethodPost, srv.URL, nil) - if err != nil { - t.Fatal(err) - } - if _, err := credential.authorize(context.Background(), req); err != nil { - t.Fatal(err) - } - if _, ok := (<-seen)[http.CanonicalHeaderKey(internalTokenHeader)]; ok { - t.Errorf("sent %s with no token configured, want the header absent", internalTokenHeader) - } -} - -// TestExchangeAcceptsAShortButUsableExpiry proves the renewal lead is a ceiling -// on how early the credential renews, not a floor on what the issuer may issue. -// Conflating the two refused every token whose whole life was shorter than the -// 30s lead — a 30s credential was rejected outright, and the error blamed this -// host's clock for an issuer setting that was deliberate and fine. A short token -// is now held and renewed at half its life. -func TestExchangeAcceptsAShortButUsableExpiry(t *testing.T) { - log.SetOutput(io.Discard) - defer log.SetOutput(os.Stderr) - - for _, ttl := range []time.Duration{10 * time.Second, 30 * time.Second, 5 * time.Minute} { - t.Run(ttl.String(), func(t *testing.T) { - gw := newFakeGateway(t, &fakeGateway{ - declared: map[string]string{"documents": "s3cret"}, - tokenTTL: ttl, - }) - credential := &moduleCredential{ - tokenURL: gw.URL + moduleRegistrationTokenPath, - internalToken: internalTokenTest, - prefix: "documents", - secret: "s3cret", - } - req, err := http.NewRequest(http.MethodPost, gw.URL+moduleRegisterPath, nil) - if err != nil { - t.Fatal(err) - } - if _, err := credential.authorize(context.Background(), req); err != nil { - t.Fatalf("refused a usable %s credential: %v", ttl, err) - } - if req.Header.Get(moduleRegistrationHeader) == "" { - t.Error("presented no token") - } - // Renewal is at half-life for a short token, and a full lead ahead - // of expiry for a long one — never after expiry, and never so early - // that the token is discarded unused. - wantLead := min(moduleTokenRenewal, ttl/2) - held := time.Until(credential.renewAt) - if held <= 0 || held > ttl-wantLead+time.Second { - t.Errorf("renewAt is %s away for a %s credential, want about %s", held, ttl, ttl-wantLead) - } - }) - } -} - -// TestSiblingURLFollowsAnOverriddenGateway proves the two federation endpoints -// stay on one gateway. The credential a registration presents is minted by the -// exchange, so an explicitly overridden registration URL must carry the -// exchange with it — minting against one host and registering with another -// yields a token the second never trusts. -func TestSiblingURLFollowsAnOverriddenGateway(t *testing.T) { - t.Setenv("PORT", "8090") - t.Setenv("GATEWAY_URL", "http://gateway:42152") - t.Setenv("HOST_REGISTER_URL", "http://frontend:21931/api/solutions/register") - t.Setenv("GATEWAY_MODULE_REGISTER_URL", "https://other-gateway:9999/modules/_register") - - cfg := loadConfig(context.Background(), "lastlogin-go") - want := "https://other-gateway:9999" + moduleRegistrationTokenPath - if cfg.moduleTokenURL != want { - t.Errorf("module token URL = %q, want %q — it must follow the overridden register URL", cfg.moduleTokenURL, want) - } -} - -// An unparseable override must surface as itself, so validate() names the one -// URL the operator actually set instead of a second one derived from it. -func TestSiblingURLPassesThroughAnUnusableBase(t *testing.T) { - if got := siblingURL("/modules/_register", moduleRegisterPath, moduleRegistrationTokenPath); got != "/modules/_register" { - t.Errorf("siblingURL(relative) = %q, want the base handed back unchanged", got) - } -} - -// TestSiblingURLKeepsTheGatewayBasePath proves the derived endpoint stays on the -// gateway's mount, not just its host. Rebuilding the URL from scheme+host -// dropped any path between them, so a gateway served under a prefix registered -// at /gw/modules/_register while exchanging at /modules/_registration-token — -// still absolute, so validate() passed it, and a 404 on every beat after that. -func TestSiblingURLKeepsTheGatewayBasePath(t *testing.T) { - t.Setenv("PORT", "8090") - t.Setenv("GATEWAY_URL", "http://gateway:42152/gw") - t.Setenv("HOST_REGISTER_URL", "http://frontend:21931/api/solutions/register") - t.Setenv(SolutionRegistrationSecretEnvironmentVariable, "s3cret") - - cfg := loadConfig(context.Background(), "lastlogin-go") - if want := "http://gateway:42152/gw" + moduleRegisterPath; cfg.moduleRegisterURL != want { - t.Errorf("module register URL = %q, want %q", cfg.moduleRegisterURL, want) - } - if want := "http://gateway:42152/gw" + moduleRegistrationTokenPath; cfg.moduleTokenURL != want { - t.Errorf("module token URL = %q, want %q — the derived endpoint must keep the gateway's base path", cfg.moduleTokenURL, want) - } - if err := cfg.validate(); err != nil { - t.Errorf("validate() = %v, want nil", err) + kind: codefly:service + name: go + version: 0.0.1 + publisher: codefly.dev +endpoints: + - name: rest +`) + // Force the SDK's local-native resolution (no injected carriers), and restore + // the shared env snapshot afterwards so later tests see a clean state. + t.Setenv("CODEFLY__ENVIRONMENT", "") + if err := codefly.LoadEnvironmentVariables(); err != nil { + t.Fatal(err) } -} - -// TestSiblingURLRefusesAnUnpairableOverride proves an override whose sibling -// cannot be derived fails loud at boot rather than resolving to a plausible -// guess that 404s on every beat. The operator must then name the token URL. -func TestSiblingURLRefusesAnUnpairableOverride(t *testing.T) { - t.Setenv("PORT", "8090") - t.Setenv("GATEWAY_URL", "http://gateway:42152") - t.Setenv("HOST_REGISTER_URL", "http://frontend:21931/api/solutions/register") - t.Setenv("GATEWAY_MODULE_REGISTER_URL", "https://other-gateway:9999/custom/registration-endpoint") - t.Setenv(SolutionRegistrationSecretEnvironmentVariable, "s3cret") + t.Cleanup(func() { _ = codefly.LoadEnvironmentVariables() }) + t.Chdir(root) - cfg := loadConfig(context.Background(), "lastlogin-go") - if cfg.moduleTokenURL != "" { - t.Errorf("module token URL = %q, want empty so validate() names it", cfg.moduleTokenURL) - } - err := cfg.validate() - if err == nil || !strings.Contains(err.Error(), "module token URL") { - t.Errorf("validate() = %v, want an error naming the module token URL", err) + // Resolving by role alone must match resolving with the module named, and + // both must actually resolve (non-empty) from the workspace map. + gotGW := resolveGateway(ctx, "", "auth-gateway") + wantGW := resolveGateway(ctx, "platform", "auth-gateway") + if wantGW == "" { + t.Fatal("resolveGateway with explicit module resolved empty; the workspace fixture did not expose the gateway endpoint") } - - // Naming it explicitly is the documented way out. - t.Setenv("GATEWAY_MODULE_REGISTRATION_TOKEN_URL", "https://other-gateway:9999/custom/token-endpoint") - if err := loadConfig(context.Background(), "lastlogin-go").validate(); err != nil { - t.Errorf("validate() with an explicit token URL = %v, want nil", err) + if gotGW != wantGW { + t.Errorf("resolveGateway(module=\"\") = %q, want %q discovered from the workspace", gotGW, wantGW) } -} -// TestBackoffGrowsWhileBrokenAndResetsWhenHealthy pins the retry schedule that -// bounds every failure path on the credential exchange. -func TestBackoffGrowsWhileBrokenAndResetsWhenHealthy(t *testing.T) { - const interval = 15 * time.Second - for _, tc := range []struct { - failures int - want time.Duration - }{ - {0, interval}, - {1, 30 * time.Second}, - {2, time.Minute}, - {3, registrationBackoffCap}, - {50, registrationBackoffCap}, - } { - if got := backoff(interval, tc.failures, registrationBackoffCap); got != tc.want { - t.Errorf("backoff(%s, %d) = %s, want %s", interval, tc.failures, got, tc.want) - } - } - // An interval longer than the cap is the caller's choice, not something to - // shorten — at any number of failures, not only at zero. Shortening it would - // have a failing beat retry sooner than a healthy one. - for _, failures := range []int{0, 1, 50} { - if got := backoff(time.Hour, failures, registrationBackoffCap); got != time.Hour { - t.Errorf("backoff(1h, %d, cap) = %s, want 1h", failures, got) - } - } } -func TestParseModuleRegistrationSecrets(t *testing.T) { - tests := []struct { - name string - raw string - want map[string]string - }{ - {name: "empty", raw: "", want: map[string]string{}}, - {name: "one", raw: "documents:s3cret", want: map[string]string{"documents": "s3cret"}}, - { - name: "several, spaced", - raw: "documents:s3cret, billing:other", - want: map[string]string{"documents": "s3cret", "billing": "other"}, - }, - // A secret is opaque and base64 may contain "=" and "+"; only the first - // ":" separates it from the prefix. - {name: "secret keeps inner colons", raw: "documents:a:b", want: map[string]string{"documents": "a:b"}}, - // The registrar trims both halves of its digest twin; parsing the two - // asymmetrically turns a pair it accepts into a lookup miss here. - {name: "spaces around both halves", raw: "documents : s3cret", want: map[string]string{"documents": "s3cret"}}, - {name: "unpaired entry dropped", raw: "documents", want: map[string]string{}}, - {name: "empty secret dropped", raw: "documents:", want: map[string]string{}}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := parseModuleRegistrationSecrets(tt.raw); !reflect.DeepEqual(got, tt.want) { - t.Errorf("parseModuleRegistrationSecrets(%q) = %v, want %v", tt.raw, got, tt.want) - } - }) +// send records an observation without ever blocking a fake host's handler: a +// full channel must not wedge the server a test is driving. +func send[T any](ch chan T, value T) { + select { + case ch <- value: + default: } } @@ -1767,6 +564,7 @@ func TestParseModuleRegistrationSecrets(t *testing.T) { // is minted, and any capability that rode along, which none should. type mintRequest struct { OrgID string `json:"orgId"` + InstallationID string `json:"installationId"` TaskID string `json:"taskId"` SessionID string `json:"sessionId"` Audience string `json:"audience"` @@ -1803,6 +601,15 @@ type workContextGateway struct { // mintDelay holds each mint open, so concurrent asks genuinely overlap // rather than serialising by luck. mintDelay time.Duration + // supersedeFirstCall answers the first module call the way a far end + // answers a capability sealed to state it has moved past: 409, with the + // installation headers the carrier put beside it. + supersedeFirstCall bool + // conflictFirstCall answers the first module call with an ORDINARY 409 — + // a duplicate, a lost update — carrying an installation header that names + // somebody else's installation. It is the shape the supersession check + // used to accept, and it must retire nothing. + conflictFirstCall bool mu sync.Mutex minted int @@ -1828,7 +635,7 @@ func (g *workContextGateway) serve(w http.ResponseWriter, r *http.Request) { var mint mintRequest _ = json.NewDecoder(r.Body).Decode(&mint) mint.Bearer = r.Header.Get("authorization") - mint.WorkContext = r.Header.Get(codefly.WorkContextHeaderName) + mint.WorkContext = r.Header.Get(workcontext.HeaderName) send(g.mints, mint) if g.mintStatus != 0 { writeJSON(w, g.mintStatus, map[string]string{ @@ -1840,7 +647,7 @@ func (g *workContextGateway) serve(w http.ResponseWriter, r *http.Request) { g.mu.Lock() g.minted++ // Two segments: the wire shape sdk-go accepts for a signed capability. - token := fmt.Sprintf("context-%s.%d", mint.Audience, g.minted) + token := capability(fmt.Sprintf("context-%s.%d", mint.Audience, g.minted)) g.mu.Unlock() if g.mintDelay > 0 { time.Sleep(g.mintDelay) @@ -1853,13 +660,36 @@ func (g *workContextGateway) serve(w http.ResponseWriter, r *http.Request) { case modulePath: call := moduleCall{ Bearer: r.Header.Get("authorization"), - WorkContext: r.Header.Get(codefly.WorkContextHeaderName), + WorkContext: r.Header.Get(workcontext.HeaderName), } send(g.calls, call) if call.WorkContext == "" { writeJSON(w, http.StatusUnauthorized, map[string]string{"error": "unauthenticated"}) return } + g.mu.Lock() + supersede := g.supersedeFirstCall + g.supersedeFirstCall = false + conflict := g.conflictFirstCall + g.conflictFirstCall = false + g.mu.Unlock() + if conflict { + // An ordinary business conflict, carrying the SAME installation + // the capability is sealed to — which is what a real module + // answers with, because a module handling this viewer's call IS in + // their installation. A foreign id here is what hid the finding: + // it made comparing the installation look sufficient. + w.Header().Set(workcontext.InstallationIDHeaderName, corework.FixtureInstallation) + w.Header().Set(workcontext.InstallationRevisionHeaderName, "9") + writeJSON(w, http.StatusConflict, map[string]string{"error": "that page already exists"}) + return + } + if supersede { + w.Header().Set(workcontext.InstallationIDHeaderName, corework.FixtureInstallation) + w.Header().Set(workcontext.InstallationRevisionHeaderName, "4") + writeJSON(w, http.StatusConflict, map[string]string{"error": "installation revision superseded"}) + return + } writeJSON(w, http.StatusOK, map[string]string{"collection": "handbook"}) default: w.WriteHeader(http.StatusOK) @@ -1911,15 +741,49 @@ func lapseCachedCapabilities(gw *Gateway) { // serveHandler runs one solution handler behind the runtime's own wrapper, so a // test exercises the same Gateway a real request produces — including the // viewer identity the gateway injects, which wrap is the only place to read. +// serveHandler serves one handler against a fake gateway, with the execution +// credential a booted runtime holds: minting for a viewer is fail-closed, so a +// handler whose solution cannot attest which module is asking never reaches the +// gateway at all. func serveHandler(t *testing.T, gatewayURL string, handler Handler) *httptest.Server { t.Helper() - s := New(Manifest{ID: "wiki", Title: "Wiki"}) + return serveHandlerWith(t, gatewayURL, attestingSource(t), handler) +} + +// serveHandlerWith is serveHandler against a credential source the test holds, +// for an assertion about the credential itself. +func serveHandlerWith(t *testing.T, gatewayURL string, source CredentialSource, handler Handler) *httptest.Server { + t.Helper() + s := New(Manifest{ID: "notes", Title: "Notes"}).Credential(source) s.cfg = config{gatewayURL: gatewayURL} - server := httptest.NewServer(s.wrap(handler)) + server := httptest.NewServer(s.wrapRequest(func(r *http.Request, gw *Gateway) (any, error) { + return handler(r.Context(), gw) + })) t.Cleanup(server.Close) return server } +// viewerBearer is the bearer a real gateway forwards: the viewer's own sealed +// capability, minted by core's authority from core's fixture identities. +// +// It was a placeholder string, which was enough while nothing here read the +// bearer. A real gateway forwards the viewer's own capability, so the fixture +// does too — one that cannot answer a question the code might ask is a fixture +// that tests the refusal path forever. +// +// The installation does NOT come from it. That was the design for one round +// and the comment here said so; the installation is read from the stamped +// x-codefly-installation-id header, because inbound a carried capability is +// caller-controlled. See viewerInstallation. The sealed bearer stays because +// it is what a gateway actually forwards, not because anything reads a seal +// out of it. +func viewerBearer() string { return sealedBearer("viewer") } + +// sealedBearer is a bearer for a named viewer, sealed the same way: a test that +// needs two distinct callers gets two capabilities rather than one placeholder +// string each. +func sealedBearer(seed string) string { return "Bearer " + capability(seed) } + const viewerOrg = "6f1d0a2e-6a21-4d0e-9a0e-2b8f6d2f0b11" // viewerSession is the session the gateway stamps from the verified claims — @@ -1934,9 +798,10 @@ func viewerRequest(t *testing.T, target string) *http.Response { if err != nil { t.Fatalf("new request: %v", err) } - req.Header.Set("authorization", "Bearer viewer-token") + req.Header.Set("authorization", viewerBearer()) req.Header.Set(orgHeader, viewerOrg) req.Header.Set(sessionHeader, viewerSession) + req.Header.Set(workcontext.InstallationIDHeaderName, testInstallation) resp, err := http.DefaultClient.Do(req) if err != nil { t.Fatalf("call solution: %v", err) @@ -1967,7 +832,7 @@ func TestForModuleMintsAndPresentsTheViewersWorkContext(t *testing.T) { } mint := <-gw.mints - if mint.Bearer != "Bearer viewer-token" { + if mint.Bearer != viewerBearer() { t.Errorf("mint presented bearer %q, want the viewer's — accounts must resolve the viewer as owner", mint.Bearer) } if mint.OrgID != viewerOrg { @@ -1990,10 +855,10 @@ func TestForModuleMintsAndPresentsTheViewersWorkContext(t *testing.T) { } call := <-gw.calls - if call.WorkContext != "context-documents.1" { + if call.WorkContext != capability("context-documents.1") { t.Errorf("module read carried work context %q, want the minted one", call.WorkContext) } - if call.Bearer != "Bearer viewer-token" { + if call.Bearer != viewerBearer() { t.Errorf("module read carried bearer %q, want the viewer's — the context is presented alongside it, not instead", call.Bearer) } } @@ -2141,13 +1006,9 @@ func TestForModuleRefusesAnExpiryThisHostCannotUse(t *testing.T) { func TestWorkContextCacheReMintsALapsedCapability(t *testing.T) { cache := newWorkContextCache() mints := 0 - lapsed := func(context.Context) (codefly.WorkContextToken, time.Time, error) { + lapsed := func(context.Context) (string, time.Time, error) { mints++ - token, err := codefly.ParseWorkContextToken(fmt.Sprintf("payload.%d", mints)) - if err != nil { - t.Fatalf("parse token: %v", err) - } - return token, time.Now().Add(-time.Second), nil + return capability(fmt.Sprintf("lapsed-%d", mints)), time.Now().Add(-time.Second), nil } for range 2 { if _, err := cache.resolve(context.Background(), "ask", lapsed); err != nil { @@ -2188,10 +1049,10 @@ func TestDerivedGatewayResolvesTheCapabilityPerRequest(t *testing.T) { t.Fatalf("solution answered %d, want 200", resp.StatusCode) } first, second := <-gw.calls, <-gw.calls - if first.WorkContext != "context-documents.1" { + if first.WorkContext != capability("context-documents.1") { t.Errorf("first read carried %q, want the first capability", first.WorkContext) } - if second.WorkContext != "context-documents.2" { + if second.WorkContext != capability("context-documents.2") { t.Errorf("read after the capability lapsed carried %q, want a freshly minted one — the derived gateway is holding a snapshot", second.WorkContext) } if got := gw.mintCount(); got != 2 { @@ -2199,13 +1060,21 @@ func TestDerivedGatewayResolvesTheCapabilityPerRequest(t *testing.T) { } } -// TestMintCarriesNoOtherModulesCapability keeps the mint on a bearer-only -// client. Riding a capability minted for one module along on the request that -// mints another's is harmless only until the first lapses: the edge verifies -// every presented context, so it would then 401 the call meant to replace it. -func TestMintCarriesNoOtherModulesCapability(t *testing.T) { +// TestMintCarriesThisWorkloadsCredentialAndNoOthers pins what rides on a mint. +// +// One capability belongs there — this execution's own, which says which module +// is asking and is the same on every mint of the process. A capability minted +// for a *module* must not: riding one along on the request that mints another's +// is harmless only until the first lapses, since the edge verifies every +// presented context and would then refuse the call meant to replace it. +func TestMintCarriesThisWorkloadsCredentialAndNoOthers(t *testing.T) { gw := newWorkContextGateway(t, &workContextGateway{}) - solution := serveHandler(t, gw.URL, func(ctx context.Context, g *Gateway) (any, error) { + source := attestingSource(t) + workload, err := source.Credential(context.Background()) + if err != nil { + t.Fatal(err) + } + solution := serveHandlerWith(t, gw.URL, source, func(ctx context.Context, g *Gateway) (any, error) { docs, err := g.ForModule(ctx, "documents", Scope{ResourceKind: "documents", Actions: []string{"read"}}) if err != nil { return nil, err @@ -2225,22 +1094,166 @@ func TestMintCarriesNoOtherModulesCapability(t *testing.T) { } for range 2 { mint := <-gw.mints - if mint.WorkContext != "" { - t.Errorf("mint for %q carried work context %q, want none", mint.Audience, mint.WorkContext) + if mint.WorkContext != workload.Token() { + t.Errorf("mint for %q carried work context %q, want this execution's own credential: the issuer has to know which module is asking", mint.Audience, mint.WorkContext) } - if mint.Bearer != "Bearer viewer-token" { + if mint.Bearer != viewerBearer() { t.Errorf("mint for %q carried bearer %q, want the viewer's", mint.Audience, mint.Bearer) } } } -// TestGatewayTrafficIsNeverProxied keeps the viewer's credentials off an -// arbitrary egress host. Every gateway target is composition-local, and these -// requests carry the bearer and the capability minted for it in headers — the -// same reasoning that already forbids proxying registration traffic. -func TestGatewayTrafficIsNeverProxied(t *testing.T) { - if gatewayTransport.Proxy != nil { - t.Error("gatewayTransport carries a proxy: with HTTP(S)_PROXY set and a NO_PROXY that misses the in-cluster gateway, the viewer's bearer and Work Context would be dialled to an arbitrary egress host") +// TestA409IsNotReadAsSupersessionAtAll replaces +// TestASupersededCapabilityIsDroppedNotReused, and the replacement is the +// finding rather than a weakening of it. +// +// That test asserted the capability is DROPPED when a module answers 409 with +// an installation header, and the behaviour it pinned was wrong in three +// successive ways: any 409 with any installation header; then any 409 whose +// header named the installation the capability is sealed to — and a module +// answering an ordinary business conflict IS in that installation, so a +// duplicate or a lost update evicted a valid capability and the next call +// minted again. +// +// The two cases cannot be told apart on the wire: 409 means "the state you +// were sealed to has moved" and it means "that page already exists". So this +// runtime infers nothing from one. A genuinely superseded capability is +// refused call by call by the far end, which is correct if noisy, and the +// renewal replaces it on the credential's own schedule. What would make an +// inference sound is a host-side discriminator — follow-up 11. +func TestA409IsNotReadAsSupersessionAtAll(t *testing.T) { + gw := newWorkContextGateway(t, &workContextGateway{supersedeFirstCall: true}) + solution := serveHandler(t, gw.URL, func(ctx context.Context, g *Gateway) (any, error) { + docs, err := g.ForModule(ctx, "documents", Scope{ResourceKind: "documents", Actions: []string{"read"}}) + if err != nil { + return nil, err + } + // Two reads through one derived gateway. The first is refused as + // superseded; the second must present a freshly minted capability + // rather than the dropped one. + if _, err := getThrough(ctx, docs); err != nil { + return nil, err + } + return getThrough(ctx, docs) + }) + resp := viewerRequest(t, solution.URL) + defer drainAndClose(resp) + + first, second := <-gw.calls, <-gw.calls + if first.WorkContext == "" || second.WorkContext == "" { + t.Fatal("a module call arrived with no capability at all") + } + if first.WorkContext != second.WorkContext { + t.Error("the 409 retired the cached capability: a module's conflict is not a statement this runtime can read as supersession, and acting on it costs an audited mint per business conflict") + } + if got := gw.mintCount(); got != 1 { + t.Errorf("minted %d capabilities, want 1: one ask mints once, and a 409 from a callee does not change that", got) + } +} + +// TestPlatformTrafficIsNeverProxiedAndPresentsThisWorkload keeps the viewer's +// credentials and this workload's own off an arbitrary egress host, and makes +// the outbound hop authenticated rather than merely https. +// +// Every platform target is composition-local, and these requests carry the +// viewer's bearer, the capability minted for them, and — on a mint — the +// projected token that attests which workload this process is. An https URL +// alone says only that the scheme is https: without a trust anchor the far end +// is verified against whatever the image's system roots happen to hold, and +// without a client certificate it cannot tell this workload from anything else +// that reached it. +func TestPlatformTrafficIsNeverProxiedAndPresentsThisWorkload(t *testing.T) { + if unauthenticatedTransport.Proxy != nil { + t.Error("the platform transport carries a proxy: with HTTP(S)_PROXY set and a NO_PROXY that misses the in-cluster gateway, the viewer's bearer and capability would be dialled to an arbitrary egress host") + } + + certFile, keyFile, bundleFile, _, _ := workloadIdentity(t, testPrincipal) + server := New(Manifest{ID: testSolutionID}) + server.cfg = config{identityCertFile: certFile, identityKeyFile: keyFile, trustBundleFile: bundleFile, + mintPeersFile: identitiesFile(t, testGatewayPrincipal), + gatewayPeersFile: identitiesFile(t, testGatewayPrincipal), + // This test inspects the transport rather than dialling, so any + // resolved destination will do. + mintURL: "https://mint.cell:443" + credentialMintPath} + server.principal = testPrincipal + client, err := server.outboundClient(nil) + if err != nil { + t.Fatalf("outboundClient: %v", err) + } + transport, ok := client.Transport.(*http.Transport) + if !ok { + t.Fatalf("outbound transport is %T, want *http.Transport", client.Transport) + } + switch { + case transport.Proxy != nil: + t.Error("the outbound client carries a proxy") + case transport.DialTLSContext == nil: + t.Fatal("the outbound client has no per-connection dialler, so its trust is whatever was snapshotted at boot") + case transport.TLSClientConfig != nil: + t.Error("the outbound transport carries a snapshotted TLS configuration: peer trust must be built per connection, or a removed root keeps authenticating the platform") + case transport.IdleConnTimeout == 0 || transport.IdleConnTimeout > outboundTrustReloadBound: + t.Errorf("idle connections live for %s, so per-dial reloading bounds nothing: want at most %s", transport.IdleConnTimeout, outboundTrustReloadBound) + } + if client.CheckRedirect == nil { + t.Error("the outbound client follows redirects: net/http copies every header but three across hosts, so a Location would be handed this workload's credentials") + } + // A pair that rotated to another workload's identity must not be presented + // to the platform, which the default path never checked: the listener's + // leaf was held to the principal and this second reloader over the same + // files was not. + rival, rivalKey, _, _, _ := workloadIdentity(t, "spiffe://codefly.test/ns/solutions/sa/another-workload") + writeFile(t, certFile, readFile(t, rival)) + writeFile(t, keyFile, readFile(t, rivalKey)) + if _, err := server.outboundClient(nil); err == nil { + t.Error("an outbound client was built presenting a leaf issued for another workload") + } + // A boot with no trust anchor cannot build one at all, which is the same + // refusal the listener makes. + server.cfg.trustBundleFile = "" + if _, err := server.outboundClient(nil); err == nil { + t.Error("an outbound client was built with no projected anchor to verify the platform against") + } +} + +// TestAHandlersGatewayDialsThroughTheBootsAuthenticatedTransport is the leg the +// test above was missing, and a reviewer showed it: everything there inspects +// the client outboundClient *returns*, and the defect this file's comments are +// about was the gateway handed to a handler not carrying it. Deleting +// gatewayFor's one assignment left every assertion above passing while a +// handler's reads went out over the unauthenticated fallback. +// +// So this drives the production gateway client at a platform host that requires +// a caller's certificate, and asks the host who called. The answer has to be +// this workload. +func TestAHandlersGatewayDialsThroughTheBootsAuthenticatedTransport(t *testing.T) { + c := newCell(t) + certFile, keyFile, bundleFile, _, _ := c.workload(t, testPrincipal) + host := newPlatformHost(t, c, testGatewayPrincipal) + + server := New(Manifest{ID: testSolutionID}) + server.cfg = config{identityCertFile: certFile, identityKeyFile: keyFile, trustBundleFile: bundleFile, + mintPeersFile: identitiesFile(t, testGatewayPrincipal), + gatewayPeersFile: identitiesFile(t, testGatewayPrincipal), gatewayURL: host.URL} + server.principal = testPrincipal + outbound, err := server.outboundClient(nil) + if err != nil { + t.Fatalf("outboundClient: %v", err) + } + server.outbound = outbound + + gateway := server.gatewayFor(http.Header{"authorization": {viewerBearer()}}) + resp, err := gateway.HTTPClient().Get(host.URL + "/v1/things/search") + if err != nil { + t.Fatalf("a handler's gateway could not reach a platform host that requires this workload's certificate: %v\n"+ + "a gateway that does not carry the boot's transport dials the platform as an anonymous client", err) + } + _ = resp.Body.Close() + called := host.called() + if len(called) == 0 { + t.Fatal("the platform host recorded no caller") + } + if called[0] != testPrincipal { + t.Errorf("the platform saw a request from %q, want this workload's own %q", called[0], testPrincipal) } } @@ -2288,7 +1301,7 @@ func TestForModuleSurfacesARefusedMint(t *testing.T) { // for a viewer with no organization selected. func TestForModuleRefusesWithoutTheViewersOrg(t *testing.T) { gw := newWorkContextGateway(t, &workContextGateway{}) - _, err := newGateway(gw.URL, "Bearer viewer-token", "", viewerSession). + _, err := newGateway(gw.URL, viewerBearer(), "", viewerSession). ForModule(context.Background(), "documents", Scope{ResourceKind: "documents", Actions: []string{"read"}}) if err == nil { t.Fatal("ForModule minted a work context with no organization") @@ -2319,7 +1332,7 @@ func TestGatewayWithoutAModuleCarriesOnlyTheBearer(t *testing.T) { if call.WorkContext != "" { t.Errorf("undelegated gateway sent work context %q, want none", call.WorkContext) } - if call.Bearer != "Bearer viewer-token" { + if call.Bearer != viewerBearer() { t.Errorf("undelegated gateway sent bearer %q, want the viewer's", call.Bearer) } if got := gw.mintCount(); got != 0 { @@ -2370,7 +1383,7 @@ func TestMintRootsTheTaskInTheViewersVerifiedSession(t *testing.T) { // session to name at all — an API key. func TestForModuleRefusesWithoutTheViewersSession(t *testing.T) { gw := newWorkContextGateway(t, &workContextGateway{}) - _, err := newGateway(gw.URL, "Bearer viewer-token", viewerOrg, ""). + _, err := newGateway(gw.URL, viewerBearer(), viewerOrg, ""). ForModule(context.Background(), "documents", Scope{ResourceKind: "documents", Actions: []string{"read"}}) if err == nil { t.Fatal("ForModule minted a work context with no viewer session") @@ -2400,20 +1413,21 @@ func TestBrowserSuppliedWorkContextIsNeverForwarded(t *testing.T) { if err != nil { t.Fatalf("new request: %v", err) } - req.Header.Set("authorization", "Bearer viewer-token") + req.Header.Set("authorization", viewerBearer()) req.Header.Set(orgHeader, viewerOrg) req.Header.Set(sessionHeader, viewerSession) - req.Header.Set(codefly.WorkContextHeaderName, "forged.capability") + req.Header.Set(workcontext.InstallationIDHeaderName, testInstallation) + req.Header.Set(workcontext.HeaderName, capability("forged")) resp, err := http.DefaultClient.Do(req) if err != nil { t.Fatalf("call solution: %v", err) } defer drainAndClose(resp) - if mint := <-gw.mints; mint.WorkContext != "" { - t.Errorf("mint carried work context %q, want none — a caller-supplied capability must not authenticate the mint", mint.WorkContext) + if mint := <-gw.mints; mint.WorkContext == capability("forged") { + t.Error("the mint presented the capability the browser sent: a caller-supplied capability must not authenticate anything") } - if call := <-gw.calls; call.WorkContext != "context-documents.1" { + if call := <-gw.calls; call.WorkContext != capability("context-documents.1") { t.Errorf("module read carried work context %q, want the minted one", call.WorkContext) } } @@ -2449,9 +1463,10 @@ func TestRefusedBoundariesAnswerAStatusTheCallerCanAct(t *testing.T) { if err != nil { t.Fatalf("new request: %v", err) } - req.Header.Set("authorization", "Bearer viewer-token") + req.Header.Set("authorization", viewerBearer()) req.Header.Set(orgHeader, tt.org) req.Header.Set(sessionHeader, tt.session) + req.Header.Set(workcontext.InstallationIDHeaderName, testInstallation) resp, err := http.DefaultClient.Do(req) if err != nil { t.Fatalf("call solution: %v", err) @@ -2480,50 +1495,22 @@ func TestRefusedBoundariesAnswerAStatusTheCallerCanAct(t *testing.T) { } } -// internalTokenAuth presents the shared cluster-internal token and nothing else. -// It is a test stub, not a credential this runtime can present: both -// registration surfaces refuse the shared token, so no production path builds -// one (module-saas-starter#540). It lives here so heartbeat's status handling -// can be exercised without standing up a credential exchange — and so that the -// fallback this runtime deliberately removed cannot be reinstated by wiring one -// line, which is what leaving it in the package would have allowed. -type internalTokenAuth string - -func (t internalTokenAuth) authorize(_ context.Context, req *http.Request) (time.Time, error) { - if t != "" { - req.Header.Set(internalTokenHeader, string(t)) - } - // No expiry: injected configuration, so it cannot lapse mid-request. - return time.Time{}, nil -} - -func (internalTokenAuth) invalidate() {} - -func (internalTokenAuth) succeeded() {} - -// TestManifestURLIsNeverTheListenAddress pins where the registered manifestUrl -// points. Unset PUBLIC_URL used to yield "http://localhost:/assets/...", -// this process's own loopback address, so every deployed solution registered a -// manifest no browser could load and the product showed it as failed to load. -// Without an explicit origin the URL is now the path on this backend, which the -// host resolves against the route by which it reaches the solution. -func TestManifestURLIsNeverTheListenAddress(t *testing.T) { - for _, tc := range []struct { - public string - want string - }{ - {public: "", want: "/assets/mf-manifest.json"}, - {public: "https://solutions.example.com/lastlogin", want: "https://solutions.example.com/lastlogin/assets/mf-manifest.json"}, - {public: "https://solutions.example.com/", want: "https://solutions.example.com/assets/mf-manifest.json"}, - } { - t.Setenv("PUBLIC_URL", tc.public) - t.Setenv("PORT", "8080") - s := New(Manifest{ID: "lastlogin-go"}) - s.cfg = loadConfig(context.Background(), s.manifest.ID) - frontend, _ := s.manifestMap()["frontend"].(map[string]any) - if got := frontend["manifestUrl"]; got != tc.want { - t.Errorf("PUBLIC_URL=%q: manifestUrl = %v, want %q", tc.public, got, tc.want) - } +// TestManifestURLIsNeverAnAbsoluteOrigin pins where the served manifestUrl +// points. It used to be absolute: on PUBLIC_URL when one was set, and otherwise +// on "http://localhost:", this process's own loopback address — so every +// deployed solution published a manifest no browser could load and the product +// showed it as failed to load. It is now the path on this backend and nothing +// else, which the host resolves against the route its presence document names; +// PUBLIC_URL is gone with the registration that was the only reason to build an +// origin here. +func TestManifestURLIsNeverAnAbsoluteOrigin(t *testing.T) { + t.Setenv("PUBLIC_URL", "https://solutions.example.com/widgets") + t.Setenv("PORT", "8080") + s := New(Manifest{ID: testSolutionID}) + s.cfg = loadConfig(context.Background(), testSolutionID, nil) + frontend, _ := s.manifestMap()["frontend"].(map[string]any) + if got := frontend["manifestUrl"]; got != federationManifestPath { + t.Errorf("manifestUrl = %v, want %q: no environment variable may make it absolute again", got, federationManifestPath) } } @@ -2559,3 +1546,521 @@ func TestWorkContextPrincipalsReportsWhomAccountsIssuedFor(t *testing.T) { t.Fatalf("minted %d capabilities, want 1: reading the principals reuses the capability", n) } } + +// optionalIdentitiesFile is identitiesFile, or no path at all for an empty set +// — the distinction a boot-refusal table needs, since "the platform never +// provisioned a path" and "the path is there and names nobody" are two +// different refusals and each has its own row. +func optionalIdentitiesFile(t *testing.T, identities string) string { + t.Helper() + if identities == "" { + return "" + } + return identitiesFile(t, identities) +} + +// TestABootWithoutAResolvedMintURLIsRefused: the previous revision derived the +// mint address from the resolved gateway and labelled the result a guess — the +// field was literally named mintURLGuessed, the code said the endpoint was "NOT +// settled" and that "neither endpoint exists yet", and the README and the boot +// log both said "settled". Both could not be true. +// +// This is the address the projected service-account token goes to, which is the +// strongest statement this process can make about which workload it is. A +// guessed address for that is what fail-closed forbids, and this package +// already refuses rather than guesses when it cannot pair a token-exchange URL. +func TestABootWithoutAResolvedMintURLIsRefused(t *testing.T) { + cfg := config{port: "8080", gatewayURL: "https://gateway:42152", profile: localProfile} + err := cfg.validate() + if err == nil { + t.Fatal("the boot accepted a configuration with no mint URL, so this runtime would POST its projected token to an address nobody resolved") + } + if !strings.Contains(err.Error(), CredentialMintURLEnvironmentVariable) { + t.Errorf("the refusal %q does not name %s, the variable that sets it", err, CredentialMintURLEnvironmentVariable) + } + // And the refusal must not read as a provisioning gap in the gateway, + // which resolved perfectly well. + if strings.Contains(err.Error(), "unresolved gateway") { + t.Errorf("the refusal %q blames the gateway, which resolved: that sends an operator to inspect endpoint resolution over a value that is simply not set", err) + } +} + +// TestNoProductionCodeDerivesTheMintAddress pins the absence. A default address +// is the thing this repository's rules single out, and the previous one came +// back as a "labelled stopgap" that the README then described as settled. +func TestNoProductionCodeDerivesTheMintAddress(t *testing.T) { + for _, name := range moduleSources(t) { + source, err := os.ReadFile(name) + if err != nil { + t.Fatalf("read %s: %v", name, err) + } + body := string(source) + for _, derived := range []string{"gatewayURL + credentialMintPath", "gatewayURL+credentialMintPath"} { + if strings.Contains(body, derived) { + t.Errorf("%s derives the credential mint address from the resolved gateway: the gateway's address says where the gateway is, not where this host mints a credential, and the projected token this runtime sends there is not something to address by assumption", name) + } + } + if strings.Contains(body, "mintURLGuessed") { + t.Errorf("%s still carries mintURLGuessed: a guessed address for the endpoint this runtime attests itself to is refused now, so there is nothing to label", name) + } + } +} + +// TestTwoAdmissionSourcesAreRefusedRatherThanRanked: an admission set answered +// by both an operator override and the platform's provisioning is two sources +// for one authorization fact. The previous revision logged which one won, which +// reports the conflict without resolving it — whichever this process picked, the +// other is a decision somebody made that is not in force, and for admission +// being wrong admits a caller. +// +// It is the stance the SDK already takes on the same question: a value +// delivered inline and by file carrier is refused as two sources for one fact. +func TestTwoAdmissionSourcesAreRefusedRatherThanRanked(t *testing.T) { + base := func(t *testing.T) config { + t.Helper() + return config{ + port: "8080", gatewayURL: "https://gateway:42152", + mintURL: "https://gateway:42152" + credentialMintPath, + profile: localProfile, + } + } + + t.Run("an override alone is accepted", func(t *testing.T) { + cfg := base(t) + cfg.allowedCallersFile = identitiesFile(t, testGatewayPrincipal) + if err := cfg.validate(); err != nil { + t.Fatalf("an override with no provisioning behind it was refused: %v", err) + } + }) + + // Driven through the real resolver, not injected. The comment that used to + // stand here said a test cannot make the SDK's workspace configuration + // answer, and that was simply false — authorityValues does it for the + // authority group, the same way. So the conflict rule had no behavioural + // test at all and three mutants of it survived. + t.Run("both answering is refused, through the resolver", func(t *testing.T) { + provisionedWorkloadValue(t, WorkloadIdentityAllowedCallersFileKey, "/provisioned/callers") + t.Setenv(IdentityAllowedCallersFileEnvironmentVariable, "/override/callers") + conflict := conflictingAdmissionSources(context.Background(), nil) + if conflict == nil { + t.Fatal("the resolver saw no conflict while both the override and the platform answered for the caller set") + } + for _, named := range []string{"/override/callers", "/provisioned/callers"} { + if !strings.Contains(conflict.Error(), named) { + t.Errorf("the refusal %q does not name %s, so an operator cannot tell which two answers are in play", conflict, named) + } + } + + cfg := base(t) + cfg.admissionConflict = conflict + err := cfg.validate() + if err == nil { + t.Fatal("a boot accepted an admission set answered by two sources: ranking them silently is how a caller set gets widened with nothing recording that the platform's decision was not in force") + } + if !strings.Contains(err.Error(), "answered twice") { + t.Errorf("the refusal %q does not say the set has two answers", err) + } + // And it is reported before every other provisioning message, or it + // reads as ordinary advice next to them. + cfg.port = "not-a-port" + if again := cfg.validate(); again == nil || !strings.Contains(again.Error(), "answered twice") { + t.Errorf("with another refusal also pending, validate reported %v: the conflicting-authorization refusal is the one that says a decision is not in force", again) + } + }) + + t.Run("the resolver finds no conflict with no override set", func(t *testing.T) { + provisionedWorkloadValue(t, WorkloadIdentityAllowedCallersFileKey, "/provisioned/callers") + if err := conflictingAdmissionSources(context.Background(), nil); err != nil { + t.Errorf("a conflict was reported with no override set: %v", err) + } + }) + + // And an override against provisioning this process cannot READ is a + // conflict too. The resolver discarded the SDK's error, saw no second + // answer, and put the override in force — against provisioning it simply + // could not see. + t.Run("an unreadable platform answer is a conflict, not an absent one", func(t *testing.T) { + t.Setenv(IdentityAllowedCallersFileEnvironmentVariable, "/override/wide") + // No workspace configuration loaded at all, so the SDK errors rather + // than answering empty. + withoutWorkloadValues(t) + // With a failed environment load, which is the one signal that + // separates "never provisioned" from "could not be read": the SDK + // answers both with the same error. + err := conflictingAdmissionSources(context.Background(), errors.New("loading the injected environment failed")) + if err == nil { + t.Fatal("an override was put in force while the platform's own answer could not be read: an unreadable answer is still an answer somebody provisioned, and choosing between two is exactly what this rule refuses") + } + if !strings.Contains(err.Error(), "cannot be read") { + t.Errorf("the refusal %q does not say the platform's answer was unreadable", err) + } + }) +} + +// TestAnAdmissionFileIsReadWholeOrRefused: each entry has to be a complete +// line, which is the part that is not obvious. +// +// The previous reader split whatever it was handed, so a file caught mid-write +// — a platform rewriting it without an atomic swap, an operator's `>` redirect +// — truncated an entry and the truncation was admitted as an identity of its +// own. The executed review showed ".../sa/gateway-internal" read as +// ".../sa/gateway", admitting a caller the set never named. +func TestAnAdmissionFileIsReadWholeOrRefused(t *testing.T) { + const gateway = "spiffe://codefly.test/ns/platform/sa/gateway" + const internal = "spiffe://codefly.test/ns/platform/sa/gateway-internal" + + read := func(t *testing.T, content string) ([]string, error) { + t.Helper() + path := filepath.Join(t.TempDir(), "identities") + writeFile(t, path, content) + return resolvedIdentities(path, "unresolved")() + } + + t.Run("a truncated last line is refused, not admitted", func(t *testing.T) { + // What a reader sees partway through a non-atomic rewrite of a file + // whose only entry is the -internal identity. + got, err := read(t, internal[:len(internal)-len("-internal")]) + if err == nil { + t.Fatalf("a file caught mid-write resolved to %v: the fragment %q is a different identity from the one being written, and admitting it lets in a caller the set never named", got, gateway) + } + if !strings.Contains(err.Error(), "newline") { + t.Errorf("the refusal %q does not say why the file is not whole", err) + } + }) + + t.Run("a terminated file is read", func(t *testing.T) { + got, err := read(t, gateway+"\n"+internal+"\n") + if err != nil { + t.Fatalf("a complete file was refused: %v", err) + } + if len(got) != 2 || got[0] != gateway || got[1] != internal { + t.Errorf("resolved %v, want both identities", got) + } + }) + + t.Run("entries that can never match are refused at the file", func(t *testing.T) { + for _, tc := range []struct{ name, content string }{ + {"a byte-order mark", "\ufeff" + gateway + "\n"}, + {"a NUL in an entry", "spiffe://codefly.test/ns/platform/sa/gate\x00way\n"}, + {"not a URI at all", "gateway\n"}, + {"another scheme", "https://codefly.test/ns/platform/sa/gateway\n"}, + {"no path", "spiffe://codefly.test\n"}, + } { + t.Run(tc.name, func(t *testing.T) { + if got, err := read(t, tc.content); err == nil { + t.Errorf("resolved %v: an entry that cannot match any peer is a provisioning mistake, and carrying it silently turns into a caller nobody admits at a handshake far from the cause", got) + } + }) + } + }) + + t.Run("a file too large to re-read every second is refused", func(t *testing.T) { + big := strings.Repeat(gateway+"\n", (admissionFileLimit/(len(gateway)+1))+16) + got, err := read(t, big) + if err == nil { + t.Fatalf("resolved %d identities from a file over the limit: this is read on every handshake, every dial, and once a second per established connection in each direction", len(got)) + } + // And refused BY THE CAP. Without this the case passes for the wrong + // reason: the reader stops at the limit, so the last line it sees is a + // fragment and the whole-line rule refuses it — which means removing + // the cap entirely left this green. + if !strings.Contains(err.Error(), "larger than") { + t.Errorf("the refusal %q is not about the file's size, so this case does not cover the cap", err) + } + }) + + t.Run("comments, CRLF and commas still work", func(t *testing.T) { + got, err := read(t, "# the gateway\r\n"+gateway+", "+internal+"\r\n") + if err != nil { + t.Fatalf("a conforming file was refused: %v", err) + } + if len(got) != 2 { + t.Errorf("resolved %v, want two identities", got) + } + }) +} + +// TestACredentialBearingURLCarriesNothingButADestination: userinfo, a query and +// a fragment were all accepted on both credential-bearing destinations, and the +// first two were written to the boot log verbatim — a secret that is disclosed +// and, in the userinfo case, that net/http strips before the request is even +// sent, so it is never used for anything except being logged. +func TestACredentialBearingURLCarriesNothingButADestination(t *testing.T) { + for _, tc := range []struct{ name, mint, names string }{ + {"userinfo", "https://ops:s3cr3t@mint.cell/platform/_credential", "userinfo"}, + {"a query", "https://mint.cell/platform/_credential?token=s3cr3t", "query"}, + {"a fragment", "https://mint.cell/platform/_credential#part", "fragment"}, + } { + t.Run(tc.name, func(t *testing.T) { + cfg := config{port: "8080", gatewayURL: "https://gateway:42152", mintURL: tc.mint, profile: localProfile} + err := cfg.validate() + if err == nil { + t.Fatalf("a credential mint URL carrying %s was accepted", tc.names) + } + if !strings.Contains(err.Error(), tc.names) { + t.Errorf("the refusal %q does not say the URL carries %s", err, tc.names) + } + }) + } + + for _, tc := range []struct{ raw, hidden string }{ + {"https://ops:s3cr3t@mint.cell/x", "s3cr3t"}, + {"https://mint.cell/x?token=s3cr3t", "s3cr3t"}, + } { + if got := redactedURL(tc.raw); strings.Contains(got, tc.hidden) { + t.Errorf("redactedURL(%q) = %q, which still carries the secret", tc.raw, got) + } + } + + // And the boot log actually uses it. Asserting on the helper alone left + // the *call site* free to pass the raw URL, which is where the secret was + // being written — the helper being correct is not the property. + t.Run("the boot log carries no secret", func(t *testing.T) { + var captured bytes.Buffer + log.SetOutput(&captured) + t.Cleanup(func() { log.SetOutput(os.Stderr) }) + + cfg := config{port: "8080", gatewayURL: "https://gateway:42152", profile: localProfile, + mintURL: "https://ops:s3cr3t-password@mint.cell/platform/_credential"} + // validate() refuses this URL, and the log line still must not carry + // the secret: the refusal and the logging are independent, which is + // the whole reason the log redacts regardless. + _ = cfg.validate() + cfg.mintURL = "https://mint.cell/platform/_credential" + if err := cfg.validate(); err != nil { + t.Fatalf("a conforming configuration was refused: %v", err) + } + cfg.mintURL = "https://ops:s3cr3t-password@mint.cell/platform/_credential" + cfg.logResolved() + if got := captured.String(); strings.Contains(got, "s3cr3t-password") { + t.Errorf("the boot log carries the secret from the mint URL:\n%s", got) + } + }) +} + +// TestTheEnvironmentLoadErrorReachesTheRefusals: three refusals and a boot log +// exist to tell "the SDK resolved nothing" apart from "loading the injected +// environment failed first", and a signature refactor dropped the value they +// read, so all of them reported the former. +// +// That is the exact mistake validate() was written to stop making: its own +// comment records that a single generic message sent an operator to inspect +// endpoint resolution over a variable they had broken themselves. +func TestTheEnvironmentLoadErrorReachesTheRefusals(t *testing.T) { + loadErr := errors.New("the injected carriers could not be read") + + t.Run("loadConfig carries it onto the configuration", func(t *testing.T) { + cfg := loadConfig(context.Background(), testSolutionID, loadErr) + if cfg.environmentLoadErr == nil { + t.Fatal("loadConfig dropped the environment-load error, so every refusal below reads as absent provisioning rather than as an environment that never loaded") + } + }) + + t.Run("an unresolved destination says which it is", func(t *testing.T) { + cfg := config{port: "8080", profile: localProfile, environmentLoadErr: loadErr} + err := cfg.validate() + if err == nil { + t.Fatal("a configuration with no gateway was accepted") + } + if !strings.Contains(err.Error(), "injected environment failed first") { + t.Errorf("the refusal %q does not say the environment never loaded, so it sends an operator to inspect provisioning that may well be in place", err) + } + }) + + t.Run("a missing path says which it is", func(t *testing.T) { + cfg := config{environmentLoadErr: loadErr} + err := cfg.requirePath("workload identity certificate", "", "OVERRIDE", "CERT_FILE") + if err == nil { + t.Fatal("an unresolved path was accepted") + } + if !strings.Contains(err.Error(), "environment failed first") { + t.Errorf("the refusal %q does not distinguish an unprovisioned path from an environment that never loaded", err) + } + }) +} + +// TestAUsernameCredentialIsNotLogged: url.Redacted() masks the password and +// keeps the username, so a token carried as a username — which is how a great +// many of them are carried — came through the redaction intact. +func TestAUsernameCredentialIsNotLogged(t *testing.T) { + for _, raw := range []string{ + "https://s3cr3t-token@mint.cell/platform/_credential", + "https://s3cr3t-token:@mint.cell/platform/_credential", + "https://user:s3cr3t-token@mint.cell/platform/_credential", + } { + if got := redactedURL(raw); strings.Contains(got, "s3cr3t-token") { + t.Errorf("redactedURL(%q) = %q, which still carries the secret: Redacted() only masks the password", raw, got) + } + // And it still says where, or it is useless in a log. + if got := redactedURL(raw); !strings.Contains(got, "mint.cell") { + t.Errorf("redactedURL(%q) = %q, which no longer names the destination", raw, got) + } + } +} + +// clearSelfEnvironment unsets every carrier the configuration tests key on, so +// a variable exported in the shell running the suite cannot decide an +// assertion. +// +// It lived in deployed_registration_test.go, which went with the +// registrations. The MCP configuration tests that arrived on main need it, and +// the reason it exists outlives what it was written for: these assertions are +// about what the SDK resolves, and an exported variable answering instead is a +// green run that proves nothing. +func clearSelfEnvironment(t *testing.T) { + t.Helper() + for _, key := range []string{"SELF_UPSTREAM", "PUBLIC_URL", "CODEFLY__RUNTIME_CONTEXT", + "CODEFLY__MODULE", "CODEFLY__SERVICE", "CODEFLY__ENVIRONMENT"} { + t.Setenv(key, "") + } + // Every self-endpoint carrier, by prefix rather than by name. + // + // The restored version named one deployment's: + // CODEFLY__SELF_ENDPOINT__LASTLOGIN_GO__BACKEND__HTTP__HTTP. A carrier's + // name is built from the module and service it belongs to, so naming one + // puts a particular deployment in a runtime that is generic by rule — and + // it clears exactly that deployment's variable and no other, which is the + // weaker half of the problem: an operator running the suite with any other + // service's carrier exported still has it answering these assertions. + for _, entry := range os.Environ() { + key, _, _ := strings.Cut(entry, "=") + if strings.HasPrefix(key, resources.SelfEndpointPrefix) { + t.Setenv(key, "") + } + } +} + +// TestAnAskCannotBeEditedAfterItsCeilingIsChecked is round fourteen's B2. +// +// workContextScopes converted Scope to its wire form, which copies the struct +// and leaves both pointing at the CALLER's backing arrays. The ask is retained +// on the delegation and its JSON is the cache key, computed once — so a caller +// could hand over Actions: []string{"read"}, take the delegated gateway, write +// "delete" into that array, and wait for the cached capability to expire. The +// next mint serialized "delete", under the key computed for "read", with the +// ceiling having been checked against "read". No concurrency, no second call +// to ForModule, and nothing in the published contract to stop it. +func TestAnAskCannotBeEditedAfterItsCeilingIsChecked(t *testing.T) { + gw := newWorkContextGateway(t, &workContextGateway{}) + mint := newHostMint(t, &hostMint{}) + tokenFile := filepath.Join(t.TempDir(), "token") + writeFile(t, tokenFile, "projected") + module := passthroughModule() + module.Scopes = []Scope{{ResourceKind: "things", Actions: []string{"read"}}} + module.Methods = nil + server := New(Manifest{ID: testSolutionID}).Consumes(module). + Credential(mintClientFor(t, mint, tokenFile)). + Contract(ModuleContract{Ceilings: map[string]map[string][]Scope{ + localProfile: {"things": {{ResourceKind: "things", Actions: []string{"read"}}}}, + }}) + server.cfg = config{gatewayURL: gw.URL, profile: localProfile, apiConsumes: consumesThings} + server.principal = testPrincipal + contract, err := server.resolveContract() + if err != nil { + t.Fatalf("resolve the published contract: %v", err) + } + server.contract, server.contractResolved = contract, true + + header := http.Header{} + header.Set("authorization", viewerBearer()) + header.Set(orgHeader, viewerOrg) + header.Set(sessionHeader, viewerSession) + header.Set(workcontext.InstallationIDHeaderName, testInstallation) + gateway := server.gatewayFor(header) + + // The ask the ceiling is checked against, in a slice the caller keeps. + actions := []string{"read"} + acting, err := gateway.ForModule(context.Background(), "things", Scope{ResourceKind: "things", Actions: actions}) + if err != nil { + t.Fatalf("the declared ask was refused: %v", err) + } + first := <-gw.mints + if len(first.AuthorityScopes) != 1 || len(first.AuthorityScopes[0].Actions) != 1 || + first.AuthorityScopes[0].Actions[0] != "read" { + t.Fatalf("the first mint asked for %+v, want read", first.AuthorityScopes) + } + + // The caller edits the array it still holds, and the capability it was + // handed is retired — which is what expiry does, with no second call to + // ForModule and no concurrency. + actions[0] = "delete" + if _, err := acting.workContext(context.Background()); err != nil { + t.Fatalf("read the delegated capability: %v", err) + } + // Expire the held capability, which is what forces the next read to mint. + // This used the cache's supersede primitive, which existed only to serve + // the 409 inference and went with it; expiring the entry directly says + // what the test means — a LATER mint must not carry the edited slice — + // without a production method nothing calls. + acting.contexts.mu.Lock() + delete(acting.contexts.minted, acting.delegation.key) + acting.contexts.mu.Unlock() + + // A refusal here is also a correct answer. What must not happen is a + // silent mint for "delete". + if _, err := acting.workContext(context.Background()); err != nil { + t.Logf("the renewal was refused, which is the other acceptable answer: %v", err) + return + } + second := <-gw.mints + for _, scope := range second.AuthorityScopes { + for _, action := range scope.Actions { + if action == "delete" { + t.Fatalf("a mint asked the issuer for %q after the ceiling had been checked against read: the ask was retained with the caller's own backing array, so editing it changed what was minted under a cache key computed for something else — and the published ceiling is the guarantee that cannot hold", action) + } + } + } +} + +// TestAnOrdinaryConflictDoesNotRetireAValidCapability is round sixteen's first +// major. +// +// supersededCapability accepted ANY 409 carrying a non-empty installation +// header, and a 409 is an ordinary business answer: a duplicate, a lost +// update, a version conflict. A module answering one with installation +// metadata beside it therefore evicted a perfectly valid cached capability, +// and the next call minted again — needless mints and needless audit traffic +// on the one path whose whole purpose is one mint per execution. +// +// The header is matched against the installation the capability THAT WAS +// PRESENTED is sealed to now, so a conflict about somebody else's +// installation says nothing about this one. +func TestAnOrdinaryConflictDoesNotRetireAValidCapability(t *testing.T) { + gw := newWorkContextGateway(t, &workContextGateway{conflictFirstCall: true}) + solution := serveHandler(t, gw.URL, func(ctx context.Context, g *Gateway) (any, error) { + docs, err := g.ForModule(ctx, "documents", Scope{ResourceKind: "documents", Actions: []string{"read"}}) + if err != nil { + return nil, err + } + // Two reads: the first gets the ordinary 409, the second must present + // the SAME capability rather than a freshly minted one. + if _, err := getThrough(ctx, docs); err != nil { + return nil, err + } + status, err := getThrough(ctx, docs) + if err != nil { + return nil, err + } + return map[string]int{"status": status}, nil + }) + + resp := viewerRequest(t, solution.URL) + defer drainAndClose(resp) + if resp.StatusCode != http.StatusOK { + t.Fatalf("the handler answered %d, want 200", resp.StatusCode) + } + + // ONE mint. An ordinary conflict that retired the capability would show up + // here as two. + if got := gw.mintCount(); got != 1 { + t.Errorf("the issuer minted %d capabilities for one ask, want 1: an ordinary 409 retired a valid capability, so every business conflict a module answers costs this solution a fresh audited mint", got) + } + // And the second call presented the same capability. + first, second := <-gw.calls, <-gw.calls + if first.WorkContext == "" || second.WorkContext == "" { + t.Fatal("a module call arrived with no capability at all") + } + if first.WorkContext != second.WorkContext { + t.Error("the second call presented a different capability than the first: the ordinary conflict evicted the cached one, which is the re-mint this check exists to prevent") + } +} diff --git a/stream.go b/stream.go index 7ae008c..b566f5f 100644 --- a/stream.go +++ b/stream.go @@ -230,7 +230,7 @@ func (g *Gateway) TranscodedStream(ctx context.Context, prefix, method string, r func readStream(ctx context.Context, call transcodedCall, httpResp *http.Response, options streamOptions, newMsg func() proto.Message, each func(proto.Message) error) error { // Closed, never drained: a stream has no end to drain to, and the // request's context cancellation has already stopped it on the way out. - defer httpResp.Body.Close() + defer func() { _ = httpResp.Body.Close() }() if httpResp.StatusCode < 200 || httpResp.StatusCode > 299 { detail, _ := io.ReadAll(io.LimitReader(httpResp.Body, gatewayErrorDetailLimit)) return fmt.Errorf("%s %s: %w", call.verb, call.template, &GatewayError{StatusCode: httpResp.StatusCode, detail: detail}) diff --git a/stream_test.go b/stream_test.go index 431a57f..20dbf80 100644 --- a/stream_test.go +++ b/stream_test.go @@ -13,7 +13,7 @@ import ( "time" "connectrpc.com/connect" - codefly "github.com/codefly-dev/sdk-go" + "github.com/codefly-dev/sdk-go/workcontext" "google.golang.org/protobuf/reflect/protoreflect" "google.golang.org/protobuf/types/dynamicpb" ) @@ -60,7 +60,7 @@ func newStreamGateway(t *testing.T) *streamGateway { g.mints = append(g.mints, mint) g.mu.Unlock() writeJSON(w, http.StatusOK, map[string]any{ - "token": "context-" + mint.Audience + ".1", "orgId": mint.OrgID, + "token": capability("context-" + mint.Audience + ".1"), "orgId": mint.OrgID, "ownerPrincipalId": "viewer", "currentActorPrincipalId": "viewer", "expiresAt": time.Now().Add(5 * time.Minute).UTC().Format(time.RFC3339Nano), }) @@ -122,9 +122,10 @@ func watchRequest(entry string) *connect.Request[dynamicpb.Message] { msg := thingMessage("GetThingRequest") setField(msg, "entry_id", protoreflect.ValueOfString(entry)) req := connect.NewRequest(msg) - req.Header().Set("authorization", "Bearer viewer") + req.Header().Set("authorization", viewerBearer()) req.Header().Set(orgHeader, "org-1") req.Header().Set(sessionHeader, "session-1") + req.Header().Set(workcontext.InstallationIDHeaderName, testInstallation) return req } @@ -138,7 +139,7 @@ func has(m *dynamicpb.Message, name string) bool { func TestPassthroughStreamsEachMessageMaskedAsTheViewer(t *testing.T) { gw := newStreamGateway(t) - s := passthroughServer(gw.URL, streamModule(ConsumedMethod{})) + s := passthroughServer(t, gw.URL, streamModule(ConsumedMethod{})) gw.lines <- `{"result":{"entryId":"e1","big":"1","secret":"s1"}}` gw.lines <- `` gw.lines <- `{"result":{"entryId":"e1","big":"2","secret":"s2","sub":{"name":"x"}}}` @@ -166,7 +167,7 @@ func TestPassthroughStreamsEachMessageMaskedAsTheViewer(t *testing.T) { if len(gw.watches) != 1 || gw.watches[0].Method != http.MethodGet || gw.watches[0].URL.Path != "/v1/things/e1/watch" { t.Fatalf("module calls = %v", gw.watches) } - if gw.watches[0].Header.Get(codefly.WorkContextHeaderName) != "context-things.1" || !strings.Contains(gw.watches[0].Header.Get("accept"), "application/x-ndjson") { + if gw.watches[0].Header.Get(workcontext.HeaderName) != capability("context-things.1") || !strings.Contains(gw.watches[0].Header.Get("accept"), "application/x-ndjson") { t.Fatalf("headers = %v", gw.watches[0].Header) } if len(gw.mints) != 1 || mintedActions(gw.mints[0]) != "things:watch" { @@ -181,7 +182,7 @@ func TestPassthroughStreamForwardsOnlyTheBearerForAModuleThatAuthenticatesTheVie for i := range module.Methods { module.Methods[i].Scopes = nil } - s := passthroughServer(gw.URL, module) + s := passthroughServer(t, gw.URL, module) close(gw.lines) stream, err := watchClient(t, s).CallServerStream(context.Background(), watchRequest("e1")) if err != nil { @@ -192,7 +193,7 @@ func TestPassthroughStreamForwardsOnlyTheBearerForAModuleThatAuthenticatesTheVie if err := stream.Err(); err != nil { t.Fatal(err) } - if len(gw.mints) != 0 || gw.watches[0].Header.Get(codefly.WorkContextHeaderName) != "" || gw.watches[0].Header.Get("authorization") != "Bearer viewer" { + if len(gw.mints) != 0 || gw.watches[0].Header.Get(workcontext.HeaderName) != "" || gw.watches[0].Header.Get("authorization") != viewerBearer() { t.Fatalf("want the viewer's bearer and no capability: mints %d, headers %v", len(gw.mints), gw.watches[0].Header) } } @@ -207,7 +208,7 @@ func receiveAll(stream *connect.ServerStreamForClient[dynamicpb.Message]) (int, func TestPassthroughStreamEndsWithTheModulesErrorLine(t *testing.T) { gw := newStreamGateway(t) - s := passthroughServer(gw.URL, streamModule(ConsumedMethod{})) + s := passthroughServer(t, gw.URL, streamModule(ConsumedMethod{})) gw.lines <- `{"result":{"entryId":"e1"}}` gw.lines <- `{"error":{"code":9,"message":"the turn was deleted"}}` stream, err := watchClient(t, s).CallServerStream(context.Background(), watchRequest("e1")) @@ -223,7 +224,7 @@ func TestPassthroughStreamEndsWithTheModulesErrorLine(t *testing.T) { func TestPassthroughStreamRelaysARefusalBeforeTheStream(t *testing.T) { gw := newStreamGateway(t) gw.status = http.StatusForbidden - s := passthroughServer(gw.URL, streamModule(ConsumedMethod{})) + s := passthroughServer(t, gw.URL, streamModule(ConsumedMethod{})) stream, err := watchClient(t, s).CallServerStream(context.Background(), watchRequest("e1")) if err != nil { t.Fatal(err) @@ -235,7 +236,7 @@ func TestPassthroughStreamRelaysARefusalBeforeTheStream(t *testing.T) { func TestPassthroughStreamRefusesAnOversizedMessageWithoutTruncating(t *testing.T) { gw := newStreamGateway(t) - s := passthroughServer(gw.URL, streamModule(ConsumedMethod{MaxStreamMessageBytes: 64})) + s := passthroughServer(t, gw.URL, streamModule(ConsumedMethod{MaxStreamMessageBytes: 64})) gw.lines <- `{"result":{"entryId":"e1"}}` gw.lines <- `{"result":{"entryId":"` + strings.Repeat("x", 200) + `"}}` stream, err := watchClient(t, s).CallServerStream(context.Background(), watchRequest("e1")) @@ -250,7 +251,7 @@ func TestPassthroughStreamRefusesAnOversizedMessageWithoutTruncating(t *testing. func TestPassthroughStreamEndsAtItsDeclaredDuration(t *testing.T) { gw := newStreamGateway(t) - s := passthroughServer(gw.URL, streamModule(ConsumedMethod{MaxStreamDuration: 300 * time.Millisecond})) + s := passthroughServer(t, gw.URL, streamModule(ConsumedMethod{MaxStreamDuration: 300 * time.Millisecond})) gw.lines <- `{"result":{"entryId":"e1"}}` stream, err := watchClient(t, s).CallServerStream(context.Background(), watchRequest("e1")) if err != nil { @@ -273,7 +274,7 @@ func TestPassthroughStreamEndsAtItsDeclaredDuration(t *testing.T) { func TestPassthroughStreamCancelsTheModuleWhenThePageGoesAway(t *testing.T) { gw := newStreamGateway(t) - s := passthroughServer(gw.URL, streamModule(ConsumedMethod{})) + s := passthroughServer(t, gw.URL, streamModule(ConsumedMethod{})) gw.lines <- `{"result":{"entryId":"e1"}}` ctx, cancel := context.WithCancel(context.Background()) stream, err := watchClient(t, s).CallServerStream(ctx, watchRequest("e1")) @@ -295,7 +296,7 @@ func TestPassthroughStreamCancelsTheModuleWhenThePageGoesAway(t *testing.T) { func TestPassthroughStreamRefusesAnAnswerThatIsNotAStream(t *testing.T) { gw := newStreamGateway(t) gw.contentType = "text/html" - s := passthroughServer(gw.URL, streamModule(ConsumedMethod{})) + s := passthroughServer(t, gw.URL, streamModule(ConsumedMethod{})) close(gw.lines) stream, err := watchClient(t, s).CallServerStream(context.Background(), watchRequest("e1")) if err != nil { diff --git a/typed_boundary_test.go b/typed_boundary_test.go new file mode 100644 index 0000000..1179d33 --- /dev/null +++ b/typed_boundary_test.go @@ -0,0 +1,1539 @@ +package solution + +import ( + "fmt" + "go/ast" + "go/types" + "path/filepath" + "strings" + "testing" + + "golang.org/x/tools/go/packages" +) + +// The boundary gates, on RESOLVED TYPES. +// +// Five review rounds each found one more spelling the syntactic gates did not +// follow: a supplied destination, a zero-value declaration, an assignment +// alias, a function-local type alias, `Decode` instead of `Unmarshal`, an +// unexported struct field, an anonymous struct result, and an SDK alias whose +// name the gate had never heard of. Every fix was correct and the next spelling +// arrived anyway, because a rule that reads syntax is a rule about names and an +// author picks the names. +// +// So these ask the type checker instead. `types.Unalias` makes every alias — +// package-level, function-local, the SDK's own — the same type; `info.TypeOf` +// gives the type of an expression however it was assigned; a struct's fields +// and method set are resolved rather than matched. The name lists are gone, +// and with them the thing each round extended. +// +// Two further rules this buys, which no probe list had: +// +// - the capability rule keys on core's GENERATED PACKAGE PATH, not on a set +// of message names, so a message core adds is covered the day it exists; +// - a handler reachable through an exported METHOD is found, which the field +// walk could never see. +// +// What type resolution still cannot decide is stated here rather than left for +// a reviewer: a destination whose static type is an INTERFACE (`proto.Message`, +// `any`) is not a capability by its type, and a capability arriving through one +// is invisible to any static rule. That is the residual, it is narrow, and it +// is the same residual core's own verifier has. +// +// The fixtures in TestTheTypedGatesCatchEveryKnownShape are COMPILED against +// this package and the pinned SDK through an overlay. That is deliberate: the +// syntactic probes only parsed their source, so two of them asserted against +// `workcontext.WorkContextV1` — a type the SDK does not export — and supplied +// three rounds of false evidence that the whole-capability rule worked. + +// gatePackages loads this module's packages with full type information. +func gatePackages(t *testing.T, overlay map[string][]byte) []*packages.Package { + t.Helper() + cfg := &packages.Config{ + Mode: packages.NeedName | packages.NeedFiles | packages.NeedSyntax | + packages.NeedTypes | packages.NeedTypesInfo | packages.NeedDeps | packages.NeedImports, + Overlay: overlay, + } + loaded, err := packages.Load(cfg, ".", "./passthroughtest") + if err != nil { + t.Fatalf("load this module for type-resolved gating: %v", err) + } + if len(loaded) < 2 { + t.Fatalf("loaded %d packages, want at least the root and passthroughtest: a gate that reads nothing passes", len(loaded)) + } + for _, pkg := range loaded { + // A fixture that does not COMPILE cannot supply evidence. This is the + // check the parse-only probes did not have. + for _, err := range pkg.Errors { + t.Fatalf("%s does not type-check, so nothing below means anything: %v", pkg.PkgPath, err) + } + if pkg.TypesInfo == nil { + t.Fatalf("%s loaded without type information", pkg.PkgPath) + } + } + return loaded +} + +// resolved reports whether a type is a real one. go/types gives the INVALID +// type for an expression it could not resolve, and types.Implements says an +// invalid type implements anything. +func resolved(t types.Type) bool { + if t == nil { + return false + } + basic, ok := types.Unalias(t).(*types.Basic) + return !ok || basic.Kind() != types.Invalid +} + +// namedPath is "." for a named type, after resolving +// aliases. Empty for anything unnamed. +func namedPath(t types.Type) string { + named, ok := types.Unalias(t).(*types.Named) + if !ok || named.Obj() == nil || named.Obj().Pkg() == nil { + return "" + } + return named.Obj().Pkg().Path() + "." + named.Obj().Name() +} + +// coreWireType reports whether a type is one of core's generated wire messages, +// by the package it comes from. +// +// By PATH, not by a list of message names. The list was the defect: it knew +// WorkContextV1 and WorkSealV1, the SDK exports those as Claims and +// SealedValues, and SealedOperationBinding was missing entirely — three rounds +// of adding names. Everything under core's generated tree is core's encoding, +// which is the actual rule. +func coreWireType(t types.Type) string { + path := namedPath(t) + if strings.Contains(path, "/codefly-dev/core/generated/") { + return path + } + return "" +} + +// mountableHandlerType reports whether a caller holding this type can serve it +// as-is. +func (g typedGate) mountableHandlerType(t types.Type) bool { + if !resolved(t) { + // An unresolved expression — a package name, a callee identifier — + // has the invalid type, and types.Implements answers TRUE for it. + // Without this guard every exported function "held a handler". + return false + } + // IMPLEMENTS http.Handler — the actual definition of servable, rather than + // a list of the types that happen to. + // + // The list was still the wrong primitive, and the compiled fixtures caught + // it on their first run: `type fixtureDefined http.Handler` is a type + // DEFINITION whose underlying is the interface, so it is mountable and + // matched no name. Asking the type checker whether it implements the + // interface covers http.Handler, HandlerFunc, ServeMux, any definition or + // alias of them, and any type a consumer writes with a ServeHTTP method — + // which is every spelling there is. + if g.handler != nil { + if types.Implements(t, g.handler) || types.Implements(types.NewPointer(t), g.handler) { + return true + } + } + // A bare func with ServeHTTP's parameters is not an http.Handler, and is + // one http.HandlerFunc away from being served. + return servesHTTPSignature(t) +} + +// typedGate carries what one LOAD resolved, because a type from one +// packages.Load is not identical to the same type from another: holding +// net/http.Handler in a package-level variable made the compiled-fixture test +// pass alone and fail after the real gates, since types.Implements compared a +// fixture's *http.Request against the previous load's. Per load, passed in. +type typedGate struct { + handler *types.Interface + // pkgPath is the package under the gate, so the walk can tell a carrier + // THIS package declared from another package's own type. + pkgPath string +} + +// gateFor resolves net/http.Handler out of one loaded package. +func gateFor(pkg *packages.Package) typedGate { + if http := pkg.Imports["net/http"]; http != nil && http.Types != nil { + if handler := http.Types.Scope().Lookup("Handler"); handler != nil { + iface, _ := handler.Type().Underlying().(*types.Interface) + return typedGate{handler: iface, pkgPath: pkg.PkgPath} + } + } + return typedGate{pkgPath: pkg.PkgPath} +} + +// handedOutDirectly is what no exported path may RETURN. It is wider than +// mountable, and the difference is load-bearing. +// +// http.RoundTripper belongs here and NOT in mountableHandlerType: returning +// this runtime's transport hands over a credential-bearing round tripper, which +// is worth refusing — but a RoundTripper is not something a caller mounts, and +// treating it as one made every *http.Client "expose a servable handler", +// because http.Client has a Transport field. Gateway.HTTPClient() returns one +// by design: it is how a handler calls a module as the viewer. The first run of +// this typed gate flagged exactly that, which is the recursion working and the +// set being wrong. +func (g typedGate) handedOutDirectly(t types.Type) bool { + return g.mountableHandlerType(t) || declaredHandlerType(t) || + namedPath(t) == "net/http.RoundTripper" +} + +// servesHTTPSignature reports whether a type is a function with ServeHTTP's +// parameters, under any name or none. +func servesHTTPSignature(t types.Type) bool { + signature, ok := types.Unalias(t).Underlying().(*types.Signature) + if !ok || signature.Params().Len() != 2 { + return false + } + return namedPath(signature.Params().At(0).Type()) == "net/http.ResponseWriter" && + namedPath(derefOnce(signature.Params().At(1).Type())) == "net/http.Request" +} + +// declaredHandlerType is this package's own Handler/RequestHandler, which a +// consumer WRITES. Handing one back is refused; carrying one in a declaration +// type is how a route is declared, because it needs a *Gateway that only a +// booted request produces. +func declaredHandlerType(t types.Type) bool { + switch namedPath(t) { + case "github.com/codefly-dev/solution-runtime-go.Handler", + "github.com/codefly-dev/solution-runtime-go.RequestHandler": + return true + } + return false +} + +// derefOnce strips ONE pointer, for a parameter like *http.Request where the +// level matters. +func derefOnce(t types.Type) types.Type { + if pointer, ok := types.Unalias(t).(*types.Pointer); ok { + return pointer.Elem() + } + return t +} + +// typeWalk is THE traversal, and every rule in this file asks its question +// through it. +// +// Four rounds of gate findings were all the same defect: a walk that +// enumerated the carriers it knew. A struct, then a map, then a signature +// result, then an interface-typed field, then a generic's type argument, then +// a concrete wrapper around a capability. Each fix was correct and the next +// carrier arrived, because the author picks the carrier. +// +// So this reaches a FIXED POINT over resolved types instead: given a type, it +// expands every way a value of that type can hold or hand back another value — +// pointer, slice, array, map key and element, channel, struct field, +// signature result, interface method result, exported method result, a named +// type's underlying type, a generic's type arguments, a type parameter's +// constraint terms — and asks the predicate at every node. There is no list of +// shapes to extend because there is no list. +// +// It FAILS CLOSED in both directions that a traversal can go wrong: +// +// - A cycle is guarded by identity, not by a depth cap. The previous walks +// returned false beyond depth 8 and depth 6, so nine nested arrays around +// an http.Handler escaped both — a traversal that gives up quietly is a +// gate that can be exhausted by nesting. +// - What it cannot resolve is REPORTED rather than skipped. An invalid type +// or a budget exhausted by a pathological graph comes back as `unresolved`, +// and every caller turns that into a finding. A gate that cannot see is not +// a gate that passes. +type typeWalk struct { + gate typedGate + match func(types.Type) bool + fields fieldPolicy + // local is this package's path. When foreignOpaque is set, a named type + // from ANOTHER package is not expanded into its fields. + // + // That distinction is the rule, not a concession: what this boundary + // forbids is THIS package building a carrier for one of core's wire + // messages. The SDK's own `Credential` transitively contains them — that + // is its API, and this runtime holds one by design — so expanding every + // foreign struct flagged `openCredential`, `mcpViewer` and `Credential` + // itself. A core wire type is still matched directly wherever it appears, + // however deep, because the match runs before any expansion. + local string + foreignOpaque bool + // storage restricts expansion to what a value's own memory contains; + // indirect adds what a decoder allocates through. + storage bool + indirect bool + // seen keys on the type's own identity, which is what makes a recursive + // named type terminate. + seen map[types.Type]bool + depth int + // budget bounds a graph this walk cannot otherwise finish. Exhausting it is + // an ANSWER — "this could not be decided" — not a false. + budget int + unresolved string +} + +// fieldPolicy says which struct fields a value's holder can reach, and — with +// the walk's graph mode — what "carries" means for the question being asked. +type fieldPolicy int + +const ( + // reachableFields: exported, or embedded (reachable by its own type name). + // What a CONSUMER of a handed-out value can read. Used with the EXPOSURE + // graph, which follows every way a value hands back another value: + // methods, callable results, interface results, elements, fields. + reachableFields fieldPolicy = iota + // everyField: visibility is irrelevant, because allocating a struct + // allocates its unexported fields too. Used with the STORAGE graph. + everyField +) + +// storageGraph restricts the walk to what a value's own MEMORY contains. +// +// This distinction is the one the first run of the fixed-point walk got wrong, +// and the finding was real: expanding every carrier reported +// `&authorityHeldSource{…}` as constructing a capability, because its +// `CredentialSource` field is an interface whose method returns the SDK's +// credential — which is an alias for one of core's wire messages. +// +// But a method RESULT is not an allocation, and neither is what a pointer, a +// slice, a map or a channel refers to: those are nil in a freshly allocated +// value. Allocating a struct allocates its FIELDS, and allocating an array +// allocates its ELEMENTS, inline. That is the whole of what construction +// means, so that is the whole of what this walk follows — which is both +// narrower and exactly right: `struct{ Seal wc.SealedValues }` allocates a +// seal, and holding a `CredentialSource` does not. +func storageGraph(w *typeWalk) { w.storage = true } + +// throughIndirection adds what a DECODER reaches: it allocates through the +// pointers and slices in its destination, which a constructor does not. +func throughIndirection(w *typeWalk) { w.storage, w.indirect = true, true } + +func (g typedGate) walk(t types.Type, match func(types.Type) bool, fields fieldPolicy, opts ...func(*typeWalk)) (bool, string) { + w := &typeWalk{gate: g, match: match, fields: fields, seen: map[types.Type]bool{}, budget: 50000} + for _, opt := range opts { + opt(w) + } + found := w.reach(t) + return found, w.unresolved +} + +// withinThisPackage stops the walk expanding another package's named types +// into their fields. +func withinThisPackage(path string) func(*typeWalk) { + return func(w *typeWalk) { w.local, w.foreignOpaque = path, true } +} + +func (w *typeWalk) reach(t types.Type) bool { + if t == nil { + return false + } + if w.budget <= 0 { + if w.unresolved == "" { + w.unresolved = "the traversal ran out of budget before it finished, so this type's carriers were never all examined" + } + return false + } + w.budget-- + if !resolved(t) { + if w.depth > 0 { + // Invalid INSIDE a graph: a carrier the gate cannot see into. + w.unresolved = "go/types could not resolve a type inside this one, so nothing can be decided about what it carries" + } + // At the root, an invalid type is an expression that is not a value at + // all — a package name, a callee identifier. There is nothing to + // decide, and reporting it would fire on every file. + return false + } + if w.seen[t] { + return false + } + w.seen[t] = true + w.depth++ + defer func() { w.depth-- }() + + if w.match(t) { + return true + } + + // An EXPORTED METHOD handing one back is the same exposure with a call in + // the way, on the value and on a pointer to it. A method result is not + // STORAGE, so this is the exposure graph only. + for _, receiver := range []types.Type{t, types.NewPointer(t)} { + if w.storage { + break + } + set := types.NewMethodSet(receiver) + for i := range set.Len() { + method := set.At(i).Obj() + if !method.Exported() { + continue + } + signature, ok := method.Type().(*types.Signature) + if !ok { + continue + } + for r := range signature.Results().Len() { + if w.reach(signature.Results().At(r).Type()) { + return true + } + } + } + } + + // A NAMED type carries its underlying type and its type ARGUMENTS: an + // instantiated generic holds whatever it was instantiated with. + if named, ok := types.Unalias(t).(*types.Named); ok { + if args := named.TypeArgs(); args != nil { + for i := range args.Len() { + if w.reach(args.At(i)) { + return true + } + } + } + if w.foreignOpaque { + if pkg := named.Obj().Pkg(); pkg != nil && pkg.Path() != w.local { + // Another package's type, and not a match itself. Its insides + // are its own API. + return false + } + } + } + + under := types.Unalias(t).Underlying() + if w.storage { + // Only what the value itself holds. + switch held := under.(type) { + case *types.Struct: + for i := range held.NumFields() { + field := held.Field(i) + if w.fields == reachableFields && !field.Exported() && !field.Embedded() { + continue + } + if w.reach(field.Type()) { + return true + } + } + case *types.Array: + if w.reach(held.Elem()) { + return true + } + case *types.Pointer: + if w.indirect && w.reach(held.Elem()) { + return true + } + case *types.Slice: + if w.indirect && w.reach(held.Elem()) { + return true + } + case *types.Map: + if w.indirect && (w.reach(held.Key()) || w.reach(held.Elem())) { + return true + } + case *types.TypeParam: + if constraint := held.Constraint(); constraint != nil && w.reach(constraint) { + return true + } + } + return false + } + switch under := under.(type) { + case *types.Signature: + // A CALLABLE hands back whatever it returns. `func() any` is a handler + // one call away when the body builds one. + for i := range under.Results().Len() { + if w.reach(under.Results().At(i).Type()) { + return true + } + } + case *types.Interface: + // An interface hands back its methods' results. + for i := range under.NumMethods() { + signature, ok := under.Method(i).Type().(*types.Signature) + if !ok { + continue + } + for r := range signature.Results().Len() { + if w.reach(signature.Results().At(r).Type()) { + return true + } + } + } + case *types.Pointer: + if w.reach(under.Elem()) { + return true + } + case *types.Slice: + if w.reach(under.Elem()) { + return true + } + case *types.Array: + if w.reach(under.Elem()) { + return true + } + case *types.Map: + // Both halves: a map keyed by a carrier carries it too. + if w.reach(under.Key()) || w.reach(under.Elem()) { + return true + } + case *types.Chan: + if w.reach(under.Elem()) { + return true + } + case *types.Struct: + for i := range under.NumFields() { + field := under.Field(i) + if w.fields == reachableFields && !field.Exported() && !field.Embedded() { + continue + } + if w.reach(field.Type()) { + return true + } + } + case *types.TypeParam: + // A type parameter carries whatever its constraint admits. + if terms := under.Constraint(); terms != nil { + if w.reach(terms) { + return true + } + } + } + return false +} + +// exposesMountableHandler: does a value of this type IS or CARRY something +// mountable, anywhere in its type graph? +func (g typedGate) exposesMountableHandler(t types.Type) (bool, string) { + return g.walk(t, g.mountableHandlerType, reachableFields) +} + +// opaqueForHandler: does this type carry an interface A HANDLER COULD BE +// STORED IN — the point at which static types stop answering? +// +// It runs through the same walk, so `func() any`, `[]any`, `struct{ H any }` +// and a method returning one are all one case. Reaching callable results is +// what closed the escape: the old predicate had no signature traversal, so a +// `func() any` result skipped the body check entirely. +func (g typedGate) opaqueForHandler(t types.Type) (bool, string) { + return g.walk(t, func(candidate types.Type) bool { + iface, ok := types.Unalias(candidate).Underlying().(*types.Interface) + if !ok || g.handler == nil { + return false + } + // `any` can hold a handler; `error` cannot, and every function returns + // one. Asking whether http.Handler implements the interface is the + // type-driven form of "could a handler hide here". + return types.Implements(g.handler, iface) + }, reachableFields) +} + +// carriesCoreWire: does this type IS or CARRY one of core's wire messages? +// +// Every field counts, exported or not: allocating a struct allocates its +// unexported fields, so `struct{ seal wc.SealedValues }` constructs a seal as +// much as an exported one does. This is what closed the concrete wrapper — +// `new(reviewBox)` where reviewBox holds a SealedValues allocates a capability, +// and pointer-stripping plus a top-level package check could not see it. +// decodesIntoCoreWire is carriesCoreWire for a DESTINATION, which a decoder +// allocates through: a `*struct{ Seal *SealedValues }` destination has a seal +// written into it even though constructing that struct allocates none. +func (g typedGate) decodesIntoCoreWire(t types.Type) (string, string) { + var carried string + _, unresolved := g.walk(t, func(candidate types.Type) bool { + if path := coreWireType(candidate); path != "" { + carried = path + return true + } + return false + }, everyField, withinThisPackage(g.pkgPath), throughIndirection) + return carried, unresolved +} + +func (g typedGate) carriesCoreWire(t types.Type) (string, string) { + var carried string + _, unresolved := g.walk(t, func(candidate types.Type) bool { + if path := coreWireType(candidate); path != "" { + carried = path + return true + } + return false + }, everyField, withinThisPackage(g.pkgPath), storageGraph) + return carried, unresolved +} + +// TestNoExportedPathHandsOutAServableHandlerByType is the handler-export gate, +// on resolved types. +func TestNoExportedPathHandsOutAServableHandlerByType(t *testing.T) { + for _, pkg := range gatePackages(t, nil) { + if strings.HasSuffix(pkg.PkgPath, "/passthroughtest") { + // passthroughtest is where a consumer's TEST is supposed to get a + // handler; it refuses a non-test binary instead, which + // TestTheSeamRefusesANonTestBinary drives. + continue + } + for _, finding := range reportHandlerShapes(t, pkg) { + t.Error(finding) + } + } +} + +// reportHandlerShapes is every exported declaration in one package that hands +// out something servable. Factored out so the compiled fixtures below drive +// THIS rule rather than a copy of its reasoning. +func reportHandlerShapes(_ *testing.T, pkg *packages.Package) []string { + g := gateFor(pkg) + var findings []string + report := func(format string, args ...any) { + findings = append(findings, fmt.Sprintf(format, args...)) + } + servable := func(where, name string, signature types.Type) { + fn, ok := signature.(*types.Signature) + if !ok { + return + } + for i := range fn.Results().Len() { + result := fn.Results().At(i).Type() + switch { + case g.handedOutDirectly(result): + report("%s exports %s returning %s: a servable handler obtained outside Serve has skipped validate(), the mTLS boot, the caller allow-list, the ceiling and authenticated outbound. Let Serve mount it.", + where, name, types.TypeString(result, nil)) + case isEmptyInterface(result): + report("%s exports %s returning the empty interface: nothing can tell from the signature whether what comes back is servable. Declare the concrete type.", + where, name) + default: + exposes, unresolved := g.exposesMountableHandler(result) + if unresolved != "" { + // FAIL CLOSED. A traversal that could not finish is not a + // traversal that found nothing — the previous walks + // returned false past a depth cap, so nesting escaped + // them. + report("%s exports %s returning %s and the gate COULD NOT DECIDE what it carries: %s. Declare a concrete type the gate can resolve.", + where, name, types.TypeString(result, nil), unresolved) + } + if exposes { + report("%s exports %s returning %s, which exposes a servable handler through a reachable field, element, callable result or method: a caller reaches it and has a handler built without the boot.", + where, name, types.TypeString(result, nil)) + } + } + } + } + // A HANDLER FLOWING INTO AN OPAQUE RESULT. + // + // `func (s *Server) Routes() struct{ H any } { return struct{ H any }{H: + // s.wrapRequest(…)} }` cannot be decided from the result type: `any` + // carries no handler statically, and refusing every interface-typed field + // would refuse Operation.Request and Operation.Response, which are `any` + // by design and hold a consumer's message rather than anything built here. + // + // So the VALUE is followed instead of the declaration. If an exported + // function's results contain an interface anywhere — the point at which + // the type system stops being able to answer — and its body has a value in + // hand whose type IS mountable, then this function demonstrably builds a + // handler and hands it out through a hole static types cannot see into. + // That is the fail-closed direction, and it is narrow: a function whose + // results are all concrete is judged by the walk, and a function that + // builds no handler is not judged at all. + for _, file := range pkg.Syntax { + for _, decl := range file.Decls { + fn, ok := decl.(*ast.FuncDecl) + if !ok || fn.Body == nil || !fn.Name.IsExported() || fn.Type.Results == nil { + continue + } + g := gateFor(pkg) + opaque, unresolved := false, "" + for _, result := range fn.Type.Results.List { + carries, why := g.opaqueForHandler(pkg.TypesInfo.TypeOf(result.Type)) + opaque = opaque || carries + if why != "" && unresolved == "" { + unresolved = why + } + } + if unresolved != "" { + report("%s exports %s and the gate COULD NOT DECIDE whether its results carry an interface a handler could hide in: %s.", + pkg.PkgPath, fn.Name.Name, unresolved) + } + if !opaque { + continue + } + ast.Inspect(fn.Body, func(node ast.Node) bool { + expr, ok := node.(ast.Expr) + if !ok { + return true + } + held := pkg.TypesInfo.TypeOf(expr) + if held == nil || !g.mountableHandlerType(held) { + return true + } + report("%s exports %s, whose results carry an interface and whose body holds a %s: a handler built here is handed out through a field or result no static type can see into, and a caller asserts it back out. Let Serve mount it.", + pkg.PkgPath, fn.Name.Name, types.TypeString(held, nil)) + return false + }) + } + } + scope := pkg.Types.Scope() + for _, name := range scope.Names() { + object := scope.Lookup(name) + if !object.Exported() { + continue + } + switch typed := object.(type) { + case *types.Func: + servable(pkg.PkgPath, name, typed.Type()) + case *types.Var, *types.Const: + exposes, unresolved := g.exposesMountableHandler(object.Type()) + if unresolved != "" { + report("%s exports %s and the gate COULD NOT DECIDE what its type carries: %s.", pkg.PkgPath, name, unresolved) + } + if exposes { + report("%s exports %s, whose type exposes a servable handler: a caller holding it serves the viewer's bearer and this workload's credential over whatever it is mounted on.", + pkg.PkgPath, name) + } + case *types.TypeName: + for _, receiver := range []types.Type{object.Type(), types.NewPointer(object.Type())} { + set := types.NewMethodSet(receiver) + for i := range set.Len() { + method := set.At(i).Obj() + if method.Exported() { + servable(pkg.PkgPath, name+"."+method.Name(), method.Type()) + } + } + } + } + } + return findings +} + +func isEmptyInterface(t types.Type) bool { + iface, ok := types.Unalias(t).Underlying().(*types.Interface) + return ok && iface.NumMethods() == 0 && iface.IsMethodSet() +} + +// TestNoCapabilityIsConstructedOrDecodedByType is the capability gate, on +// resolved types: nothing here allocates one of core's wire messages, and +// nothing decodes into one. +// +// Reading one is untouched — holdSealedIdentity takes a *SealedValues and reads +// it through GetInstallationId — because the rule is on allocation and on +// decode destinations, not on the type appearing. +func TestNoCapabilityIsConstructedOrDecodedByType(t *testing.T) { + for _, pkg := range gatePackages(t, nil) { + for _, finding := range reportCapabilityShapes(t, pkg) { + t.Error(finding) + } + } +} + +// reportCapabilityShapes is every construction of, and decode into, one of +// core's wire messages in one package. The fixtures below drive this rule. +func reportCapabilityShapes(_ *testing.T, pkg *packages.Package) []string { + var findings []string + report := func(format string, args ...any) { + findings = append(findings, fmt.Sprintf(format, args...)) + } + info := pkg.TypesInfo + g := gateFor(pkg) + // Every instantiation in the package, walked for what its type arguments + // CARRY — not merely what they are. A generic's `new(T)` sees a type + // parameter, so the instantiation is where the concrete type exists; and + // `alloc[box, *box]()` carries the capability one field inside box, which + // is where two type parameters and an alias hid it. + for ident, instance := range info.Instances { + if instance.TypeArgs == nil { + continue + } + for i := range instance.TypeArgs.Len() { + path, unresolved := g.carriesCoreWire(instance.TypeArgs.At(i)) + if unresolved != "" { + report("%s instantiates %s and the gate COULD NOT DECIDE what the type argument carries: %s.", pkg.PkgPath, ident.Name, unresolved) + continue + } + if path != "" { + report("%s instantiates %s with a type carrying %s: a generic over one of core's wire messages allocates and decodes one wherever it is used, and the allocation site sees only a type parameter. Read the seal through Credential.Seal().", + pkg.PkgPath, ident.Name, path) + } + } + } + for _, file := range pkg.Syntax { + // Per declaration, so a finding names the function it is in. An + // assertion per SHAPE is possible then, where a count over the whole + // file let a missing rule hide behind the other shapes' findings — + // which is exactly what mutant T6 showed: dropping `Decode` from the + // matcher removed one finding and the count still passed. + for _, decl := range file.Decls { + where := pkg.PkgPath + if fn, ok := decl.(*ast.FuncDecl); ok { + where += "." + fn.Name.Name + } + inspectForCapabilities(g, decl, where, info, report) + } + } + return findings +} + +// inspectForCapabilities is the capability rule over one declaration. +// reportCarrier asks one of the carrier walks and turns its answer — including +// "could not decide" — into a finding. Fail-closed is the default: a gate that +// cannot see is not a gate that passes. +func reportCarrier(ask func(types.Type) (string, string), t types.Type, said, where string, report func(string, ...any), args ...any) { + path, unresolved := ask(t) + if unresolved != "" { + report("%s: the gate COULD NOT DECIDE whether %s carries one of core's wire messages: %s. Use a concrete type the gate can resolve.", + where, types.TypeString(t, nil), unresolved) + return + } + if path == "" { + return + } + report(said, append(args, path)...) +} + +func inspectForCapabilities(g typedGate, decl ast.Node, where string, info *types.Info, report func(string, ...any)) { + // EVERY question here goes through the same fixed-point walk, so a + // CONCRETE WRAPPER is one case with everything else. `type box struct{ + // Seal wc.SealedValues }` allocated with `new(box)` constructs a + // capability, and decoding into a `*box` decodes one — pointer-stripping + // plus a top-level package check saw neither, because the wire type was + // one field in. + carries := func(t types.Type, said string, args ...any) { + reportCarrier(g.carriesCoreWire, t, said, where, report, args...) + } + // A DESTINATION is reached through indirection: a decoder allocates + // through the pointers and slices in it. + destination := func(t types.Type, said string, args ...any) { + reportCarrier(g.decodesIntoCoreWire, t, said, where, report, args...) + } + ast.Inspect(decl, func(node ast.Node) bool { + switch typed := node.(type) { + case *ast.CompositeLit: + carries(info.TypeOf(typed), + "%s constructs a value carrying %s: allocating one of core's wire messages is what decoding into it requires, and a second reader of that encoding is this boundary's whole subject. Read the seal through Credential.Seal().", + where) + case *ast.ValueSpec: + if typed.Type == nil { + return true + } + carries(info.TypeOf(typed.Type), + "%s declares a value carrying %s, whose zero value allocates core's wire message: see Credential.Seal().", + where) + case *ast.CallExpr: + if called, ok := typed.Fun.(*ast.Ident); ok && called.Name == "new" && len(typed.Args) == 1 { + carries(info.TypeOf(typed.Args[0]), + "%s allocates a value carrying %s with new(): see Credential.Seal().", where) + return true + } + // A GENERIC is asked at its INSTANTIATION, because `new(T)` sees + // only a type parameter. The type ARGUMENT is then walked, so + // `alloc[box, *box]()` is caught through box's own field — two + // type parameters and an alias deep. + if ident, ok := typed.Fun.(*ast.Ident); ok { + if instance, instantiated := info.Instances[ident]; instantiated && instance.TypeArgs != nil { + for i := range instance.TypeArgs.Len() { + carries(instance.TypeArgs.At(i), + "%s instantiates %s with a type carrying %s: a generic over one of core's wire messages allocates and decodes one wherever it is used, and the allocation site sees only a type parameter. Read the seal through Credential.Seal().", + where, ident.Name) + } + } + } + if selector, ok := typed.Fun.(*ast.SelectorExpr); ok { + if instance, instantiated := info.Instances[selector.Sel]; instantiated && instance.TypeArgs != nil { + for i := range instance.TypeArgs.Len() { + carries(instance.TypeArgs.At(i), + "%s instantiates %s with a type carrying %s: see Credential.Seal().", + where, selector.Sel.Name) + } + } + } + // HANDING A CAPABILITY TO SOMETHING THAT TAKES IT AS AN OPAQUE + // MESSAGE is handing it to a codec, and that is decided by the + // callee's SIGNATURE rather than its name — so a decoder reached + // through a function variable is one case with a named one. + // `holdSealedIdentity(seal *workcontext.SealedValues)` takes it + // CONCRETELY and is the supported way to read one, so it is not + // flagged. + if signature, ok := types.Unalias(info.TypeOf(typed.Fun)).(*types.Signature); ok { + for i, arg := range typed.Args { + parameter := parameterAt(signature, i) + if parameter == nil { + continue + } + if _, opaque := types.Unalias(parameter).Underlying().(*types.Interface); !opaque { + continue + } + // The carried path is the LAST argument reportCarrier + // appends, so it is the last placeholder here. It was + // second, and the refusal printed + // `%!s(int=1)` and `%!d(string=…/WorkSealV1)` — a garbled + // message on the one path whose job is to tell an author + // exactly what they did. + destination(info.TypeOf(arg), + "%s hands a value to something that takes it as an opaque message (parameter %d is an interface), and that value carries %s: that is a codec, whatever the call is spelled like. Read the seal through Credential.Seal().", + where, i) + } + } + selector, ok := typed.Fun.(*ast.SelectorExpr) + if !ok { + return true + } + // Any decoder, by the shape of its name. The DESTINATION decides, + // and its type is resolved — an assignment, a parameter, a field, + // a package-level alias and a function-local alias are one case. + if !strings.Contains(selector.Sel.Name, "Unmarshal") && !strings.Contains(selector.Sel.Name, "Decode") { + return true + } + // The RECEIVER is a destination too: `b.Decode(raw)` on a + // `*box` decodes into box, and only the arguments were examined. + destinations := append([]ast.Expr{selector.X}, typed.Args...) + for _, arg := range destinations { + destination(info.TypeOf(arg), + "%s decodes into a value carrying %s: that is a second reader of core's wire encoding, whoever allocated the destination. Read the seal through Credential.Seal().", + where) + } + } + return true + }) +} + +// TestTheTypedGatesCatchEveryKnownShape drives the typed rules against every +// shape five review rounds produced — in fixtures that COMPILE. +// +// The overlay puts the fixture into this very package, so it is type-checked +// against the real `*Server`, the real `Handler`, and the pinned SDK. That is +// the point: the syntactic probes only parsed their text, so two of them +// asserted against `workcontext.WorkContextV1`, a type the SDK does not +// export, and reported for three rounds that the whole-capability rule worked. +// A fixture that does not compile now fails the load, loudly, before any +// assertion runs. +func TestTheTypedGatesCatchEveryKnownShape(t *testing.T) { + dir, err := filepath.Abs(".") + if err != nil { + t.Fatal(err) + } + fixture := filepath.Join(dir, "zz_typed_gate_fixture.go") + + // Every handler-export shape, each in its own exported declaration so the + // gate names it. R13 G5u, R14 B4, R15 F2 and the earlier rounds' shapes. + handlers := `package solution + +import ( + nh "net/http" +) + +type fixtureHidden = nh.Handler +type fixtureDefined nh.Handler +type fixtureChain = fixtureHidden +type fixtureServes func(nh.ResponseWriter, *nh.Request) +type fixtureCarrier struct{ H fixtureHidden } +type fixtureNested struct{ In fixtureCarrier } +type fixtureEmbeds struct{ nh.Handler } + +// An embedded type whose own NAME is unexported still promotes its exported +// field, so a caller reaches .H. The field walk's embedded-field clause is +// what covers it: without this shape, mutant T4 (dropping that clause) went +// uncaught, because every other embedded fixture happened to have an exported +// type name. (No backticks here: this comment lives inside a raw string.) +type fixtureEmbedsUnexported struct{ fixtureCarrier } +type fixtureUnexported struct{ H nh.Handler } +type fixtureByMethod struct{} + +func (fixtureByMethod) Mux() nh.Handler { return nil } + +func (s *Server) FixtureAliasResult() fixtureHidden { return nil } +func (s *Server) FixtureDefinedResult() fixtureDefined { return nil } +func (s *Server) FixtureChainResult() fixtureChain { return nil } +func (s *Server) FixtureServesResult() fixtureServes { return nil } +func (s *Server) FixtureBareHandler() Handler { return nil } +func (s *Server) FixtureRequestHandler() RequestHandler { return nil } +func (s *Server) FixtureCollection() map[string]fixtureHidden { return nil } +func (s *Server) FixtureCarrier() fixtureCarrier { return fixtureCarrier{} } +func (s *Server) FixtureNested() fixtureNested { return fixtureNested{} } +func (s *Server) FixtureEmbeds() fixtureEmbeds { return fixtureEmbeds{} } +func (s *Server) FixtureEmbedsUnexported() fixtureEmbedsUnexported { + return fixtureEmbedsUnexported{} +} +func (s *Server) FixtureUnexportedCarrier() fixtureUnexported { return fixtureUnexported{} } +func (s *Server) FixtureAnonymous() struct{ H nh.Handler } { return struct{ H nh.Handler }{} } +func (s *Server) FixtureAnonymousAliased() struct{ H fixtureHidden } { return struct{ H fixtureHidden }{} } +func (s *Server) FixtureAnonymousNested() struct{ In struct{ H nh.Handler } } { + return struct{ In struct{ H nh.Handler } }{} +} +func (s *Server) FixtureAnonymousEmbeds() struct{ nh.Handler } { return struct{ nh.Handler }{} } +func (s *Server) FixtureAnonymousSlice() []struct{ H nh.Handler } { return nil } +func (s *Server) FixtureByMethod() fixtureByMethod { return fixtureByMethod{} } +func (s *Server) FixtureAny() any { return nil } +func (s *Server) FixtureRoundTripper() nh.RoundTripper { return nil } + +// The CONTROL: a declaration type, which is how a consumer writes routes. +// Operation.Handler is this package's own Handler, and it needs a *Gateway +// that only a booted request produces — so one carried here is inert, and +// refusing it would refuse the documented API. It must NOT be flagged. +type fixtureDeclaration struct { + Handler Handler + RequestHandler RequestHandler +} + +func (s *Server) FixtureDeclaration() []fixtureDeclaration { return nil } +` + wantFlagged := []string{ + "FixtureAliasResult", "FixtureDefinedResult", "FixtureChainResult", "FixtureServesResult", + "FixtureBareHandler", "FixtureRequestHandler", "FixtureCollection", + "FixtureCarrier", "FixtureNested", "FixtureEmbeds", "FixtureUnexportedCarrier", + "FixtureAnonymous", "FixtureAnonymousAliased", "FixtureAnonymousNested", + "FixtureAnonymousEmbeds", "FixtureAnonymousSlice", "FixtureEmbedsUnexported", + "FixtureByMethod", "FixtureAny", "FixtureRoundTripper", + } + flagged := gateFindings(t, fixture, handlers, reportHandlerShapes) + for _, want := range wantFlagged { + if !strings.Contains(flagged, want) { + t.Errorf("the handler gate did not flag %s: that shape hands out a handler built without the boot", want) + } + } + // And the one that must NOT be flagged: a struct a consumer DECLARES + // routes with. Operation.Handler is this package's own Handler, which + // needs a *Gateway only a booted request produces. + if strings.Contains(flagged, "FixtureDeclaration") { + t.Errorf("the handler gate flagged a declaration type, which is the documented way to use this package: %s", flagged) + } + + // Every capability shape: R13's supplied destination, R14's zero value, + // R15's assignment and function-local aliases, Decode, and the SDK's real + // whole-capability alias. + capabilities := `package solution + +import ( + "encoding/json" + "io" + + "google.golang.org/protobuf/proto" + wc "github.com/codefly-dev/sdk-go/workcontext" +) + +type fixtureSeal = wc.SealedValues + +func fixtureSupplied(raw []byte, dst *wc.SealedValues) error { return proto.Unmarshal(raw, dst) } +func fixtureZeroValue(raw []byte) error { + var dst wc.SealedValues + return proto.Unmarshal(raw, &dst) +} +func fixtureAssigned(raw []byte, dst *wc.SealedValues) error { + target := dst + return proto.Unmarshal(raw, target) +} +func fixtureTwoHops(raw []byte, dst *wc.SealedValues) error { + a := dst + b := a + return proto.Unmarshal(raw, b) +} +func fixturePackageAlias(raw []byte) error { + var dst fixtureSeal + return proto.Unmarshal(raw, &dst) +} +func fixtureLexicalAlias(raw []byte) error { + type localSeal = wc.SealedValues + var dst localSeal + return proto.Unmarshal(raw, &dst) +} +func fixtureWholeCapability(raw []byte) error { + var dst wc.Claims + return proto.Unmarshal(raw, &dst) +} +func fixtureOperationBinding(raw []byte, dst *wc.SealedOperationBinding) error { + return proto.Unmarshal(raw, dst) +} +func fixtureJSONDecode(r io.Reader, dst *wc.SealedValues) error { + return json.NewDecoder(r).Decode(dst) +} +func fixtureLiteral() *wc.SealedValues { return &wc.SealedValues{} } +func fixtureNew() *wc.Claims { return new(wc.Claims) } +func fixtureThroughAField(raw []byte, holder *struct{ Seal *wc.SealedValues }) error { + return proto.Unmarshal(raw, holder.Seal) +} +func fixtureAccessorOnly(seal *wc.SealedValues) string { return seal.GetInstallationId() } +` + decoded := gateFindings(t, fixture, capabilities, reportCapabilityShapes) + // ONE ASSERTION PER SHAPE, by the function it is in. + // + // This was a count, and mutant T6 showed why that is not enough: dropping + // `Decode` from the matcher removed one finding out of thirteen and the + // count still passed. A missing rule hid behind the other shapes. + for _, want := range []string{ + "fixtureSupplied", "fixtureZeroValue", "fixtureAssigned", "fixtureTwoHops", + "fixturePackageAlias", "fixtureLexicalAlias", "fixtureWholeCapability", + "fixtureOperationBinding", "fixtureJSONDecode", "fixtureLiteral", + "fixtureNew", "fixtureThroughAField", + } { + if !strings.Contains(decoded, want) { + t.Errorf("the capability gate did not flag %s, so that construction or decode shape escapes:\n%s", want, decoded) + } + } + // And the one that must NOT be flagged: reading a seal through the SDK's + // accessors, which is what credential.go does. + if strings.Contains(decoded, "fixtureAccessorOnly") { + t.Errorf("the capability gate flagged reading a seal through Credential.Seal()'s accessors, which is the one supported way to do it:\n%s", decoded) + } +} + +// parameterAt is the type of the parameter one argument lands on, accounting +// for a variadic tail. +func parameterAt(signature *types.Signature, i int) types.Type { + params := signature.Params() + if i < params.Len() { + if signature.Variadic() && i == params.Len()-1 { + if slice, ok := params.At(i).Type().(*types.Slice); ok { + return slice.Elem() + } + } + return params.At(i).Type() + } + if signature.Variadic() && params.Len() > 0 { + if slice, ok := params.At(params.Len() - 1).Type().(*types.Slice); ok { + return slice.Elem() + } + } + return nil +} + +// gateFindings loads this package with one extra COMPILED file and returns +// whatever the given rule reports about it. +func gateFindings(t *testing.T, path, source string, rule func(*testing.T, *packages.Package) []string) string { + t.Helper() + // A sub-test collects the findings instead of failing: these fixtures are + // SUPPOSED to be refused. + var findings []string + for _, pkg := range gatePackages(t, map[string][]byte{path: []byte(source)}) { + if pkg.PkgPath != "github.com/codefly-dev/solution-runtime-go" { + continue + } + findings = append(findings, rule(t, pkg)...) + } + return strings.Join(findings, "\n") + "\n" +} + +// TestTheTypedGateRefusesAFixtureThatDoesNotCompile is the check the syntactic +// probes did not have, and the reason three rounds of evidence were false. +// +// Two of those probes asserted against `workcontext.WorkContextV1`. The SDK +// does not export that name — it exports `Claims` — and because a probe only +// PARSED its fixture, a type that cannot exist reported that the +// whole-capability rule worked. Here a fixture naming a nonexistent type must +// fail the load, so a fixture can never again be evidence for a rule it did +// not reach. +func TestTheTypedGateRefusesAFixtureThatDoesNotCompile(t *testing.T) { + dir, err := filepath.Abs(".") + if err != nil { + t.Fatal(err) + } + fixture := filepath.Join(dir, "zz_typed_gate_noncompiling.go") + // The exact mistake: a type the pinned SDK does not export. + source := `package solution + +import wc "github.com/codefly-dev/sdk-go/workcontext" + +func fixtureNonexistent() *wc.WorkContextV1 { return nil } +` + cfg := &packages.Config{ + Mode: packages.NeedName | packages.NeedFiles | packages.NeedSyntax | + packages.NeedTypes | packages.NeedTypesInfo | packages.NeedDeps | packages.NeedImports, + Overlay: map[string][]byte{fixture: []byte(source)}, + } + loaded, err := packages.Load(cfg, ".") + if err != nil { + t.Fatalf("load: %v", err) + } + errors := 0 + for _, pkg := range loaded { + errors += len(pkg.Errors) + } + if errors == 0 { + t.Fatal("a fixture naming workcontext.WorkContextV1 type-checked, which cannot be: the SDK exports Claims and not that name. If this ever passes, a fixture can assert against a type that does not exist — which is how two probes reported for three rounds that the whole-capability rule worked.") + } + // And the gate helper must REFUSE such a load rather than reporting on it. + // + // In a goroutine of its own, because gatePackages refuses with t.Fatalf — + // which is runtime.Goexit, and Goexit cannot be recovered: run inline it + // terminates this test rather than being observed by it. + refused := &testing.T{} + done := make(chan struct{}) + go func() { + defer close(done) + gatePackages(refused, map[string][]byte{fixture: []byte(source)}) + }() + <-done + if !refused.Failed() { + t.Error("gatePackages accepted a package that does not type-check, so a non-compiling fixture could still supply findings") + } +} + +// TestTheTypedDecodeGateNeedsNoSpelling pins the shapes round sixteen named, +// where the destination or the callee is written in a way no name-matching +// rule follows. +// +// Each of these was reported as an escape of the syntactic rule, and each is +// one case here rather than a branch: a parenthesized destination and a call +// inside a function literal are resolved by `info.TypeOf`, and a decoder +// reached through a function VARIABLE is caught by the callee's signature — +// it takes the capability as an opaque interface, which is what a codec does. +func TestTheTypedDecodeGateNeedsNoSpelling(t *testing.T) { + dir, err := filepath.Abs(".") + if err != nil { + t.Fatal(err) + } + fixture := filepath.Join(dir, "zz_typed_spelling_fixture.go") + source := `package solution + +import ( + "google.golang.org/protobuf/proto" + wc "github.com/codefly-dev/sdk-go/workcontext" +) + +// A parenthesized destination. +func fixtureParenthesized(raw []byte, dst *wc.SealedValues) error { + return proto.Unmarshal(raw, (dst)) +} + +// The decoder reached through a function VARIABLE, so there is no selector to +// match and no name to read. +func fixtureThroughAVariable(raw []byte, dst *wc.SealedValues) error { + decode := proto.Unmarshal + return decode(raw, dst) +} + +// Inside a function literal, with its own binding. +func fixtureInsideAClosure(raw []byte) func() error { + return func() error { + var dst wc.SealedValues + return proto.Unmarshal(raw, &dst) + } +} + +// A decoder whose destination parameter is CONCRETE. The signature rule does +// not fire — nothing is being handed over as an opaque message — so this is +// the shape the name check still earns its keep on, and the reason both rules +// exist. A generated UnmarshalVT-style method has exactly this shape. +type fixtureCodec struct{} + +func (fixtureCodec) DecodeSeal(raw []byte, dst *wc.SealedValues) error { return nil } + +func fixtureConcreteDecoder(raw []byte, dst *wc.SealedValues) error { + return fixtureCodec{}.DecodeSeal(raw, dst) +} + +// The supported read, which must stay unflagged: a concrete parameter, and no +// decoder in sight. +func fixtureSpellingAccessor(seal *wc.SealedValues) string { return seal.GetImageDigest() } +` + found := gateFindings(t, fixture, source, reportCapabilityShapes) + for _, want := range []string{ + "fixtureParenthesized", "fixtureThroughAVariable", "fixtureInsideAClosure", + // Caught by the NAME rule alone: its destination parameter is + // concrete, so the signature rule has nothing to say about it. + "fixtureConcreteDecoder", + } { + if !strings.Contains(found, want) { + t.Errorf("the capability gate did not flag %s, so that spelling still escapes:\n%s", want, found) + } + } + if strings.Contains(found, "fixtureSpellingAccessor") { + t.Errorf("the capability gate flagged a read through the SDK's accessors, which takes the seal as a CONCRETE parameter and is the one supported way to read one:\n%s", found) + } +} + +// TestTheTypedGatesFollowValuesNotShapes pins the confirming round's two gate +// findings, in the reviewer's own shapes. +// +// Both were the class the go/types rebuild was meant to close, and both got +// past it because the traversal was still enumerating shapes it knew — a +// struct, a slice, a map — rather than following every value whose TYPE +// carries a handler or a decoder. An interface-typed field is where the type +// system stops answering, and a generic's allocation site sees a type +// parameter rather than the concrete instantiation. +func TestTheTypedGatesFollowValuesNotShapes(t *testing.T) { + dir, err := filepath.Abs(".") + if err != nil { + t.Fatal(err) + } + + // A handler built HERE, handed out through `any`. The caller mounts + // `Routes().H.(http.Handler)` and has the wrapper without the boot. + opaque := filepath.Join(dir, "zz_typed_opaque_fixture.go") + handlers := gateFindings(t, opaque, `package solution + +import "net/http" + +func (s *Server) FixtureOpaqueField() struct{ H any } { + return struct{ H any }{H: s.wrapRequest(func(*http.Request, *Gateway) (any, error) { return nil, nil })} +} + +func (s *Server) FixtureCallableResult() struct{ H func() http.Handler } { + return struct{ H func() http.Handler }{} +} +`, reportHandlerShapes) + for _, want := range []string{"FixtureOpaqueField", "FixtureCallableResult"} { + if !strings.Contains(handlers, want) { + t.Errorf("the handler gate did not flag %s:\n%s", want, handlers) + } + } + + // A generic allocation, and a decoder reached through a function variable + // whose destination is a DOUBLE pointer. + generic := filepath.Join(dir, "zz_typed_generic_fixture.go") + capabilities := gateFindings(t, generic, `package solution + +import ( + "encoding/json" + + wc "github.com/codefly-dev/sdk-go/workcontext" +) + +func fixtureAlloc[T any]() *T { return new(T) } + +func fixtureReadSeal(raw []byte) (*wc.SealedValues, error) { + dst := fixtureAlloc[wc.SealedValues]() + decode := json.Unmarshal + err := decode(raw, &dst) + return dst, err +} + +// The DESTINATION rule on its own. This codec's parameter is concrete, so the +// opaque-parameter rule does not fire and only dereferencing the destination +// all the way down reaches the wire type. Without it the two rules overlapped +// on one fixture and neither was pinned alone. +type fixtureCodec struct{} + +func (fixtureCodec) UnmarshalInto(dst **wc.SealedValues) {} + +func fixtureHandRoundAboutDestination() { + dst := fixtureAlloc[wc.SealedValues]() + var codec fixtureCodec + codec.UnmarshalInto(&dst) +} +`, reportCapabilityShapes) + // The instantiation is the construction, and the double-pointer + // destination is the decode. Both must be named. + if !strings.Contains(capabilities, "fixtureAlloc") { + t.Errorf("the capability gate did not flag the generic instantiated with core's wire message, so a generic allocation still escapes:\n%s", capabilities) + } + for _, want := range []string{"fixtureReadSeal", "fixtureHandRoundAboutDestination"} { + if !strings.Contains(capabilities, want) { + t.Errorf("the capability gate did not flag %s, so a destination behind two pointers still escapes:\n%s", want, capabilities) + } + } + + // CONTROLS, because both new rules are the fail-closed kind and a gate + // nobody can satisfy gets loosened by whoever hits it next. + controls := filepath.Join(dir, "zz_typed_control_fixture.go") + clean := gateFindings(t, controls, `package solution + +import ( + "net/http" + + wc "github.com/codefly-dev/sdk-go/workcontext" +) + +// `+"`error`"+` is an interface, and every function here returns one. A handler +// cannot be stored in it, so it is not a hole a handler hides in — and this +// one BUILDS a mountable value, which is what makes it discriminating: a rule +// that treated every interface as opaque would flag it, and a mutant that +// removes the "a handler could be stored in it" condition is caught here. +// That is Serve's own shape. +func FixtureBuildsAMuxAndReturnsAnError() error { + mux := http.NewServeMux() + _ = mux + return nil +} + +// `+"`any`"+` fields that hold a consumer's message, which is Operation.Request +// and Operation.Response. Nothing mountable is built here. +func FixtureDeclaresBodies() struct { + Request any + Response any +} { + return struct { + Request any + Response any + }{} +} + +// Reading a seal through the SDK's accessors. +func FixtureReadsASeal(seal *wc.SealedValues) string { return seal.GetInstallationId() } +`, reportHandlerShapes) + if clean != "\n" && strings.TrimSpace(clean) != "" { + t.Errorf("the handler gate refused shapes that are the documented way to use this package:\n%s", clean) + } +} + +// TestTheTypedGatesReachAFixedPoint pins the confirming round's two escapes, +// in the reviewer's own shapes, plus the exhaustion both earlier walks failed +// open on. +// +// These are not two more carriers added to a list. They are the two the fixed +// point exists for: an interface-typed RESULT (the previous rule closed the +// interface-typed FIELD and had no signature traversal at all) and a CONCRETE +// WRAPPER around a capability, behind two type parameters and an alias method +// (the previous rules stripped pointers and checked the top-level package, so +// a wire type one field in was invisible). +func TestTheTypedGatesReachAFixedPoint(t *testing.T) { + dir, err := filepath.Abs(".") + if err != nil { + t.Fatal(err) + } + + // A callable returning an opaque handler, and nine nested arrays. + callable := filepath.Join(dir, "zz_typed_callable_fixture.go") + handlers := gateFindings(t, callable, `package solution + +import "net/http" + +func (s *Server) FixtureCallableOpaque() func() any { + return func() any { + return s.wrapRequest(func(*http.Request, *Gateway) (any, error) { return nil, nil }) + } +} + +// Depth the old walks failed OPEN on: both returned false past their caps, so +// nesting escaped them. The fixed point has no cap. +func (s *Server) FixtureDeeplyNested() [1][1][1][1][1][1][1][1][1]http.Handler { + return [1][1][1][1][1][1][1][1][1]http.Handler{} +} +`, reportHandlerShapes) + for _, want := range []string{"FixtureCallableOpaque", "FixtureDeeplyNested"} { + if !strings.Contains(handlers, want) { + t.Errorf("the handler gate did not flag %s:\n%s", want, handlers) + } + } + + // A concrete wrapper around a capability, behind two type parameters and + // an alias method. + wrapper := filepath.Join(dir, "zz_typed_wrapper_fixture.go") + capabilities := gateFindings(t, wrapper, `package solution + +import ( + "encoding/json" + + wc "github.com/codefly-dev/sdk-go/workcontext" +) + +type fixtureBox struct{ Seal wc.SealedValues } +type fixtureBoxAlias = fixtureBox + +func fixtureTwoParams[T any, P ~*T]() P { return P(new(T)) } + +func (b *fixtureBoxAlias) DecodeFixture(raw []byte) error { return json.Unmarshal(raw, b) } + +func fixtureReadWrapped(raw []byte) (string, error) { + b := fixtureTwoParams[fixtureBoxAlias, *fixtureBoxAlias]() + err := b.DecodeFixture(raw) + return b.Seal.GetInstallationId(), err +} + +// Even the plain allocation, which the reviewer noted escapes too. +func fixturePlainAllocation() *fixtureBoxAlias { return new(fixtureBoxAlias) } + +// A decoder whose own body decodes NOTHING, so the only rule that can catch +// the call below is "the receiver is a destination too". Without it the +// receiver rule rode on the opaque-parameter rule firing inside +// DecodeFixture's body, and neither was pinned alone. +type fixtureReceiverOnly struct{ Seal wc.SealedValues } + +func (b *fixtureReceiverOnly) UnmarshalFixture(raw []byte) error { + b.Seal.InstallationId = string(raw) + return nil +} + +func fixtureDecodeThroughReceiver(raw []byte, b *fixtureReceiverOnly) error { + return b.UnmarshalFixture(raw) +} + +// An UNEXPORTED field is still allocated, so visibility cannot be the rule. +type fixtureHiddenBox struct{ seal wc.SealedValues } + +func fixtureAllocateHidden() fixtureHiddenBox { return fixtureHiddenBox{} } +`, reportCapabilityShapes) + for _, want := range []string{ + "fixtureTwoParams", // the instantiation, through two type parameters + "DecodeFixture", // the receiver as a decode destination + "fixturePlainAllocation", // new() on a wrapper + "fixtureAllocateHidden", // an unexported field + // The receiver as a destination, with nothing else able to catch it. + "fixtureDecodeThroughReceiver", + } { + if !strings.Contains(capabilities, want) { + t.Errorf("the capability gate did not flag %s, so a concrete wrapper still hides a capability:\n%s", want, capabilities) + } + } + + // CONTROLS. Both walks are fail-closed, and a gate nobody can satisfy gets + // loosened by whoever hits it next. These are the shapes production + // actually has. + controls := filepath.Join(dir, "zz_typed_fixedpoint_control_fixture.go") + clean := gateFindings(t, controls, `package solution + +import ( + "context" + + "github.com/codefly-dev/sdk-go/workcontext" +) + +// Holding a source whose method returns the SDK's credential — which is an +// alias for one of core's wire messages — is not constructing one. The first +// run of the fixed point reported authorityHeldSource for exactly this, and +// the finding was mine, not the code's: a method RESULT is not an allocation. +type fixtureHolder struct { + inner CredentialSource + anchor *workcontext.Credential +} + +func fixtureHold(s CredentialSource) *fixtureHolder { return &fixtureHolder{inner: s} } + +func fixtureReadThrough(ctx context.Context, s CredentialSource) (string, error) { + held, err := s.Credential(ctx) + if err != nil { + return "", err + } + return held.Token(), nil +} + +// Reading a seal through the SDK's accessors, which is the supported way. +func fixtureReadSealed(seal *workcontext.SealedValues) string { return seal.GetInstallationId() } +`, reportCapabilityShapes) + if strings.TrimSpace(clean) != "" { + t.Errorf("the capability gate refused shapes production has: holding the SDK's credential is not constructing core's wire message:\n%s", clean) + } +} + +// TestTheWalkRefusesWhatItCannotFinish pins the fail-closed half of the fixed +// point, which no fixture can reach: the budget is 50,000 nodes and a type +// graph that large cannot be written into a fixture, so the branch is asked +// directly. +// +// It matters because the two walks this replaced failed OPEN past a depth cap +// — they returned false, which reads as "carries nothing" — so a gate could be +// exhausted by nesting. A traversal that gives up has not answered, and the +// difference between those two is why this returns a reason alongside its +// verdict. +func TestTheWalkRefusesWhatItCannotFinish(t *testing.T) { + pkgs := gatePackages(t, nil) + if len(pkgs) == 0 { + t.Fatal("no package loaded") + } + g := gateFor(pkgs[0]) + never := func(types.Type) bool { return false } + + // A graph larger than the budget allows. + deep := types.Type(types.Typ[types.String]) + for range 64 { + deep = types.NewPointer(deep) + } + w := &typeWalk{gate: g, match: never, fields: everyField, + seen: map[types.Type]bool{}, budget: 8, indirect: true, storage: true} + if w.reach(deep) { + t.Fatal("the walk matched nothing and said it found something") + } + if w.unresolved == "" { + t.Error(`the walk ran out of budget and reported nothing: a traversal that could not finish must not read as "carries nothing", which is how a depth cap let nesting escape the two walks this replaced`) + } + + // An invalid type INSIDE a graph is the same answer. At the root it is not: + // an expression with no type is not a value to decide about. + w = &typeWalk{gate: g, match: never, fields: everyField, + seen: map[types.Type]bool{}, budget: 50000, indirect: true, storage: true} + w.reach(types.NewPointer(types.Typ[types.Invalid])) + if w.unresolved == "" { + t.Error("a type the checker could not resolve, inside a graph, was skipped rather than reported") + } +} diff --git a/work_context_boundary_test.go b/work_context_boundary_test.go new file mode 100644 index 0000000..44b70cc --- /dev/null +++ b/work_context_boundary_test.go @@ -0,0 +1,792 @@ +package solution + +import ( + "go/ast" + "go/parser" + "go/token" + "io/fs" + "path/filepath" + "strings" + "testing" +) + +// There is one implementation of a Work Context — core's — and this runtime is +// not it. This package obtains a capability through the SDK's mint client, +// carries it, and hands it to whatever verifies it; it never signs one, never +// parses one, and never declares the fields of one. +// +// The gate exists because the duplication it prevents already happened once, in +// a module whose only job was to carry these things: a second signed encoding +// grew inside an SDK beside core's, with its own payload struct, its own +// signer, its own verifier and its own error taxonomy, and 3,532 lines had to +// be deleted to get back to one. Every step of that was locally reasonable. The +// cheapest way to not repeat it is to make the first step fail a test here. +// +// What this checks is narrow on purpose: a signing primitive, and a type or +// function of this package's own that claims to be a Work Context. It does not +// forbid encoding/json — this package serves JSON documents for a living — and +// the one capability-shaped JSON this package does touch, the mint endpoint's +// HTTP bodies, lives in the SDK client and not here. +func TestNoWorkContextImplementationGrowsHere(t *testing.T) { + fset := token.NewFileSet() + // Every source first: a capability type can be aliased in one file and + // decoded into in another. + sources := map[string]*ast.File{} + for _, name := range moduleSources(t) { + file, err := parser.ParseFile(fset, name, nil, parser.ParseComments) + if err != nil { + t.Fatalf("parse %s: %v", name, err) + } + sources[name] = file + } + for name, file := range sources { + // issuesTLSMaterialOnly relaxes the signing-primitive rule for ONE + // narrow shape, and the condition is checked rather than asserted. + // + // sdk-go#51 holds the mint endpoint to the SPIFFE ID in its single URI + // SAN, so the fake host in passthroughtest has to issue a certificate + // carrying one — httptest's shared certificate has no URI SAN at all, + // and a client with AdmittedPeers refuses it, correctly. The seam's + // code lives in non-test files because a consumer imports it from + // their tests, so it cannot hide behind the _test.go exclusion the + // root package's own cell helpers use. + // + // The relaxation is conditional on the file touching NOTHING + // capability-shaped: issuing TLS material has no business importing a + // work-context package, and a file that does both is exactly what this + // gate is for. Every other rule below still applies to it — the + // capability-message construction rule, the verifier rule, the JOSE + // rule and the declaration-name rule — and + // TestTheSigningRelaxationIsConditional pins that. + issuesTLSMaterialOnly := issuesTLSMaterialOnly(name, file) + for _, imported := range file.Imports { + path := strings.Trim(imported.Path.Value, `"`) + switch path { + case "crypto/ed25519", "crypto/ecdsa", "crypto/hmac", "crypto/rsa", "crypto/ed25519/internal/edwards25519": + if issuesTLSMaterialOnly { + continue + } + t.Errorf("%s imports %s: signing a capability is core's, and a runtime that can sign one is a second implementation waiting to happen. Carry the credential the mint client hands you.", + name, path) + } + // Core's generated protobuf package is the other road to a + // capability's fields: holding basev0 means declaring, building + // and decoding core's wire types directly, which is a second + // reader of the encoding whatever it is named. Nothing here needs + // it — the SDK re-exports the one type this runtime reads as + // workcontext.SealedValues, and reads it through accessors. + if strings.Contains(path, "codefly-dev/core/generated/") { + t.Errorf("%s imports %s: core's generated wire types are how a second reader of the encoding starts. The credential's seal is reached through the SDK's Credential.Seal() accessor, which needs no generated import.", + name, path) + } + // A JOSE or raw-segment path reaches the same place by another + // road: the 3,532 lines that had to be deleted were a payload + // struct, a signer, a verifier and an error taxonomy, and none of + // them needed crypto/ed25519 by name. + switch { + case strings.HasPrefix(path, "golang.org/x/crypto"), + strings.Contains(path, "go-jose"), strings.Contains(path, "jwt"), + strings.Contains(path, "jws"), strings.Contains(path, "jwk"): + t.Errorf("%s imports %s: a second signed encoding beside core's is what this boundary exists to prevent, and it does not have to be spelled crypto/ed25519 to be one.", + name, path) + } + } + // Constructing and decoding one of core's capability messages used to + // be checked here, syntactically. Both are in + // typed_boundary_test.go now + // (TestNoCapabilityIsConstructedOrDecodedByType), keyed on core's + // GENERATED PACKAGE PATH rather than on a set of message names — so an + // alias, a function-local alias, an assignment, a supplied destination + // and a message core adds tomorrow are all one case. The list of names + // was the defect: it knew WorkContextV1 and WorkSealV1, the SDK + // exports those as Claims and SealedValues, and three rounds went into + // adding names to it. + + // A declaration of this package's own that names itself after the thing + // core owns. The two this package legitimately has are a refusal it + // returns to a handler and the principals it reports from a mint + // answer — neither is a capability, and both are named below so a third + // has to be argued for rather than appearing. + allowed := map[string]bool{"WorkContextRefusal": true, "WorkContextPrincipals": true} + for _, decl := range file.Decls { + var declared string + switch node := decl.(type) { + case *ast.FuncDecl: + declared = node.Name.Name + case *ast.GenDecl: + for _, spec := range node.Specs { + if typeSpec, ok := spec.(*ast.TypeSpec); ok && strings.Contains(typeSpec.Name.Name, "WorkContext") && !allowed[typeSpec.Name.Name] { + t.Errorf("%s declares the type %s: a Work Context type in this package is core's type copied. Use the SDK's alias of core's, or carry the capability as the string it travels as.", + name, typeSpec.Name.Name) + } + } + continue + } + if strings.Contains(declared, "SignWorkContext") || strings.Contains(declared, "ParseWorkContext") || strings.Contains(declared, "VerifyWorkContext") { + t.Errorf("%s declares %s: signing, parsing and verifying a capability are core's. This runtime presents one and lets the far end decide.", name, declared) + } + } + } +} + +// And the other half of not being an implementation: this runtime does not +// verify, and that is a design position rather than an omission. +// +// Core's verifier requires four live sources — the authorization revision, +// replay, grants and seals — and refuses everything without them, deliberately, +// so that the strongest check in the model cannot be the easiest to skip. A +// solution runtime holds none of those: it is the party presenting a +// capability, not the party that decides on one. So there is no verifier here +// to configure, and a future change that adds one has to answer where those +// four sources come from. +// +// This is not an exemption from Core's conformance kit. The kit runs — against +// the boundary this package does own, carriage, in +// TestCoreConformanceFixturesThroughTheCarrier — and it is what keeps the line +// below checkable: every fixture whose refusal is a property of the bytes in +// hand is refused here with Core's own sentinel, and every fixture whose +// refusal is a judgement against live state is carried, because claiming that +// judgement without the issuer's four sources is the silent downgrade this +// single-implementation rule exists to prevent. +func TestThisRuntimeVerifiesNothing(t *testing.T) { + fset := token.NewFileSet() + for _, name := range moduleSources(t) { + file, err := parser.ParseFile(fset, name, nil, 0) + if err != nil { + t.Fatalf("parse %s: %v", name, err) + } + for _, reached := range verifiersReachedBy(file) { + t.Errorf("%s names %s: verifying a capability needs the issuer's live revision, replay, grant and seal sources, which this runtime does not hold. It presents its own credential and lets the component that holds them decide.", + name, reached) + } + if dotted := dotImportedWorkContext(file); dotted != "" { + // A dot-import makes every name in that package unqualified, so + // nothing above could find them. It is also not a style this + // package uses anywhere. + t.Errorf("%s dot-imports %s: the names a verifier is built from would then be unqualified, which this gate cannot see and a reader cannot either", name, dotted) + } + } +} + +// verifiersReachedBy is every way one file reaches a Work Context verifier, +// rendered as alias.Name. +// +// Separated from the test so the gate can be run against a file the test +// writes, which is the only way to show it catches an escape rather than +// merely passing over this package's current contents. +func verifiersReachedBy(file *ast.File) []string { + var reached []string + for alias := range workContextImports(file) { + if alias == "." { + continue + } + ast.Inspect(file, func(node ast.Node) bool { + selector, ok := node.(*ast.SelectorExpr) + if !ok { + return true + } + pkg, ok := selector.X.(*ast.Ident) + if !ok || pkg.Name != alias { + return true + } + // Every name a verifier can be reached by: the constructor, the + // type (so a declared field or variable is caught as well as a + // call), and the methods. A verifier cannot be obtained without + // naming one of them through the package it lives in, which is + // what makes matching on the name sufficient here — the earlier + // rule matched only the call, so declaring the type and calling + // a method on the value escaped it. + // + // Contains, not HasPrefix: the probe below caught that a prefix + // rule misses NewVerifier, which is the most ordinary name a + // verifier constructor can have. Nothing this package legitimately + // names through these imports contains it. + if strings.Contains(selector.Sel.Name, "Verif") { + reached = append(reached, pkg.Name+"."+selector.Sel.Name) + } + return true + }) + } + return reached +} + +// dotImportedWorkContext is the Work Context package this file dot-imports, if +// it does. +func dotImportedWorkContext(file *ast.File) string { + for alias, path := range workContextImports(file) { + if alias == "." { + return path + } + } + return "" +} + +// TestTheVerifierGateCatchesASubpackage runs the gate against the escapes it +// has actually been shown to miss. +// +// The rule matched two import paths exactly, and +// core/workcontext/conformance — the published conformance kit this suite +// already uses — is neither of them while exporting Verifier(), which returns +// core's own *workcontext.Verifier. So the single import that hands this +// package a working verifier walked past the gate written to refuse exactly +// that. A gate on an enumerated path is only as good as the enumeration. +func TestTheVerifierGateCatchesASubpackage(t *testing.T) { + for _, tc := range []struct { + name string + source string + }{ + { + "the conformance subpackage under its own name", + `package solution +import "github.com/codefly-dev/core/workcontext/conformance" +func f() { _ = conformance.Verifier() }`, + }, + { + "the conformance subpackage under an alias", + `package solution +import kit "github.com/codefly-dev/core/workcontext/conformance" +func f() { _ = kit.Verifier() }`, + }, + { + "a deeper subpackage nobody has written yet", + `package solution +import "github.com/codefly-dev/sdk-go/workcontext/whatever/next" +func f() { _ = next.NewVerifier() }`, + }, + { + "the package itself, which already failed the gate", + `package solution +import "github.com/codefly-dev/core/workcontext" +func f() { _ = workcontext.Verifier() }`, + }, + } { + t.Run(tc.name, func(t *testing.T) { + file, err := parser.ParseFile(token.NewFileSet(), "escape.go", tc.source, 0) + if err != nil { + t.Fatalf("parse the probe: %v", err) + } + if reached := verifiersReachedBy(file); len(reached) == 0 { + t.Error("the gate did not see a verifier this file reaches, so a file shaped like this could verify a capability here with nothing noticing") + } + }) + } + + // And the control: importing the kit without reaching a verifier is not + // flagged. conformance_test.go does exactly this — it reads core's + // published fixtures — so a gate that refused the import itself would + // refuse the thing this suite is built on. + file, err := parser.ParseFile(token.NewFileSet(), "fine.go", `package solution +import "github.com/codefly-dev/core/workcontext/conformance" +func f() { _ = conformance.Fixtures() }`, 0) + if err != nil { + t.Fatalf("parse the control: %v", err) + } + if reached := verifiersReachedBy(file); len(reached) != 0 { + t.Errorf("the gate flagged %v in a file that reaches no verifier: reading core's published fixtures is how this suite gets a capability it did not sign itself", reached) + } +} + +// TestATestingTBParameterIsNotAGate records why the seam's refusal is +// testing.Testing() and not its signature. +// +// testing.TB has an unexported method, which stops a type *declaring* the +// interface — and not a type that EMBEDS it. So a production program can +// satisfy testing.TB in a few lines, which is what the review's reproducer +// did, and a function taking one proves nothing about where it was called +// from. +func TestATestingTBParameterIsNotAGate(t *testing.T) { + // The point is that this line compiles: a struct embedding testing.TB is a + // testing.TB anywhere, including in a deployment, so a function taking one + // proves nothing about where it was called from. + var satisfied testing.TB = struct{ testing.TB }{} + _ = satisfied +} + +// workContextImports is the local name each Work Context package is imported +// under in one file, which is what the gate above has to match on rather than +// the name this package happens to use. +func workContextImports(file *ast.File) map[string]string { + imports := map[string]string{} + for _, imported := range file.Imports { + path := strings.Trim(imported.Path.Value, `"`) + // Prefix, not equality, so a *subpackage* is matched too. The rule was + // written against the two package paths exactly, and + // core/workcontext/conformance is neither of them while exporting + // Verifier(), which returns core's own *workcontext.Verifier — so the + // one import that hands this package a working verifier passed the + // gate meant to refuse exactly that. A gate on a path is only as good + // as the paths it enumerates, which is the same defect as an + // enumerated posture check. + if !isWorkContextPath(path) { + continue + } + alias := path[strings.LastIndex(path, "/")+1:] + if imported.Name != nil { + alias = imported.Name.Name + } + if alias == "_" { + continue + } + imports[alias] = path + } + return imports +} + +// isWorkContextPath reports whether an import path is a Work Context package or +// anything beneath one, in either repository that owns one. +func isWorkContextPath(path string) bool { + for _, root := range []string{ + "github.com/codefly-dev/core/workcontext", + "github.com/codefly-dev/sdk-go/workcontext", + } { + if path == root || strings.HasPrefix(path, root+"/") { + return true + } + } + return false +} + +// The handler-export, capability-decoder and seal-construction rules used to +// live here, syntactically: name lists, rendered type spellings, alias maps and +// a probe list that every review round extended by one shape. They are in +// typed_boundary_test.go now, built on go/types, where an alias, an anonymous +// struct, a supplied destination and an assignment are all resolved rather than +// matched. What stays in this file is what is genuinely syntactic: the import +// rules, the declaration-name rule, the verifier rule and the seam's call +// graph. + +// TestNoExportedPathBuildsACredentialBearingHandlerWithoutTheBoot pins the one +// API this round removed. +// +// Server.PassthroughHandler was exported, and that made a usable +// credential-bearing handler reachable without validate(), the mTLS boot, the +// caller allow-list, the published ceiling or authenticated outbound: a +// deployment calling it completed a viewer mint and a module call over +// plaintext and answered 200, with the viewer's bearer and this workload's own +// credential on the wire. It is reachable from package passthroughtest through +// internal/seam now, which Go's internal-package rule keeps inside this module. +// +// The gate is here because the pressure to put it back is real — it is two +// lines and it makes a consumer's test shorter — and because nothing else +// notices an exported identifier reappearing. +// The gate was two shapes too narrow, and both narrowings were about the +// previous round's removal rather than the property it was removed for. It +// skipped passthroughtest/ wholesale, and it matched only an exported func with +// a *receiver* whose name contained "Passthrough" — so passthroughtest.Handler, +// a plain exported function in a skipped file, was a production-importable path +// to the same credential-bearing handler. It now reads every call to the seam +// in the module and requires the calling function to refuse a non-test binary. +func TestNoExportedPathBuildsACredentialBearingHandlerWithoutTheBoot(t *testing.T) { + reachesTheSeam, refusesANonTestBinary := seamCallGraph(t) + fset := token.NewFileSet() + for _, name := range moduleSources(t) { + file, err := parser.ParseFile(fset, name, nil, 0) + if err != nil { + t.Fatalf("parse %s: %v", name, err) + } + for _, decl := range file.Decls { + switch declared := decl.(type) { + case *ast.FuncDecl: + if strings.Contains(name, "internal/seam") { + continue + } + // Whoever reaches the seam is an exported path to a handler + // holding a real credential, whatever it is called and whether + // it hangs off a receiver or not. + // + // Transitively. The rule matched only a function that calls + // seam.Passthrough *directly*, so an exported Handler + // delegating to an unexported helper — and a root-package + // wrapper over passthroughSeam — both walked past it. One hop + // of indirection is the first thing anyone writes. + if !reachesTheSeam[declared.Name.Name] { + continue + } + if !declared.Name.IsExported() { + continue + } + if !refusesANonTestBinary[declared.Name.Name] { + t.Errorf("%s exports %s, which builds a passthrough through internal/seam without refusing a non-test binary: that handler holds a real execution credential and skips validate(), the mTLS boot, the caller allow-list, the ceiling and authenticated outbound, so an exported path to it has to call mustBeATest()", + name, declared.Name.Name) + } + case *ast.GenDecl: + for _, spec := range declared.Specs { + if typed, ok := spec.(*ast.TypeSpec); ok && typed.Name.Name == "PassthroughEnvironment" { + t.Errorf("%s exports PassthroughEnvironment, the argument of the constructor this change removed", name) + } + // An exported *variable* holding the seam is the same + // bypass with no function to inspect: `var + // BuildPassthrough = passthroughSeam` in the root package, + // or `var Build = seam.Passthrough` in passthroughtest, + // both passed a gate that only read function bodies. + value, ok := spec.(*ast.ValueSpec) + if !ok { + continue + } + for i, declaredName := range value.Names { + if !declaredName.IsExported() || i >= len(value.Values) { + continue + } + if named := renderedReference(value.Values[i]); reachesTheSeam[named] || named == "seam.Passthrough" { + t.Errorf("%s exports the variable %s holding %s: that is a path to a credential-bearing handler with no function body for a gate to read and no refusal in front of it", + name, declaredName.Name, named) + } + } + } + } + } + } +} + +// renderedReference is pkg.Name or Name for an expression that merely refers to +// something, which is what an exported variable's initialiser is. +func renderedReference(expr ast.Expr) string { + switch named := expr.(type) { + case *ast.Ident: + return named.Name + case *ast.SelectorExpr: + if pkg, ok := named.X.(*ast.Ident); ok { + return pkg.Name + "." + named.Sel.Name + } + } + return "" +} + +// seamCallGraph is, for every function in the module, whether it reaches the +// passthrough seam and whether it refuses a non-test binary — both +// transitively, because one hop of indirection is the first thing anyone +// writes and the previous rule only looked at direct calls. +func seamCallGraph(t *testing.T) (reaches, refuses map[string]bool) { + t.Helper() + fset := token.NewFileSet() + var files []*ast.File + for _, name := range moduleSources(t) { + if strings.Contains(name, "internal/seam") { + continue + } + file, err := parser.ParseFile(fset, name, nil, 0) + if err != nil { + t.Fatalf("parse %s: %v", name, err) + } + files = append(files, file) + } + return seamReachability(files) +} + +// seamReachability is seamCallGraph over files already parsed, so a probe can +// supply its own. +func seamReachability(files []*ast.File) (reaches, refuses map[string]bool) { + calls := map[string][]string{} + reaches, refuses = map[string]bool{}, map[string]bool{} + // defeated is every function that swallows a panic or guards a branch it + // cannot take, whether the refusal it defeats is its own call or one it + // inherits from a callee. Recording it for EVERY function, not only for + // the ones calling mustBeATest, is what makes the propagation below + // honest: the defeat check used to run only where the refusal was called + // directly, so the same `if false` moved one hop away was inherited as a + // refusal and passed the gate. Probed both ways in + // TestTheSeamGateIsNotDefeatedOneHopAway. + defeated := map[string]bool{} + for _, file := range files { + for _, decl := range file.Decls { + fn, ok := decl.(*ast.FuncDecl) + if !ok || fn.Body == nil { + continue + } + if defeatsItsOwnRefusal(fn) { + defeated[fn.Name.Name] = true + } + if callsThe(fn, "seam", "Passthrough") { + reaches[fn.Name.Name] = true + } + // A refusal swallowed by recover(), or sitting on a branch that + // cannot be taken, is not a refusal. Counting the call alone made + // both of those read as guarded. + if (callsThe(fn, "", "mustBeATest") || callsThe(fn, "", "refuseOutsideTest")) && !defeated[fn.Name.Name] { + refuses[fn.Name.Name] = true + } + ast.Inspect(fn.Body, func(node ast.Node) bool { + call, ok := node.(*ast.CallExpr) + if !ok { + return true + } + if callee, ok := calleeName(call); ok { + calls[fn.Name.Name] = append(calls[fn.Name.Name], callee) + } + return true + }) + } + } + // Propagate both properties up the call graph until nothing changes. + for changed := true; changed; { + changed = false + for caller, callees := range calls { + for _, callee := range callees { + if reaches[callee] && !reaches[caller] { + reaches[caller], changed = true, true + } + // A caller inherits a callee's refusal only if it does not + // defeat it. `if false { return guard() }` reaches the seam by + // its real path and inherited guard()'s refusal, which is the + // defeat this gate already refused one hop closer in. + if refuses[callee] && !refuses[caller] && !defeated[caller] { + refuses[caller], changed = true, true + } + } + } + } + return reaches, refuses +} + +// defeatsItsOwnRefusal reports whether fn recovers from a panic or guards the +// refusal behind a constant false, either of which makes the call decorative. +func defeatsItsOwnRefusal(fn *ast.FuncDecl) bool { + defeated := false + ast.Inspect(fn, func(node ast.Node) bool { + if call, ok := node.(*ast.CallExpr); ok { + if named, ok := call.Fun.(*ast.Ident); ok && named.Name == "recover" { + defeated = true + } + } + if branch, ok := node.(*ast.IfStmt); ok { + if cond, ok := branch.Cond.(*ast.Ident); ok && cond.Name == "false" { + defeated = true + } + } + return true + }) + return defeated +} + +// callsThe reports whether fn calls pkg.name, or bare name when pkg is empty. +func callsThe(fn *ast.FuncDecl, pkg, name string) bool { + found := false + ast.Inspect(fn, func(node ast.Node) bool { + call, ok := node.(*ast.CallExpr) + if !ok { + return true + } + if pkg == "" { + if called, ok := call.Fun.(*ast.Ident); ok && called.Name == name { + found = true + } + return true + } + selector, ok := call.Fun.(*ast.SelectorExpr) + if !ok { + return true + } + qualifier, ok := selector.X.(*ast.Ident) + if ok && qualifier.Name == pkg && selector.Sel.Name == name { + found = true + } + return true + }) + return found +} + +// moduleSources is every non-test Go source in this module, which is what both +// gates above read. +// +// They read only the root directory until now, and passthroughtest/ is where a +// capability is actually signed — with core's published fixture key, which is +// the right way to do it and exactly the file a second implementation would +// grow in, since it is the one place already holding a signing key. A gate that +// does not look at the directory most likely to break it is a gate that reports +// success for the wrong reason. +func moduleSources(t *testing.T) []string { + t.Helper() + var sources []string + err := filepath.WalkDir(".", func(path string, entry fs.DirEntry, err error) error { + if err != nil { + return err + } + if entry.IsDir() { + // Nothing generated and nothing vendored: those are not this + // repository's declarations to answer for. + if name := entry.Name(); path != "." && (name == "vendor" || strings.HasPrefix(name, ".")) { + return fs.SkipDir + } + return nil + } + if filepath.Ext(path) == ".go" && !strings.HasSuffix(path, "_test.go") { + sources = append(sources, path) + } + return nil + }) + if err != nil { + t.Fatalf("walk module sources: %v", err) + } + if len(sources) < 2 { + t.Fatalf("found %d sources to gate, which cannot be right: a gate that reads nothing passes", len(sources)) + } + return sources +} + +// TestTheSeamGateFollowsIndirection: the rule matched only a function calling +// seam.Passthrough *directly*, so an exported wrapper one hop away passed it — +// and one hop of indirection is the first thing anyone writes. +func TestTheSeamGateFollowsIndirection(t *testing.T) { + parse := func(t *testing.T, source string) []*ast.File { + t.Helper() + file, err := parser.ParseFile(token.NewFileSet(), "probe.go", source, 0) + if err != nil { + t.Fatalf("parse the probe: %v", err) + } + return []*ast.File{file} + } + + t.Run("an exported path one hop from the seam is seen", func(t *testing.T) { + reaches, refuses := seamReachability(parse(t, `package solution +func ExportedIndirect() (http.Handler, error) { return viaHelper() } +func viaHelper() (http.Handler, error) { return seam.Passthrough(nil, "", "") }`)) + if !reaches["ExportedIndirect"] { + t.Error("the gate did not see that ExportedIndirect reaches the seam: it delegates, so a rule matching only direct callers reports an exported path to a credential-bearing handler as if it were not one") + } + if refuses["ExportedIndirect"] { + t.Error("the gate thinks ExportedIndirect refuses a non-test binary, which nothing in it does") + } + }) + + t.Run("a refusal one hop away is seen too", func(t *testing.T) { + // Or the gate would flag a path that is in fact guarded, and a rule + // that cries wolf gets relaxed rather than obeyed. + _, refuses := seamReachability(parse(t, `package solution +func Guarded() (http.Handler, error) { return guard() } +func guard() (http.Handler, error) { mustBeATest(); return seam.Passthrough(nil, "", "") }`)) + if !refuses["Guarded"] { + t.Error("the gate did not see that Guarded refuses a non-test binary through its helper") + } + }) + + t.Run("a function that reaches nothing is not flagged", func(t *testing.T) { + reaches, _ := seamReachability(parse(t, `package solution +func Unrelated() error { return nil }`)) + if reaches["Unrelated"] { + t.Error("the gate flagged a function that does not reach the seam") + } + }) +} + +// TestTheSeamGateIsNotDefeatedOneHopAway is B6: the defeat check ran only where +// mustBeATest was called directly, and the propagation loop then handed the +// refusal to any caller of a refusing callee without asking whether that +// caller defeats it. So the gate caught `if false { mustBeATest() }` and +// passed the same defeat one function away. +func TestTheSeamGateIsNotDefeatedOneHopAway(t *testing.T) { + parse := func(t *testing.T, source string) []*ast.File { + t.Helper() + file, err := parser.ParseFile(token.NewFileSet(), "probe.go", source, 0) + if err != nil { + t.Fatalf("parse the probe: %v", err) + } + return []*ast.File{file} + } + const guard = ` +func guard() (http.Handler, error) { mustBeATest(); return seam.Passthrough(nil, "", "") }` + + defeats := []struct{ shape, source string }{ + { + "the refusal is called directly on a branch that cannot be taken", + `package solution +func Exported() (http.Handler, error) { + if false { mustBeATest() } + return seam.Passthrough(nil, "", "") +}`, + }, + { + "the guarded helper is called on a branch that cannot be taken, and the real path is not", + `package solution +func Exported() (http.Handler, error) { + if false { return guard() } + return seam.Passthrough(nil, "", "") +}` + guard, + }, + { + "the guarded helper's panic is swallowed by the caller", + `package solution +func Exported() (h http.Handler, err error) { + defer func() { recover() }() + return guard() +}` + guard, + }, + } + for _, probe := range defeats { + t.Run(probe.shape, func(t *testing.T) { + _, refuses := seamReachability(parse(t, probe.source)) + if refuses["Exported"] { + t.Errorf("the gate reads Exported as refusing a non-test binary, and it does not — %s:\n%s", probe.shape, probe.source) + } + }) + } + + t.Run("passes: a refusal one hop away that nothing defeats", func(t *testing.T) { + _, refuses := seamReachability(parse(t, `package solution +func Exported() (http.Handler, error) { return guard() }`+guard)) + if !refuses["Exported"] { + t.Error("the gate stopped seeing a genuine refusal through a helper, which would fail every guarded path in the module") + } + }) +} + +// issuesTLSMaterialOnly is THE condition the signing-primitive relaxation +// turns on, as one function. +// +// It was a copy: the gate computed the condition inline and the test that +// claimed to pin it recomputed the same expression. An executed round changed +// the real gate to permit a signing import in every passthroughtest file and +// the test still passed — which makes it worthless as evidence for the gate, +// and it is the exact failure I had written into the gate's own comments as +// the thing to avoid ("a probe that re-implements the check proves only that +// two copies agree"). Struck as evidence and replaced by this. +func issuesTLSMaterialOnly(name string, file *ast.File) bool { + return strings.HasPrefix(name, "passthroughtest/") && + len(workContextImports(file)) == 0 && dotImportedWorkContext(file) == "" +} + +// TestTheSigningRelaxationIsConditional pins the one relaxation in the +// signing-primitive rule, in both directions. +// +// A file under passthroughtest that issues TLS material may hold a key +// primitive, because sdk-go#51 requires the fake mint endpoint to present a +// SPIFFE ID and httptest's shared certificate has none. The same file may NOT +// hold one once it touches anything capability-shaped — a file doing both is +// precisely what this gate exists to refuse, and "it is only the test seam" is +// how that would arrive. +func TestTheSigningRelaxationIsConditional(t *testing.T) { + parse := func(t *testing.T, source string) *ast.File { + t.Helper() + file, err := parser.ParseFile(token.NewFileSet(), "probe.go", source, 0) + if err != nil { + t.Fatalf("parse the probe: %v", err) + } + return file + } + // THE rule the gate uses, not a copy of it. The copy survived the real + // condition being removed, which is why this calls the function the + // scanner calls. + relaxed := issuesTLSMaterialOnly + + tlsOnly := parse(t, `package passthroughtest +import ( + "crypto/ecdsa" + "crypto/x509" +) +func leaf() (*x509.Certificate, *ecdsa.PrivateKey) { return nil, nil }`) + if !relaxed("passthroughtest/identity.go", tlsOnly) { + t.Error("the relaxation does not cover a passthroughtest file that only issues TLS material, so the seam cannot present the SPIFFE ID the SDK now requires of the mint endpoint") + } + if relaxed("credential.go", tlsOnly) { + t.Error("the relaxation reaches the root package: this is where the runtime lives, and a signing primitive here is the first step of the duplication this gate exists to refuse") + } + + alsoCapabilities := parse(t, `package passthroughtest +import ( + "crypto/ecdsa" + "github.com/codefly-dev/sdk-go/workcontext" +) +func sign(seal *workcontext.SealedValues, key *ecdsa.PrivateKey) []byte { return nil }`) + if relaxed("passthroughtest/identity.go", alsoCapabilities) { + t.Error("a passthroughtest file holding BOTH a signing primitive and a work-context import is relaxed: that is a capability signer in the test seam, which is the shape this gate was written for and the one most likely to be argued for as harmless") + } +} diff --git a/zz_load_test.go b/zz_load_test.go new file mode 100644 index 0000000..0f6fc73 --- /dev/null +++ b/zz_load_test.go @@ -0,0 +1,36 @@ +package solution + +import ( + "go/types" + "testing" + + "golang.org/x/tools/go/packages" +) + +func TestZZLoad(t *testing.T) { + cfg := &packages.Config{Mode: packages.NeedName | packages.NeedSyntax | packages.NeedTypes | packages.NeedTypesInfo | packages.NeedDeps | packages.NeedImports} + pkgs, err := packages.Load(cfg, ".", "./passthroughtest") + if err != nil { + t.Fatal(err) + } + for _, p := range pkgs { + t.Logf("%s types=%v errors=%d syntax=%d", p.PkgPath, p.Types != nil, len(p.Errors), len(p.Syntax)) + for _, e := range p.Errors[:min(2, len(p.Errors))] { + t.Logf(" err: %v", e) + } + } + // can we resolve the SDK alias to core's generated type? + for _, p := range pkgs { + if p.Types == nil { + continue + } + for _, imp := range p.Types.Imports() { + if imp.Path() == "github.com/codefly-dev/sdk-go/workcontext" { + o := imp.Scope().Lookup("SealedValues") + t.Logf("SealedValues -> %s", types.TypeString(types.Unalias(o.Type()), nil)) + c := imp.Scope().Lookup("Claims") + t.Logf("Claims -> %s", types.TypeString(types.Unalias(c.Type()), nil)) + } + } + } +}