From cc7b501397811cff42034210cd4dff1c4d6bbe43 Mon Sep 17 00:00:00 2001 From: antoinetoussaint-byte <312536060+antoinetoussaint-byte@users.noreply.github.com> Date: Sun, 11 Oct 2026 05:41:14 -0400 Subject: [PATCH 1/2] =?UTF-8?q?feat!:=20mint=20once=20=E2=80=94=20a=20solu?= =?UTF-8?q?tion=20authenticates=20with=20its=20pod's=20projected=20token?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The platform host no longer accepts a solution registration secret or any identity secret (codefly-dev/module-saas-starter#1068): every mint is bound to the calling pod's projected ServiceAccount token (audience accounts, ≤ 600 s), its running build and its one principal. - The runtime reads /var/run/secrets/codefly/token fresh and bounded before every mint and presents it as the bearer; a missing, empty or unreadable token fails closed. The principal is the solution's declared name. - A refusal (401/403) is terminal and retained; only 503 and transient transport failures are retried. - Mint errors carry fixed messages only; credential-source errors drop diagnostic text and underlying causes. - Breaking: WorkContextRefusal.Code and .Message are removed; use .StatusCode. The registration-secret configuration is deleted. Tests: token defaults, rotation and bounds, unreadable projections, error and log redaction, concurrent refusal retention. The full race suite passes. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 46 +++++ README.md | 42 +++-- credential.go | 11 +- credential_test.go | 20 ++- facade_claim_test.go | 62 +++++-- mcp.go | 7 +- mcp_test.go | 5 +- mint_errors.go | 52 ++++++ mint_migration_test.go | 227 ++++++++++++++++++++++++ passthrough.go | 15 +- passthrough_scopes_test.go | 4 +- passthroughtest/passthroughtest_test.go | 4 +- solution.go | 66 +++---- solution_test.go | 11 +- 14 files changed, 476 insertions(+), 96 deletions(-) create mode 100644 CHANGELOG.md create mode 100644 mint_errors.go create mode 100644 mint_migration_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..d2f5074 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,46 @@ +# Changelog + +## Unreleased — partial mint-once migration + +Breaking API change: `WorkContextRefusal.Code` and `WorkContextRefusal.Message` +are removed. Use `StatusCode` (or unwrap to `GatewayError`) to classify failures. +Errors and passthrough responses no longer expose broker diagnostics. +No registration-secret API remains in the starting revision of this repository. + +The projected token path defaults to `/var/run/secrets/codefly/token`; the CLI +mounts it read-only for audience `accounts` with a lifetime of at most 600 seconds. +The runtime reads the bounded file again before each mint. Missing, empty, +unreadable and oversized projections fail closed. + +The requested `saas-sdk-go` revision +`v0.0.3-0.20261011091114-217bf9fe3f10` cannot build alongside the current +`sdk-go/workcontext`: its `moduleauthority` package requires the removed +`WorkContextToken` and `ParseWorkContextToken` API. Its `Seams` also requires an +internal perimeter token that this runtime no longer provisions. Switching the +broker and supplying the solution's declared name as its prefix remain blocked +on a compatible SDK. The offline `go get` succeeded, but the subsequent +`moduleauthority` build failed on those symbols. The unusable dependency change +was reverted and `go mod tidy` left the original pins unchanged. No SDK source +was copied or modified locally. + +The retained execution SDK marks every non-401/403 HTTP status unavailable and +exposes no typed status for narrowing retries. Therefore the requested retry +policy (only 503 and transient transport failures) is not complete. Execution +refusals still latch; errors from default and consumer-supplied credential sources are now redacted. + +### Removed authentication configuration (archive) + +These identifiers are deleted, not compatibility inputs: +`MODULE_IDENTITY_SECRET`, `MODULE_IDENTITY_SECRETS`, `MODULE_IDENTITY_PREFIX`, +`MODULE_REGISTRATION_SECRETS`, `SOLUTION_REGISTRATION_SECRETS`, +`CODEFLY__SOLUTION_REGISTRATION_SECRET`, `MintModuleRegistration`, and +`MintSolutionRegistration`. + +The compiler-based facade gate reads the two wire spellings below from this +archive and still checks constants, conversions, aliases and compiled fixtures. +Tests never provision a removed identity secret. + +| Kind | Removed wire spelling | +| --- | --- | +| header | `X-Codefly-Module-Secret` | +| carrier | `CODEFLY__MODULE_REGISTRATION_SECRETS` | diff --git a/README.md b/README.md index b878337..eff2894 100644 --- a/README.md +++ b/README.md @@ -62,14 +62,32 @@ Four things follow, and each is a boot failure rather than a degraded run: | The authority-bearing values are read once and frozen | the credential is sealed to the values it was minted under, so a value that drifts is an error, never a reload | | The contract is published, and the declaration is held to it | an ask outside its declared ceiling fails the boot — see [The published contract](#the-published-contract) | +The remaining migration to the platform's module broker is **blocked by SDK +compatibility**: `saas-sdk-go` at `217bf9fe3f10` references a capability API +removed from this runtime's `sdk-go/workcontext`, and requires an internal-token +seam this runtime no longer has. The existing execution SDK remains in use; +its broader retry policy below still needs an upstream change. See +[the migration record](CHANGELOG.md) for the exact blocker. + +`WorkContextRefusal.Code` and `.Message` have been removed (breaking API change). +Consumers should inspect `.StatusCode` or unwrap to `GatewayError`. Broker bodies, +headers, redirect locations and raw transport diagnostics are not returned. +A viewer request retains a 401/403 for the same mint ask and never re-mints it. + +The CLI projects the service-account token read-only at +`/var/run/secrets/codefly/token`, for audience `accounts`, with lifetime at most +600 seconds. That path is the boot default; the existing path configuration is +still available. Token reads are bounded to 64 KiB and repeated for every mint; +missing, empty, unreadable or oversized files fail closed before a request. + ## Handler errors Return `ClientError` for an explicitly public validation message. Connect errors returned by `Unary` retain actionable statuses: authentication 401, permission 403, missing resource 404, invalid input 400, conflict 409, precondition 412, quota 429, unavailable 503 and deadline 504. Other RPC failures return 502. -Only generic status text reaches the browser; wrapped/upstream details remain -available to the handler. No request is retried or replayed. +Only generic status text reaches the browser. Mint failures also discard upstream +diagnostics before reaching handlers or logs. No request is retried or replayed. REST adapters can return `&solution.GatewayError{StatusCode: resp.StatusCode}` after a non-success response through the gateway. The runtime preserves 4xx, @@ -81,7 +99,7 @@ or flattening failures into strings must adopt these typed errors to benefit. ## Configuration -The runtime hardcodes nothing. On boot `Serve` calls +On boot `Serve` calls `codefly.LoadEnvironmentVariables()` to load Codefly's injected carriers, then `loadConfig` resolves every address, port, and path through the codefly Go SDK (`github.com/codefly-dev/sdk-go`), falling back to the local native workspace @@ -93,7 +111,7 @@ The two MCP values have none, because a Codefly composition cannot set a bare environment variable on a service: every value a service receives is either an endpoint the SDK resolves or a **declared configuration** the render projects. Their override is therefore a value in the `mcp` workspace-configuration group, -resolved through the SDK the way the registration secret is — see [The resource +resolved through the SDK — see [The resource identifier and the issuer](#the-resource-identifier-and-the-issuer). | What | SDK resolution | Env override (default) | @@ -101,7 +119,7 @@ identifier and the issuer](#the-resource-identifier-and-the-issuer). | Own listen port | `codefly.For(ctx).Endpoint("http").NetworkInstance()` — the Codefly-assigned port, not a fixed default | `PORT` | | Gateway URL (auth-gateway `rest`) | resolved by role — the single module owning the `auth-gateway` `rest`/`rest` endpoint, discovered from the injected carriers (or the workspace, run locally). Must be `https` | `GATEWAY_URL` | | Credential mint URL | **not resolved — required explicitly.** Must be `https`. Nothing derives it: whether the host mints through the gateway or the workload posts straight to the issuer is unsettled and neither endpoint is deployed, and this is the address the projected service-account token goes to, so an assumed one is refused at boot. Every boot logs where it will mint, and a 404 there means that endpoint does not exist on that host — not that this build was refused | `CODEFLY__CREDENTIAL_MINT_URL` (**required**) | -| Projected service-account token | `codefly.For(ctx).WorkspaceConfiguration("workload-identity", "TOKEN_FILE")` — a **path**, re-read at every mint | `CODEFLY__WORKLOAD_TOKEN_FILE` | +| Projected service-account token | `codefly.For(ctx).WorkspaceConfiguration("workload-identity", "TOKEN_FILE")` — a **path**, re-read at every mint; defaults to `/var/run/secrets/codefly/token`, the CLI projection | `CODEFLY__WORKLOAD_TOKEN_FILE` | | Workload identity certificate | `workload-identity`/`CERT_FILE` | `CODEFLY__WORKLOAD_IDENTITY_CERT_FILE` | | Workload identity private key | `workload-identity`/`KEY_FILE` | `CODEFLY__WORKLOAD_IDENTITY_KEY_FILE` | | Peer trust anchor | `workload-identity`/`TRUST_BUNDLE_FILE` — **required**: the listener requires and verifies a caller's certificate against it, and the outbound client verifies the platform against it | `CODEFLY__WORKLOAD_IDENTITY_TRUST_BUNDLE_FILE` | @@ -164,7 +182,6 @@ even for that is an open follow-up. **Removed in this cutover, with the registrations they fed**: `SELF_UPSTREAM`, `HOST_REGISTER_URL`, `GATEWAY_REGISTER_URL`, `GATEWAY_MODULE_REGISTER_URL`, `GATEWAY_MODULE_REGISTRATION_TOKEN_URL`, `GATEWAY_SOLUTION_REGISTRATION_TOKEN_URL`, `CODEFLY_INTERNAL_TOKEN`, -`CODEFLY__SOLUTION_REGISTRATION_SECRET`, `CODEFLY__MODULE_REGISTRATION_SECRETS`, and `CODEFLY__SOLUTION_REGISTRATION_INTERVAL`. Setting any of them now does nothing. @@ -467,7 +484,7 @@ is waited for, within a bound.** The two are different answers: statuses, `401` and `403`) is the issuer judging this workload: this build is not the one its presence document approved, the pod's identity does not match, the token attests to another subject. No number of attempts changes any of those - answers, so the boot reports what the issuer said and exits non-zero. A loop + answers, so the boot reports the refusal class and exits non-zero. A loop around it is what this change exists to delete: the old runtime answered every refusal by beating again, minting a fresh single-use token each time, so one undeployable solution produced an audited mint every 15 seconds for as long as @@ -538,6 +555,10 @@ A consumer whose issuer is reached another way supplies its own source: solution.New(manifest).Credential(mySource).Serve() // Credential(ctx) (workcontext.Credential, error) ``` +Source errors preserve only the `ErrMintRefused` and `ErrMintUnavailable` +classifications; text and underlying causes are discarded before callers or +logs can receive them. Other source failures get a fixed failure message. + ### This runtime neither signs, parses nor verifies a capability There is **one** implementation of a Work Context — Core's (`core/workcontext`: @@ -742,8 +763,7 @@ else: the gateway holds them in its durable registry, from the presence documents, and this runtime registers no upstream for them. The three heartbeats that used to — one per consumed module, each brokering a signed prefix-bound token through the gateway, plus the two self-registrations — are -deleted, along with `CODEFLY__MODULE_REGISTRATION_SECRETS` and the per-module -secrets it carried. +deleted, along with the credentials they carried. Codefly still projects the targets into `CODEFLY__API_CONSUMES`, and that projection is still what the passthrough declaration is checked against — at @@ -1015,7 +1035,7 @@ solution.ConsumedModule{ Each call mints only its own method's authority. A viewer who lacks one method's authority is refused that method as `permission_denied`, with the -issuer's reason (which names the missing permission), and is still served +fixed refusal message, and is still served every other method; the mint is never widened or retried with less. Every method of a module that is not `ViewerBearer` needs scopes, its own or the module's, and `ViewerBearer` takes none at either level. @@ -1102,7 +1122,7 @@ client := widgetsv1connect.NewWidgetsClient(page.Client(), page.BaseURL("widgets | --- | --- | | `NewHost(t)` | The fake host gateway: it mints the viewer's Work Context (the accounts `StartTask` procedure) and forwards `/v1//*` to each routed module, streams included, flushed as they arrive. | | `Host.Module(as, upstream)` | Routes a consumed module to the test's own server, path unchanged. Every routed module is also in the solution's api.consumes; one the declaration names but the host does not route is refused as `Serve` refuses it. | -| `Host.RefuseMints(func(Mint) *Refusal)` | Decides each mint: a non-nil `Refusal` refuses it as accounts would (default status 403), so the page sees `permission_denied` with its message. | +| `Host.RefuseMints(func(Mint) *Refusal)` | Decides each mint: a non-nil `Refusal` refuses it as accounts would (default status 403), so the page sees `permission_denied` with a fixed refusal message. | | `Host.Mints()`, `Host.Calls()` | Every mint asked for (audience, scopes, org, session, bearer, the token answered), and every call forwarded to a module, each with the `Mint` whose capability it presented — how a test asserts the authority each method was called under. | | `Start(t, host, consumes...)` / `Handler(host, consumes...)` | The passthrough served for the declaration, or the bare handler and the error `Serve` would refuse the declaration with. | | `Solution.Client()`, `ClientAs(Viewer)`, `BaseURL(as)` | A client that calls as `DefaultViewer` (or the viewer given), setting the bearer and the `x-org-id` / `x-session-id` headers the gateway stamps; and the base URL a module's generated client takes. | diff --git a/credential.go b/credential.go index 3e793f2..edf0be7 100644 --- a/credential.go +++ b/credential.go @@ -35,9 +35,14 @@ type CredentialSource interface { // Credential sets where this execution's credential comes from, replacing the // platform's mint endpoint. A test, or a consumer whose issuer is reached -// another way, supplies its own. Chainable. +// another way, supplies its own. Source errors retain only ErrMintRefused or +// ErrMintUnavailable classification; their text and causes are discarded. +// Chainable. func (s *Server) Credential(source CredentialSource) *Server { s.credential = source + if source != nil { + s.credential = &redactedMintSource{client: source} + } return s } @@ -146,7 +151,7 @@ func (s *Server) platformCredentialSource() (CredentialSource, error) { if err != nil { return nil, fmt.Errorf("configure this workload's credential mint at %s: %w", s.cfg.mintURL, err) } - return client, nil + return &redactedMintSource{client: client}, nil } // heldToTheFrozenAuthority rechecks the authority whenever a source hands back @@ -306,7 +311,7 @@ func (s *Server) mintOptions( // passed alongside the pin meant the drift check guarded a value the // mint did not use. Audience: workcontext.AuthorityValue{Name: AuthorityGroup, Key: AuthorityAudienceKey}, - ProjectedToken: workcontext.ProjectedTokenFile(s.cfg.projectedTokenPath), + ProjectedToken: projectedTokenFile(s.cfg.projectedTokenPath), ProjectionAudience: projectionAudience, // The frozen reader, not a value read again here: it is what rechecks // the authority-bearing values before each mint, including the first. diff --git a/credential_test.go b/credential_test.go index 50e8d07..9daf4ae 100644 --- a/credential_test.go +++ b/credential_test.go @@ -311,16 +311,16 @@ func TestEveryCallerOfOneExecutionSharesOneCarrier(t *testing.T) { } } -// TestABootedRuntimeRegistersNothingWithEveryLegacyKeySet is the cutover +// TestABootedRuntimeRegistersNothing is the cutover // evidence the earlier version of this test did not provide. // // That version called the SDK's mint client directly and asserted no legacy // header was on the request — which would have stayed green with the whole // registration path still in the runtime. This one boots the real runtime with -// every deleted configuration key set to a value a host would once have acted -// on, and asserts what the host received: one request, to the mint, and +// the deleted endpoint settings aimed at an observer, and asserts what the +// host received: one request, to the mint, and // nothing resembling a registration. -func TestABootedRuntimeRegistersNothingWithEveryLegacyKeySet(t *testing.T) { +func TestABootedRuntimeRegistersNothing(t *testing.T) { mint := newHostMint(t, &hostMint{}) // A listener of its own for every deleted URL, which boot cannot move. @@ -347,7 +347,7 @@ func TestABootedRuntimeRegistersNothingWithEveryLegacyKeySet(t *testing.T) { })) t.Cleanup(legacy.Close) - // Every key the cutover deleted. If any of them still reaches a code path, + // Removed endpoint settings must reach no code path. If any still does, // the observer above sees a request — or a header it should never be shown // again. for key, value := range map[string]string{ @@ -359,8 +359,6 @@ func TestABootedRuntimeRegistersNothingWithEveryLegacyKeySet(t *testing.T) { "GATEWAY_MODULE_REGISTRATION_TOKEN_URL": legacy.URL + "/modules/_registration-token", "GATEWAY_SOLUTION_REGISTRATION_TOKEN_URL": legacy.URL + "/solutions/_registration-token", "CODEFLY_INTERNAL_TOKEN": "internal-token-that-must-not-travel", - "CODEFLY__SOLUTION_REGISTRATION_SECRET": "solution-secret-that-must-not-travel", - "CODEFLY__MODULE_REGISTRATION_SECRETS": "things:module-secret-that-must-not-travel", "CODEFLY__SOLUTION_REGISTRATION_INTERVAL": "1s", "CODEFLY_HOST_FRONTEND": "frontend", } { @@ -476,7 +474,7 @@ func TestABootedRuntimeRegistersNothingWithEveryLegacyKeySet(t *testing.T) { "x-codefly-internal-token", "x-codefly-solution-registration", "x-codefly-module-registration", - "x-codefly-module-secret", + removedClaimShape(t, "header"), } { if value := mint.headers[0].Get(header); value != "" { t.Errorf("the one request this runtime made presented %s=%q: this runtime has no registration credential to present", header, value) @@ -775,7 +773,7 @@ func TestATerminalRenewalRefusalEndsTheProcessRatherThanServing503Forever(t *tes default: t.Error("the process was not asked to end: a credential the host has stopped honouring cannot be waited out") } - if stored := server.terminalErr.Load(); stored == nil || !errors.Is(*stored, tc.err) { + if stored := server.terminalErr.Load(); stored == nil || !errors.Is(*stored, workcontext.ErrMintRefused) { t.Error("the reason this process is ending was not recorded, so its exit would say nothing") } } else { @@ -2389,6 +2387,10 @@ func TestTheMintHopIsHeldToTheSameThingsTheGatewayHopIs(t *testing.T) { // the readers under test. func freshMint(t *testing.T, source CredentialSource, held workcontext.Credential) (workcontext.Credential, error) { t.Helper() + // These checks inspect the SDK transport directly, below error redaction. + if redacted, ok := source.(*redactedMintSource); ok { + source = redacted.client + } client, ok := source.(*workcontext.MintClient) if !ok { t.Fatalf("the source under test is %T, not the SDK's mint client, so this would not exercise the transport at all", source) diff --git a/facade_claim_test.go b/facade_claim_test.go index b742b4f..de68a18 100644 --- a/facade_claim_test.go +++ b/facade_claim_test.go @@ -5,6 +5,7 @@ import ( "go/constant" "go/token" "go/types" + "os" "path/filepath" "sort" "strconv" @@ -33,18 +34,42 @@ import ( // evaluate. A skip is a silent pass, and this compiles to the forbidden key // while matching none of its fragments: // -// const key = "CODEFLY__MODULE_" + string(rune(82)) + "EGISTRATION_SECRETS" +// const key = prefix + string(rune(82)) + suffix // // `string(rune(82))` is a constant conversion, which the hand evaluator did not // implement. There is no list of expression forms to finish here: the compiler // has already folded every constant, conversions and concatenation alike, so // the value is taken from it. That fixture is a compiled negative case below. -var claimShapes = map[string]string{ - "/modules/_register": "the gateway's module route-claim path", - "/modules/_registration-token": "the exchange that credentials a module route claim", - "X-Codefly-Module-Secret": "the header carrying a module's route-claim secret", - "X-Codefly-Module-Registration": "the header carrying a module route-claim credential", - "CODEFLY__MODULE_REGISTRATION_SECRETS": "the carrier delivering another module's route-claim secret", +// The removed vocabulary is archived in CHANGELOG.md. Keeping the gate's +// inputs there lets configuration and source trees stay free of those names +// without weakening the compiled regression fixtures. +func removedClaimShape(t *testing.T, kind string) string { + t.Helper() + raw, err := os.ReadFile("CHANGELOG.md") + if err != nil { + t.Fatal(err) + } + for _, line := range strings.Split(string(raw), "\n") { + parts := strings.Split(line, "|") + if len(parts) == 4 && strings.TrimSpace(parts[1]) == kind { + value := strings.Trim(strings.TrimSpace(parts[2]), "`") + if value != "" { + return value + } + } + } + t.Fatalf("missing removed claim %q in the migration archive", kind) + return "" +} + +func claimShapes(t *testing.T) map[string]string { + return map[string]string{ + "/modules/_register": "the gateway's module route-claim path", + "/modules/_registration-token": "the exchange that credentials a module route claim", + "X-Codefly-Module-Registration": "the header carrying a module route-claim credential", + removedClaimShape(t, "header"): "the removed module credential header", + removedClaimShape(t, "carrier"): "the removed module credential carrier", + } } // TestNoModuleFacadeClaimOnTheSDKSurface refuses any constant string in the @@ -67,6 +92,7 @@ func claimShapeFindings(t *testing.T, pkg *packages.Package) []string { t.Helper() var findings []string + shapes := claimShapes(t) evaluated := 0 for _, file := range pkg.Syntax { name := filepath.Base(pkg.Fset.Position(file.Pos()).Filename) @@ -85,7 +111,7 @@ func claimShapeFindings(t *testing.T, pkg *packages.Package) []string { return true } evaluated++ - for shape, what := range claimShapes { + for shape, what := range shapes { if strings.Contains(foldShape(value), foldShape(shape)) { findings = append(findings, pkg.Fset.Position(expr.Pos()).String()+ ": the constant "+strconv.Quote(value)+" is "+what+" ("+shape+")") @@ -202,7 +228,7 @@ import "os" // A constant conversion splicing the carrier's name. func FacadeCredentialViaConversion() string { - const key = "CODEFLY__MODULE_" + string(rune(82)) + "EGISTRATION_SECRETS" + const key = __CARRIER_CONVERSION__ return env(key, "") } @@ -213,7 +239,7 @@ func FacadeClaimPath() string { return facadeClaimPath } // A header assembled through a conversion. func FacadeSecretHeader() string { - return "X-Codefly-Module-" + string(rune(83)) + "ecret" + return __HEADER_CONVERSION__ } // Function-local constants, which a package-level-only collector never saw. @@ -223,7 +249,7 @@ func FacadeClaimLocal(gatewayURL string) string { } func FacadeCredentialLocal() string { - const carrier = "CODEFLY__MODULE" + "_REGISTRATION_SECRETS" + const carrier = __CARRIER_LOCAL__ return env(carrier, "") } @@ -298,6 +324,16 @@ func joinedKey(parts []string) string { } ` + carrier, header := removedClaimShape(t, "carrier"), removedClaimShape(t, "header") + conversion := func(value string) string { + return strconv.Quote(value[:1]) + " + string(rune(" + strconv.Itoa(int(value[1])) + ")) + " + strconv.Quote(value[2:]) + } + source = strings.NewReplacer( + "__CARRIER_CONVERSION__", conversion(carrier), + "__HEADER_CONVERSION__", conversion(header), + "__CARRIER_LOCAL__", strconv.Quote(carrier[:len(carrier)/2])+" + "+strconv.Quote(carrier[len(carrier)/2:]), + ).Replace(source) + // One load for both rules: each shells out to the go command, and the // fixture is the same package either way. shapes, keys := "", "" @@ -309,9 +345,9 @@ func joinedKey(parts []string) string { keys = strings.Join(unresolvedKeyFindings(t, pkg), "\n") } for _, want := range []string{ - "CODEFLY__MODULE_REGISTRATION_SECRETS", + carrier, "/modules/_register", - "X-Codefly-Module-Secret", + header, } { if !strings.Contains(shapes, want) { t.Errorf("the claim-shape gate did not catch %q.\nfindings:\n%s", want, shapes) diff --git a/mcp.go b/mcp.go index 86f5593..2e02e29 100644 --- a/mcp.go +++ b/mcp.go @@ -691,10 +691,6 @@ func firstForwarded(r *http.Request, name string) string { // bare environment variables when ServeMCP landed, which named a provisioning // path that does not exist. // -// This used to say "the way the registration secret is", naming -// SolutionRegistrationSecretGroup. That group and the registrations it fed are -// deleted; the comparison outlived the thing it compared to. -// // MCPIssuerURLKey is the origin MCP clients authenticate against — the host as // an authorization server. A deployment must supply it: what the SDK resolves // is the address this composition dials the host at, in-cluster and reachable @@ -769,8 +765,7 @@ func resolveMCPPublicURL(ctx context.Context, publicURL, id string) (string, boo // a deployment that was told the origin clients authenticate against from one // that resolved the host's in-cluster address, which validate() refuses. // -// The SDK's error is dropped for the reason solutionRegistrationSecret drops -// it: it is the same "no workspace configuration value" whether the group was +// The SDK reports the same "no workspace configuration value" whether the group was // never declared or its carriers never loaded, and the one signal that // distinguishes them is config.environmentLoadErr. func resolveMCPIssuer(ctx context.Context, frontendURL string) (string, bool) { diff --git a/mcp_test.go b/mcp_test.go index 9c2e6be..cb37fa2 100644 --- a/mcp_test.go +++ b/mcp_test.go @@ -900,9 +900,8 @@ func TestAnMCPToolCallIsHeldToThePublishedCeiling(t *testing.T) { // mcpConfigurationCarrier is the environment variable Codefly's render projects // one value of the `mcp` workspace-configuration group into. The runtime never -// reads it: it resolves the value through the SDK, exactly as it resolves the -// registration secret. The tests below write the carrier so the SDK's real -// accessor is what answers — a fake in its place would prove the runtime agrees +// reads it: it resolves the value through the SDK. The tests write the carrier +// so the SDK's real accessor answers — a fake would prove the runtime agrees // with the fake about an encoding that is the SDK's to own. func mcpConfigurationCarrier(key string) string { return resources.WorkspaceConfigurationPrefix + "__" + diff --git a/mint_errors.go b/mint_errors.go new file mode 100644 index 0000000..68cec71 --- /dev/null +++ b/mint_errors.go @@ -0,0 +1,52 @@ +package solution + +import ( + "context" + "errors" + + "github.com/codefly-dev/sdk-go/workcontext" +) + +// The pinned execution client predates the broker's redacted error contract. +// Preserve its classifications, but discard its diagnostic and entire cause +// chain before any runtime caller can return or log response or transport text. +type redactedMintSource struct{ client CredentialSource } + +func (s *redactedMintSource) Credential(ctx context.Context) (workcontext.Credential, error) { + credential, err := s.client.Credential(ctx) + if err == nil { + return credential, nil + } + var projection *projectedTokenError + if errors.As(err, &projection) { + return workcontext.Credential{}, &projectedTokenError{} + } + switch { + case errors.Is(err, workcontext.ErrMintRefused): + return workcontext.Credential{}, workcontext.ErrMintRefused + case errors.Is(err, workcontext.ErrMintUnavailable): + return workcontext.Credential{}, workcontext.ErrMintUnavailable + default: + return workcontext.Credential{}, errors.New("credential mint failed") + } +} + +// Keep the SDK's bounded, fresh read while making local projection failures +// recognizable without retaining filesystem paths or underlying error text. +type projectedTokenFile string + +func (path projectedTokenFile) ProjectedToken() (string, error) { + token, err := workcontext.ProjectedTokenFile(path).ProjectedToken() + if err != nil { + return "", &projectedTokenError{} + } + return token, nil +} + +type projectedTokenError struct{} + +func (*projectedTokenError) Error() string { + return "projected service-account token is missing, empty, unreadable or exceeds 65536 bytes" +} + +func (*projectedTokenError) Unwrap() error { return workcontext.ErrMintUnavailable } diff --git a/mint_migration_test.go b/mint_migration_test.go new file mode 100644 index 0000000..9d2576e --- /dev/null +++ b/mint_migration_test.go @@ -0,0 +1,227 @@ +package solution + +import ( + "context" + "errors" + "fmt" + "io" + "log" + "net/http" + "os" + "path/filepath" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + codefly "github.com/codefly-dev/sdk-go" + "github.com/codefly-dev/sdk-go/workcontext" +) + +func TestProjectedTokenDefaultsToTheCLIProjection(t *testing.T) { + t.Setenv(ProjectedTokenFileEnvironmentVariable, "") + t.Setenv("CODEFLY__WORKSPACE_CONFIGURATION__WORKLOAD_IDENTITY__TOKEN_FILE", "") + if err := codefly.LoadEnvironmentVariables(); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = codefly.LoadEnvironmentVariables() }) + cfg := loadConfig(context.Background(), testSolutionID, nil) + if cfg.projectedTokenPath != "/var/run/secrets/codefly/token" { + t.Fatalf("token path = %q, want the CLI projection", cfg.projectedTokenPath) + } + t.Setenv(ProjectedTokenFileEnvironmentVariable, "/operator/token") + if got := loadConfig(context.Background(), testSolutionID, nil).projectedTokenPath; got != "/operator/token" { + t.Fatalf("explicit token path = %q", got) + } +} + +func TestRuntimeProjectedTokenReaderReadsFreshAndFailsClosed(t *testing.T) { + path := filepath.Join(t.TempDir(), "token") + source := projectedTokenFile(path) + for _, value := range []string{"first-projection", "rotated-projection"} { + writeFile(t, path, "\n"+value+"\n") + got, err := source.ProjectedToken() + if err != nil || got != value { + t.Fatalf("fresh token = %q, %v", got, err) + } + } + for _, value := range []string{"", " \n\t", strings.Repeat("x", 65537)} { + writeFile(t, path, value) + got, err := source.ProjectedToken() + if got != "" || !errors.Is(err, workcontext.ErrMintUnavailable) { + t.Fatalf("unusable projection returned a token or wrong class: %v", err) + } + } +} + +// Exercise the production source, including its TLS readers and error boundary. +func productionMintSource(t *testing.T, mint *hostMint, tokenFile string) CredentialSource { + t.Helper() + cert, key, bundle, _, _, _ := bootIdentityIn(t, mint, testPrincipal) + s := New(Manifest{ID: testSolutionID}) + s.cfg = config{ + identityCertFile: cert, identityKeyFile: key, trustBundleFile: bundle, + allowedCallersFile: identitiesFile(t, testGatewayPrincipal), + mintPeersFile: identitiesFile(t, testGatewayPrincipal), + gatewayPeersFile: identitiesFile(t, testGatewayPrincipal), + mintURL: mint.URL + credentialMintPath, projectedTokenPath: tokenFile, + } + s.principal = testPrincipal + s.authority = authorityFor(t, context.Background()) + source, err := s.platformCredentialSource() + if err != nil { + t.Fatal(err) + } + return source +} + +func TestUnusableProjectedTokenNeverReachesTheMint(t *testing.T) { + for _, kind := range []string{"missing", "unreadable", "empty", "oversized"} { + t.Run(kind, func(t *testing.T) { + path := filepath.Join(t.TempDir(), "token") + switch kind { + case "unreadable": + // A directory fails a read even under an elevated test runner. + if err := os.Mkdir(path, 0700); err != nil { + t.Fatal(err) + } + case "empty": + writeFile(t, path, " \n") + case "oversized": + writeFile(t, path, strings.Repeat("x", 65537)) + } + mint := newHostMint(t, &hostMint{}) + source := productionMintSource(t, mint, path) + credential, err := source.Credential(context.Background()) + if credential.Token() != "" || !errors.Is(err, workcontext.ErrMintUnavailable) { + t.Fatalf("unusable projection did not fail closed: %v", err) + } + if !strings.Contains(err.Error(), "projected service-account token") || strings.Contains(err.Error(), path) { + t.Fatalf("projection error is unclear or discloses its raw path: %v", err) + } + if mint.count() != 0 { + t.Fatal("unusable token reached the broker") + } + }) + } +} + +func TestExecutionMintErrorsDiscardTheirEntireCauseChain(t *testing.T) { + const diagnostic = "broker-reflected-credential" + for _, class := range []error{workcontext.ErrMintRefused, workcontext.ErrMintUnavailable, errors.New("unclassified")} { + raw := fmt.Errorf("%w: %s", class, diagnostic) + source := New(Manifest{ID: testSolutionID}).Credential(refusingSource{err: raw}).credential + _, err := source.Credential(context.Background()) + if err == nil || strings.Contains(err.Error(), diagnostic) || errors.Is(err, raw) { + t.Fatalf("broker diagnostic escaped: %v", err) + } + if class == workcontext.ErrMintRefused || class == workcontext.ErrMintUnavailable { + if !errors.Is(err, class) { + t.Fatalf("lost mint classification: %v", err) + } + } + } +} + +type reflectedCredentialSource struct{ calls atomic.Int64 } + +func (s *reflectedCredentialSource) Credential(context.Context) (workcontext.Credential, error) { + s.calls.Add(1) + return workcontext.Credential{}, fmt.Errorf("%w: broker-reflected-credential", workcontext.ErrMintRefused) +} + +func TestCredentialRefusalNeverLogsDiagnosticsOrRetries(t *testing.T) { + output := &syncBuffer{} + previous := log.Writer() + log.SetOutput(output) + defer log.SetOutput(previous) + source := &reflectedCredentialSource{} + server := New(Manifest{ID: testSolutionID}).Credential(source) + for range 3 { + err := server.actingForAViewer(context.Background()) + if !errors.Is(err, ErrCredentialRefused) || strings.Contains(err.Error(), "broker-reflected-credential") { + t.Fatalf("credential refusal lost its class or exposed diagnostics: %v", err) + } + } + if source.calls.Load() != 1 { + t.Fatalf("refused source was asked %d times", source.calls.Load()) + } + if strings.Contains(output.String(), "broker-reflected-credential") || !strings.Contains(output.String(), "refused") { + t.Fatalf("refusal log disclosed diagnostics or omitted its classification: %s", output.String()) + } +} + +type mintRoundTrip func(*http.Request) (*http.Response, error) + +func (f mintRoundTrip) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) } + +func TestViewerMintDiscardsResponseAndTransportDiagnostics(t *testing.T) { + const diagnostic = "broker-reflected-credential" + for _, kind := range []string{"401", "403", "503", "redirect", "malformed-json", "transport"} { + t.Run(kind, func(t *testing.T) { + gw := newGateway("https://gateway.example", "viewer", "org", "session") + gw.workload = attestingSource(t).Credential + calls := 0 + gw.transport = mintRoundTrip(func(r *http.Request) (*http.Response, error) { + calls++ + if kind == "transport" { + return nil, errors.New(diagnostic) + } + status := map[string]int{"401": 401, "403": 403, "503": 503, "redirect": 307, "malformed-json": 200}[kind] + return &http.Response{ + StatusCode: status, Request: r, + Header: http.Header{"Location": {"https://elsewhere.example/" + diagnostic}, "X-Diagnostic": {diagnostic}}, + Body: io.NopCloser(strings.NewReader(`{"expiresAt":"` + diagnostic + `","message":"` + diagnostic + `"}`)), + }, nil + }) + _, _, err := gw.mint(context.Background(), startTaskRequest{Audience: "documents"}) + if err == nil || strings.Contains(err.Error(), diagnostic) || strings.Contains(relayedError(err).Error(), diagnostic) { + t.Fatalf("mint error disclosed response or transport text: %v", err) + } + if calls != 1 { + t.Fatalf("mint followed a redirect or retried: %d calls", calls) + } + var refusal *WorkContextRefusal + if kind == "401" || kind == "403" || kind == "503" || kind == "redirect" { + if !errors.As(err, &refusal) { + t.Fatalf("lost response classification: %v", err) + } + } + }) + } +} + +func TestViewerMintRefusalsAreTerminalForTheSameAsk(t *testing.T) { + for _, status := range []int{401, 403} { + t.Run(fmt.Sprint(status), func(t *testing.T) { + cache := newWorkContextCache() + var calls atomic.Int64 + refusal := &WorkContextRefusal{StatusCode: status} + mint := func(context.Context) (string, time.Time, error) { calls.Add(1); return "", time.Time{}, refusal } + var wait sync.WaitGroup + for range 20 { + wait.Add(1) + go func() { + defer wait.Done() + if _, err := cache.resolve(context.Background(), "same-ask", mint); err != refusal { + t.Errorf("refusal = %v", err) + } + }() + } + wait.Wait() + for range 3 { + if _, err := cache.resolve(context.Background(), "same-ask", mint); err != refusal { + t.Fatalf("refusal = %v", err) + } + } + if calls.Load() != 1 { + t.Fatalf("refusal automatically re-minted %d times", calls.Load()) + } + _, _ = cache.resolve(context.Background(), "different-ask", mint) + if calls.Load() != 2 { + t.Fatal("refusal incorrectly latched a different ask") + } + }) + } +} diff --git a/passthrough.go b/passthrough.go index 2934e68..b2a3129 100644 --- a/passthrough.go +++ b/passthrough.go @@ -530,20 +530,7 @@ func (s *Server) authorize(ctx context.Context, route passthroughRoute, header h func relayedError(err error) error { var refusal *WorkContextRefusal if errors.As(err, &refusal) && refusal != nil && refusal.StatusCode >= 400 && refusal.StatusCode < 500 { - // The issuer refused the viewer's authority for this call: say so, in - // its own code and words (they name the missing permission), rather - // than as a failed module call. It is the viewer's own authorization, - // so nothing of another principal is disclosed. - code := codeForStatus(refusal.StatusCode) - var parsed connect.Code - if refusal.Code != "" && parsed.UnmarshalText([]byte(refusal.Code)) == nil && parsed != connect.CodeUnknown { - code = parsed - } - message := "the viewer's authority for the " + refusal.Audience + " module was refused" - if refusal.Message != "" { - message += ": " + truncated(refusal.Message) - } - return connect.NewError(code, errors.New(message)) + return connect.NewError(codeForStatus(refusal.StatusCode), errors.New("the viewer's authority for this module was refused")) } var streamErr *StreamError if errors.As(err, &streamErr) && streamErr != nil { diff --git a/passthrough_scopes_test.go b/passthrough_scopes_test.go index 20bfb81..aa7e521 100644 --- a/passthrough_scopes_test.go +++ b/passthrough_scopes_test.go @@ -85,8 +85,8 @@ func TestPassthroughRefusesOnlyTheMethodWhoseAuthorityTheViewerLacks(t *testing. if status != http.StatusForbidden || body["code"] != "permission_denied" { t.Fatalf("get without inspect: status %d, body %v; want 403 permission_denied", status, body) } - if msg, _ := body["message"].(string); !strings.Contains(msg, "things:inspect") { - t.Fatalf("message %q does not name the missing permission", msg) + if msg, _ := body["message"].(string); msg != "the viewer's authority for this module was refused" { + t.Fatalf("message %q disclosed broker diagnostics or lost the fixed refusal", msg) } if len(gw.calls) != 0 { t.Fatalf("the module was called without a capability: %v", gw.calls) diff --git a/passthroughtest/passthroughtest_test.go b/passthroughtest/passthroughtest_test.go index 2e4bf82..7fae458 100644 --- a/passthroughtest/passthroughtest_test.go +++ b/passthroughtest/passthroughtest_test.go @@ -350,8 +350,8 @@ func TestARefusedMintRefusesOnlyThatMethod(t *testing.T) { err = stream.Err() _ = stream.Close() } - if connect.CodeOf(err) != connect.CodePermissionDenied || !strings.Contains(err.Error(), "widgets:watch") { - t.Fatalf("watch = %v, want permission_denied naming the missing permission", err) + if connect.CodeOf(err) != connect.CodePermissionDenied || !strings.Contains(err.Error(), "the viewer's authority for this module was refused") || strings.Contains(err.Error(), "widgets:watch") { + t.Fatalf("watch = %v, want permission_denied with a fixed refusal and no broker diagnostics", err) } if _, err := search.CallUnary(context.Background(), request("w1")); err != nil { t.Fatalf("search, whose authority the viewer holds, was refused: %v", err) diff --git a/solution.go b/solution.go index 3250e98..e1969b8 100644 --- a/solution.go +++ b/solution.go @@ -693,6 +693,11 @@ func loadConfig(ctx context.Context, id string, environmentLoadErr error) config // resolve it are gone. frontendURL := strings.TrimRight(resolveFrontend(ctx, hostModule, hostFrontend), "/") + projectedTokenPath := workloadPath(ctx, ProjectedTokenFileEnvironmentVariable, WorkloadIdentityTokenFileKey) + if projectedTokenPath == "" { + projectedTokenPath = "/var/run/secrets/codefly/token" + } + cfg := config{ port: port, gatewayURL: gatewayURL, @@ -710,7 +715,7 @@ func loadConfig(ctx context.Context, id string, environmentLoadErr error) config // how this package already treats a token-exchange URL it cannot pair. mintURL: strings.TrimSpace(env(CredentialMintURLEnvironmentVariable, "")), dialled: dialledAddresses(gatewayURL, strings.TrimSpace(env(CredentialMintURLEnvironmentVariable, ""))), - projectedTokenPath: workloadPath(ctx, ProjectedTokenFileEnvironmentVariable, WorkloadIdentityTokenFileKey), + projectedTokenPath: projectedTokenPath, identityCertFile: workloadPath(ctx, IdentityCertFileEnvironmentVariable, WorkloadIdentityCertFileKey), identityKeyFile: workloadPath(ctx, IdentityKeyFileEnvironmentVariable, WorkloadIdentityKeyFileKey), trustBundleFile: workloadPath(ctx, IdentityTrustBundleFileEnvironmentVariable, WorkloadIdentityTrustBundleFileKey), @@ -3304,21 +3309,23 @@ func (g *Gateway) workContext(ctx context.Context) (string, error) { }) } -// WorkContextRefusal is the issuer's refusal to mint a Work Context: most -// often an authority the viewer does not hold ("owner is not allowed -// kind:action at requested scope"). It keeps the issuer's Connect code and -// message, so a caller can tell "you lack this permission" from "the gateway -// never routed the mint", and unwraps to the GatewayError it has always been. +// WorkContextRefusal carries only the HTTP status and the locally requested +// audience. Broker response diagnostics are never retained or returned. +// Code and Message were removed; consumers should inspect StatusCode. type WorkContextRefusal struct { Audience string StatusCode int - // Code is the issuer's Connect code ("permission_denied"), when it sent one. - Code string - Message string } func (e *WorkContextRefusal) Error() string { - return fmt.Sprintf("work context mint for %q rejected (status %d, %q): %s", e.Audience, e.StatusCode, e.Code, e.Message) + message := "request failed" + switch e.StatusCode { + case http.StatusUnauthorized, http.StatusForbidden: + message = "request refused" + case http.StatusServiceUnavailable: + message = "execution not established" + } + return fmt.Sprintf("work context mint: HTTP %d: %s", e.StatusCode, message) } func (e *WorkContextRefusal) Unwrap() error { return &GatewayError{StatusCode: e.StatusCode} } @@ -3364,7 +3371,7 @@ func (g *Gateway) mint(ctx context.Context, ask startTaskRequest) (string, time. defer cancel() post, err := http.NewRequestWithContext(ctx, http.MethodPost, g.baseURL+workContextStartTaskProcedure, bytes.NewReader(body)) if err != nil { - return "", time.Time{}, err + return "", time.Time{}, errors.New("work context mint: request could not be constructed") } post.Header.Set("content-type", "application/json") // The viewer's bearer says on whose behalf; this execution's credential @@ -3377,30 +3384,19 @@ func (g *Gateway) mint(ctx context.Context, ask startTaskRequest) (string, time. } resp, err := g.bearerClient().Do(post) if err != nil { - return "", time.Time{}, fmt.Errorf("work context mint for %q: %w", ask.Audience, err) + return "", time.Time{}, errors.New("work context mint: transport failed") } defer drainAndClose(resp) if resp.StatusCode != http.StatusOK { - // Connect reports a refusal as a code and message under a mapped HTTP - // status. Carry it through: an authority the viewer does not hold reads - // nothing like a gateway that never routed the mint, and the status - // alone cannot tell them apart. - var refusal struct { - Code string `json:"code"` - Message string `json:"message"` - } - _ = json.NewDecoder(io.LimitReader(resp.Body, 4<<10)).Decode(&refusal) - return "", time.Time{}, &WorkContextRefusal{ - Audience: ask.Audience, StatusCode: resp.StatusCode, Code: refusal.Code, Message: refusal.Message, - } + return "", time.Time{}, &WorkContextRefusal{Audience: ask.Audience, StatusCode: resp.StatusCode} } var issued struct { Token string `json:"token"` ExpiresAt time.Time `json:"expiresAt"` WorkContextPrincipals } - if err := json.NewDecoder(resp.Body).Decode(&issued); err != nil { - return "", time.Time{}, fmt.Errorf("work context mint for %q returned invalid json: %w", ask.Audience, err) + if err := json.NewDecoder(io.LimitReader(resp.Body, 64<<10)).Decode(&issued); err != nil { + return "", time.Time{}, errors.New("work context mint: invalid response") } // The capability travels as a string and is not parsed here. There is one // implementation of a Work Context — core's — and this runtime is not it: @@ -3429,8 +3425,8 @@ func (g *Gateway) mint(ctx context.Context, ask startTaskRequest) (string, time. ask.Audience) case lifetime < workContextMinimumLifetime: return "", time.Time{}, fmt.Errorf( - "work context mint for %q returned the expiry %s, already past or less than %s away (check this host's clock against the issuer's)", - ask.Audience, issued.ExpiresAt.UTC().Format(time.RFC3339), workContextMinimumLifetime) + "work context mint for %q returned an expiry already past or less than %s away (check this host's clock against the issuer's)", + ask.Audience, workContextMinimumLifetime) } // Renew ahead of expiry, but never by more than half the capability's own // life: a lead longer than the lifetime would make every capability @@ -3443,8 +3439,7 @@ func (g *Gateway) mint(ctx context.Context, ask startTaskRequest) (string, time. // Honoured, but say so once: at this lifetime a capability cannot be // held across a whole request, so the mint rate is the issuer's // setting and not a defect here. - log.Printf("work contexts are issued with a %s lifetime, shorter than the %s renewal lead: one will be re-minted roughly every %s", - lifetime.Round(time.Second), workContextRenewal, lead.Round(time.Second)) + log.Print("work context mint: issued lifetime requires an earlier renewal") }) } g.contexts.remember(token, issued.WorkContextPrincipals) @@ -3501,6 +3496,8 @@ type workContextCache struct { // fans out would otherwise spend one audited mint per goroutine for a // capability they all share. minting map[string]*pendingMint + // Refusals persist for this viewer request; another caller must not re-mint. + refused map[string]error // issuedFor holds, per issued capability, whom accounts said it acts for. issuedFor map[string]WorkContextPrincipals } @@ -3522,6 +3519,7 @@ func newWorkContextCache() *workContextCache { return &workContextCache{ minted: map[string]cachedCapability{}, minting: map[string]*pendingMint{}, + refused: map[string]error{}, issuedFor: map[string]WorkContextPrincipals{}, } } @@ -3555,6 +3553,10 @@ func (c *workContextCache) resolve( mint func(context.Context) (string, time.Time, error), ) (string, error) { c.mu.Lock() + if err := c.refused[key]; err != nil { + c.mu.Unlock() + return "", err + } if issued, ok := c.minted[key]; ok && time.Now().Before(issued.reuseUntil) { c.mu.Unlock() return issued.token, nil @@ -3579,6 +3581,10 @@ func (c *workContextCache) resolve( c.mu.Lock() delete(c.minting, key) + var refusal *WorkContextRefusal + if errors.As(err, &refusal) && (refusal.StatusCode == http.StatusUnauthorized || refusal.StatusCode == http.StatusForbidden) { + c.refused[key] = err + } if err == nil { c.minted[key] = cachedCapability{token: token, reuseUntil: reuseUntil} } diff --git a/solution_test.go b/solution_test.go index 47366f4..dca9ed0 100644 --- a/solution_test.go +++ b/solution_test.go @@ -1257,8 +1257,8 @@ func TestAHandlersGatewayDialsThroughTheBootsAuthenticatedTransport(t *testing.T } } -// A refused mint retains diagnostics for the handler but returns an actionable -// status, without disclosing upstream details, to the browser. +// A refused mint exposes its status, with no upstream diagnostics in either +// the handler error or the browser response. func TestForModuleSurfacesARefusedMint(t *testing.T) { gw := newWorkContextGateway(t, &workContextGateway{mintStatus: http.StatusForbidden}) var mintErr error @@ -1275,11 +1275,16 @@ func TestForModuleSurfacesARefusedMint(t *testing.T) { if mintErr == nil { t.Fatal("ForModule succeeded against a refusing authority") } - for _, want := range []string{"documents", "403", "permission_denied", "caller holds no such authority"} { + for _, want := range []string{"403", "request refused"} { if !strings.Contains(mintErr.Error(), want) { t.Errorf("mint error %q does not mention %q", mintErr, want) } } + for _, forbidden := range []string{"permission_denied", "caller holds no such authority"} { + if strings.Contains(mintErr.Error(), forbidden) { + t.Errorf("mint error disclosed broker text: %v", mintErr) + } + } body, err := io.ReadAll(resp.Body) if err != nil { t.Fatal(err) From 793279ec816a25c51a7dbe7b74399ac58e9d0901 Mon Sep 17 00:00:00 2001 From: antoinetoussaint-byte <312536060+antoinetoussaint-byte@users.noreply.github.com> Date: Sun, 11 Oct 2026 06:41:39 -0400 Subject: [PATCH 2/2] fix: the runtime speaks the host's broker contract and sends its prefix MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review of the mint-once change found that the execution mint still spoke the older work-context exchange and never submitted the solution's prefix, that retries covered more than 503 and transient transport failures, and that the legacy-secret test no longer exercised a secret-enabled path. - The execution mint now speaks the host's broker protocol as saas-sdk-go's moduleauthority defines it — the path, the projected bearer, the internal perimeter header and the request fields including the declared prefix — implemented natively (importing the SDK conflicts with this runtime's credential SDK). A conformance test drives a fake broker with the exact request shape. - Only 503 and transient transport failures retry; 400, 404, 429 and other statuses are terminal errors and 401/403 terminal refusals. - The legacy-secret test sets non-empty legacy variables and proves no path reads them. - Tests whose fixtures encoded the old protocol now assert the same behaviour under the broker contract; dead code is deleted. Nine mutations caught; the full suite passes; lint reports no issues. Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 90 +++----- CHANGELOG.md | 44 ++-- README.md | 79 +++---- boot_test.go | 69 +++--- broker_credential.go | 294 ++++++++++++++++++++++++ broker_credential_test.go | 423 +++++++++++++++++++++++++++++++++++ contract_test.go | 6 +- credential.go | 164 +++++--------- credential_test.go | 109 ++++----- environment_boundary_test.go | 63 ------ facade_claim_test.go | 45 ++-- identity_test.go | 20 +- mcp_test.go | 36 +-- mint_errors.go | 129 ++++++++++- mint_migration_test.go | 16 +- projection_boundary_test.go | 7 +- solution.go | 87 ++++--- solution_test.go | 38 ++-- 18 files changed, 1199 insertions(+), 520 deletions(-) create mode 100644 broker_credential.go create mode 100644 broker_credential_test.go diff --git a/AGENTS.md b/AGENTS.md index 2822c92..d000de9 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -23,8 +23,8 @@ runtime removed. It owns none of the counterparts it talks to. The mint endpoint and the admission rules belong to the host (the composition's own host module) and the -gateway; the mint-once client, the authority reader and the TLS reloader to -`codefly-dev/sdk-go`; endpoint, port and secret resolution to `sdk-go` as well; +gateway; the authority reader and the TLS reloader to `codefly-dev/sdk-go`; the broker +wire contract to the host, implemented natively here; endpoint, port and secret resolution to `sdk-go` as well; workspace, manifest and presence/authority document types to `codefly-dev/core`; running a composition, provisioning its secrets and rendering the authority document to `codefly-dev/cli`. Everything here is a library — one Go package at @@ -205,8 +205,8 @@ embedding it. wire shapes — folded, so a shape split across a concatenation, parenthesised, built from named constants or cased differently is the same shape to it. - **Never sign, parse or verify a Work Context here.** There is one - implementation, Core's; this runtime obtains a credential through the SDK's - mint client, carries it as the string it travels as, and lets the far end + implementation, Core's; this runtime obtains a credential through the native broker + client and delegates structural inspection to Core, carries it as the string it travels as, and lets the far end decide. It declares none of a capability's fields and holds no verifier — core's needs the issuer's live revision, replay, grant and seal sources, which a solution does not have. `work_context_boundary_test.go` fails on a signing @@ -333,7 +333,7 @@ embedding it. set spanning both authorises each to stand in for the other at the other's address — the mint being the one that receives the projected token. `MINT_PEERS_FILE` governs the mint hop again, per handshake, through the - SDK's `AdmittedPeers` reader (sdk-go#51) — so the question it was kept open + native broker's live admitted-peer reader — so the question it was kept open for is settled: it is the authorization set for the destination that receives the projected service-account token. It also still narrows the shared-address case, where the mint and the gateway answer at one address and a dial cannot @@ -383,8 +383,7 @@ embedding it. every mint is viewer-driven and `ForModule` refuses a caller without an org and a session. - **This execution's own credential is held to what it was FIRST sealed to.** - The host seals it from the projected token — `MintOptions` carries no - installation field — so what this runtime owns is refusing one sealed to no + The host seals it from the projected token — the broker request carries no installation field — so what this runtime owns is refusing one sealed to no installation, refusing one sealed to no **execution** (image digest and build incarnation, which core seals as a pair), and refusing a renewal sealed to a *different* installation, digest or incarnation. The last is one value, not @@ -438,61 +437,30 @@ embedding it. answer is recorded **where the answer lands** rather than by whichever caller was still waiting — the ask is detached, so it can finish with nobody listening. -- **The mint hop is held to the same three things the gateway hop is, and the - SDK owns the transport that holds it.** `MintOptions` takes no pool, no - client and no dialler: it takes `TrustAnchor`, `ClientCertificate` and - `AdmittedPeers` as **readers**, each consulted during the handshake - (sdk-go#51). So the mint request presents this workload's X.509-SVID, the - endpoint is admitted by the SPIFFE ID in its single URI SAN against - `MINT_PEERS_FILE`, and the anchor is re-read before and after every - handshake and never cached. Withdrawal of a root or of a peer takes effect on - the next handshake rather than on the next process. - The SDK still refuses a caller-supplied `*http.Client`, and that reasoning is - sound and worth keeping in mind: a supplied client is a hole it cannot - inspect — a nil `Transport` means the global mutable default, a - `DialTLSContext` bypasses `TLSClientConfig` entirely, and a caller holding - the same `*http.Transport` can turn verification off after construction. The - answer was never for this runtime to take the transport back; it was for the - transport to take readers instead of values, which is what it now does. - **This paragraph said the opposite, and said it was unreachable.** It - recorded that the mint presented no certificate, that `MINT_PEERS_FILE` did - not govern that hop, and that the rest of the posture could not be restored - from outside the SDK — accurate at `v0.3.0`, and the reason it was an sdk-go - issue rather than a local workaround. What was wrong was not the diagnosis - but treating the resulting gap as a property of this runtime: a reviewer - pointed out that the fixtures had been weakened to accommodate it - (`VerifyClientCertIfGiven`, the mint caller's identity expected to be empty), - which establishes a changed behaviour rather than closing a finding. The - fixtures demand the certificate again, and - `TestTheMintHopIsHeldToTheSameThingsTheGatewayHopIs` drives the negative - controls through a real handshake: a peer the anchor signed but the set does - not name, peer withdrawal with the anchor unchanged, an unreadable anchor, - and a certificate reader that cannot answer. - **There is no established mint connection to bound, and this file said there - was.** It recorded a residual — "the client keeps idle connections, so a - withdrawal takes effect on the next handshake and not on a connection already - open" — which was true of `v0.3.0` and is not true of the pinned leaf. That - transport sets `DisableKeepAlives`, `MaxIdleConnsPerHost: 0` and - `ForceAttemptHTTP2: false`, so **every mint is a fresh handshake**; and after - the handshake completes it re-reads the anchor and the peer set and verifies - the peer again, because `GetClientCertificate` and the server's own - processing run *after* the first admission decision and a withdrawal landing - in that window must not authorize the HTTP write. It also refuses an endpoint - that did not request a client certificate, so this hop cannot silently become - anonymous. The residual to carry is narrower and belongs to the SDK: the - interval between that post-handshake recheck and the write itself. -- **Only 401 and 403 latch a credential refusal.** The rule was "429 and 5xx - retry, everything else is terminal", and sdk-go v0.3.0 inverted it after - measuring the cost: a 408 from a proxy, with a valid credential in hand, - refused and latched for the life of the process, and a 404 from an ingress - mid-rollout would do the same. Those two are the statuses the host signs — - the projected token is not acceptable, or the build is not approved. **A 404 - on the mint is now an outage**, so an old runtime against a host of this - generation waits out its bounded window and exits non-zero for the - orchestrator rather than failing on the first answer. It still does not - recover; what changed is the shape, not the outcome. +- **The execution mint follows the host's module broker contract natively.** + `saas-sdk-go/moduleauthority` at `217bf9fe3f10` is the wire reference; its + capability dependency is incompatible with this runtime. Send exactly + `prefix: Manifest.ID` to `/modules/_work-context`, with the projected token + as bearer and `X-Codefly-Internal-Token` from the declared workspace secret + `module-authority/INTERNAL_TOKEN`. Resolve it in `loadConfig`, refuse absence + in `validate()`, and never resurrect a legacy secret reader. The projection + audience is `accounts`; the prefix never replaces the frozen SPIFFE principal. + The native transport owns a fresh mutually authenticated TLS handshake for + every mint, checks live trust and admission before and after it, disables + redirects/proxies/reuse, and holds the presented leaf to the frozen principal. + The SDK still owns the certificate reloader and authority reader. Core's + `Inspect` is the only structural reader of the returned token; no signed + message is constructed or decoded here, and inspection verifies no signature. + A private credential view keeps the public SDK `CredentialSource` compatible. +- **Only 503 and transient transport failures retry.** Timeout, connection + refused and connection reset are identified by error types/identities, never + text. 401/403 are terminal refusals. Every other HTTP status, TLS failure, + permanent DNS failure, cancellation, unclassified transport failure and + invalid broker response is a terminal error. Retain safe status/class/category + only, with no raw diagnostic cause. Boot and renewal use the same decision; + terminal answers are retained where they land, even after waiters leave. - **A terminal credential refusal ends the process; a transient one does not.** - `ErrMintRefused` at renewal fails `/health` and ends `serve` with the reason. + `ErrMintRefused` or a terminal broker error at renewal fails `/health` and ends `serve` with the reason. `ErrMintUnavailable` must not: it is transient by construction and exiting on it is a crash loop. `ErrRevoked` is not in either set, and three review rounds found this file still saying it was: the mint client does not return it at all diff --git a/CHANGELOG.md b/CHANGELOG.md index d2f5074..0ed0fa1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Changelog -## Unreleased — partial mint-once migration +## Unreleased — native module-broker execution mint Breaking API change: `WorkContextRefusal.Code` and `WorkContextRefusal.Message` are removed. Use `StatusCode` (or unwrap to `GatewayError`) to classify failures. @@ -12,21 +12,28 @@ mounts it read-only for audience `accounts` with a lifetime of at most 600 secon The runtime reads the bounded file again before each mint. Missing, empty, unreadable and oversized projections fail closed. -The requested `saas-sdk-go` revision -`v0.0.3-0.20261011091114-217bf9fe3f10` cannot build alongside the current -`sdk-go/workcontext`: its `moduleauthority` package requires the removed -`WorkContextToken` and `ParseWorkContextToken` API. Its `Seams` also requires an -internal perimeter token that this runtime no longer provisions. Switching the -broker and supplying the solution's declared name as its prefix remain blocked -on a compatible SDK. The offline `go get` succeeded, but the subsequent -`moduleauthority` build failed on those symbols. The unusable dependency change -was reverted and `go mod tidy` left the original pins unchanged. No SDK source -was copied or modified locally. - -The retained execution SDK marks every non-401/403 HTTP status unavailable and -exposes no typed status for narrowing retries. Therefore the requested retry -policy (only 503 and transient transport failures) is not complete. Execution -refusals still latch; errors from default and consumer-supplied credential sources are now redacted. +Execution minting now implements the published `saas-sdk-go/moduleauthority` +contract at `217bf9fe3f10` natively: `POST /modules/_work-context`, the declared +`Manifest.ID` as `prefix`, the fresh projected bearer, and the perimeter token +from the declared workspace secret `module-authority/INTERNAL_TOKEN`. The legacy +internal-token variable remains ignored. This is a runtime protocol fix; no SDK +package is vendored or copied and dependency pins are unchanged. + +The broker client accepts the camel-case response and delegates structural +capability inspection to Core. The public `CredentialSource` signature is +unchanged. TLS identity, fresh trust/admission checks, frozen authority, +installation/build holds, single-flight acquisition and expiry-driven renewal +remain enforced. + +Only 503 and typed timeout/connection-refused/connection-reset failures retry. +401/403 remain terminal refusals; every other status and permanent transport or +response error is terminal. Errors retain safe classification and discard raw +diagnostics. Conformance tests exercise the actual request and response using +an in-memory fake broker, including boot and renewal retry decisions. + +The legacy-secret regression again sets nonempty deleted secret variables. +Compiler gates pin the canonical forbidden spellings independently of this +archive; compiled fixtures prove that reintroduced readers are rejected. ### Removed authentication configuration (archive) @@ -36,9 +43,8 @@ These identifiers are deleted, not compatibility inputs: `CODEFLY__SOLUTION_REGISTRATION_SECRET`, `MintModuleRegistration`, and `MintSolutionRegistration`. -The compiler-based facade gate reads the two wire spellings below from this -archive and still checks constants, conversions, aliases and compiled fixtures. -Tests never provision a removed identity secret. +These archived spellings are also independently pinned by the compiler-based +facade gate. Adversarial tests deliberately provision removed secrets. | Kind | Removed wire spelling | | --- | --- | diff --git a/README.md b/README.md index eff2894..51ce0ce 100644 --- a/README.md +++ b/README.md @@ -62,12 +62,15 @@ Four things follow, and each is a boot failure rather than a degraded run: | The authority-bearing values are read once and frozen | the credential is sealed to the values it was minted under, so a value that drifts is an error, never a reload | | The contract is published, and the declaration is held to it | an ask outside its declared ceiling fails the boot — see [The published contract](#the-published-contract) | -The remaining migration to the platform's module broker is **blocked by SDK -compatibility**: `saas-sdk-go` at `217bf9fe3f10` references a capability API -removed from this runtime's `sdk-go/workcontext`, and requires an internal-token -seam this runtime no longer has. The existing execution SDK remains in use; -its broader retry policy below still needs an upstream change. See -[the migration record](CHANGELOG.md) for the exact blocker. +Execution minting uses the host's module broker protocol natively. The reference +is `saas-sdk-go/moduleauthority` at `217bf9fe3f10`; importing that package is +incompatible with this runtime's SDK capability API, so no dependency was added. +The request is `POST /modules/_work-context`, with exactly +`{"prefix":""}`, `Authorization: Bearer `, and +`X-Codefly-Internal-Token`. The prefix is the declared solution ID, separate from +the frozen SPIFFE principal. The answer carries `token`, `expiresAt`, +`principalId`, and `tenant`. Core alone inspects the returned capability; the +runtime does not implement a signed encoding or verify an issuer's authority. `WorkContextRefusal.Code` and `.Message` have been removed (breaking API change). Consumers should inspect `.StatusCode` or unwrap to `GatewayError`. Broker bodies, @@ -118,17 +121,18 @@ identifier and the issuer](#the-resource-identifier-and-the-issuer). |---|---|---| | Own listen port | `codefly.For(ctx).Endpoint("http").NetworkInstance()` — the Codefly-assigned port, not a fixed default | `PORT` | | Gateway URL (auth-gateway `rest`) | resolved by role — the single module owning the `auth-gateway` `rest`/`rest` endpoint, discovered from the injected carriers (or the workspace, run locally). Must be `https` | `GATEWAY_URL` | -| Credential mint URL | **not resolved — required explicitly.** Must be `https`. Nothing derives it: whether the host mints through the gateway or the workload posts straight to the issuer is unsettled and neither endpoint is deployed, and this is the address the projected service-account token goes to, so an assumed one is refused at boot. Every boot logs where it will mint, and a 404 there means that endpoint does not exist on that host — not that this build was refused | `CODEFLY__CREDENTIAL_MINT_URL` (**required**) | +| Credential mint URL | Explicit HTTPS endpoint ending in `/modules/_work-context`. The host's broker path is published; its origin must still be supplied. A 404 is a terminal endpoint/configuration error | `CODEFLY__CREDENTIAL_MINT_URL` (**required**) | +| Broker perimeter credential | `codefly.For(ctx).WorkspaceSecret("module-authority", "INTERNAL_TOKEN")`, required at boot. Sent only to the execution broker as `X-Codefly-Internal-Token`; it does not replace the projected token or workload identity | — (declared workspace secret, no legacy env fallback) | | Projected service-account token | `codefly.For(ctx).WorkspaceConfiguration("workload-identity", "TOKEN_FILE")` — a **path**, re-read at every mint; defaults to `/var/run/secrets/codefly/token`, the CLI projection | `CODEFLY__WORKLOAD_TOKEN_FILE` | | Workload identity certificate | `workload-identity`/`CERT_FILE` | `CODEFLY__WORKLOAD_IDENTITY_CERT_FILE` | | Workload identity private key | `workload-identity`/`KEY_FILE` | `CODEFLY__WORKLOAD_IDENTITY_KEY_FILE` | | Peer trust anchor | `workload-identity`/`TRUST_BUNDLE_FILE` — **required**: the listener requires and verifies a caller's certificate against it, and the outbound client verifies the platform against it | `CODEFLY__WORKLOAD_IDENTITY_TRUST_BUNDLE_FILE` | | Allowed callers | `workload-identity`/`ALLOWED_CALLERS_FILE` — **required**, a **path** to a file of identities, one per line (normally the gateway's); re-read per handshake. Verifying against the anchor says a caller holds an identity the platform issued; this says which of them this solution serves | `CODEFLY__WORKLOAD_IDENTITY_ALLOWED_CALLERS_FILE` | -| Credential mint peer | `workload-identity`/`MINT_PEERS_FILE` — **required**, a **path** to a file of identities; re-read per handshake. This is the set the credential mint endpoint is admitted by: the SDK's mint client reads it during the handshake and refuses a peer whose single URI SAN is not in it, even when the certificate was signed by the accepted anchor. It also narrows the shared-address case, where the mint and the gateway answer at one address and a dial cannot be attributed: what is admitted there is the intersection of this set and the gateway's | `CODEFLY__WORKLOAD_IDENTITY_MINT_PEERS_FILE` | +| Credential mint peer | `workload-identity`/`MINT_PEERS_FILE` — **required**, a **path** to a file of identities; re-read per handshake. This is the set the credential mint endpoint is admitted by: the native broker client reads it during the handshake and refuses a peer whose single URI SAN is not in it, even when the certificate was signed by the accepted anchor. It also narrows the shared-address case, where the mint and the gateway answer at one address and a dial cannot be attributed: what is admitted there is the intersection of this set and the gateway's | `CODEFLY__WORKLOAD_IDENTITY_MINT_PEERS_FILE` | | Gateway peer | `workload-identity`/`GATEWAY_PEERS_FILE` — **required**, same shape. One set **per destination**: the mint and the gateway are two parties, and a single set spanning both authorises each to stand in for the other at the other's address — and the mint is the destination that receives the projected service-account token. A dial to an address that is neither has no set and is refused. When the two are at the **same** host and port — the brokered shape — a dial there cannot be attributed to one of them, so it is held to the **intersection** of both sets, and an empty intersection is refused at boot | `CODEFLY__WORKLOAD_IDENTITY_GATEWAY_PEERS_FILE` | | Principal this workload runs as | `module-authority`/`PRINCIPAL`, read once and frozen | — | | Audience it mints against | `module-authority`/`AUDIENCE`, read once and frozen | — | -| Audience of its own projected token | `module-authority`/`PROJECTION_AUDIENCE`, read once and frozen | — | +| Audience of its own projected token | `module-authority`/`PROJECTION_AUDIENCE`, required to be `accounts`, read once and frozen | — | | Contract profile | the Codefly environment's own name, which is how Core resolves a profile for an environment that declares none | `CODEFLY__CONTRACT_PROFILE` | | MF assets | `Manifest.Assets` when set (see below), else the `../fe-remote/dist` directory | `ASSETS_DIR` (directory only) | | Host issuer (MCP) | `codefly.For(ctx).WorkspaceConfiguration("mcp", "issuer-url")` — the declared value the composition supplies; without one, the resolved host frontend origin (see [Exposing an MCP server](#exposing-an-mcp-server)). Checked at boot only when `ServeMCP` is declared | — (declared configuration, no env override) | @@ -190,8 +194,8 @@ with the registrations and then came back for one consumer, the same way `PUBLIC_URL` did: it names the host's frontend service, whose resolved URL is the fallback the MCP issuer identifier is derived from when no `mcp/issuer-url` is declared (`resolveMCPIssuer`). It is read in `loadConfig` like every other -value here. Nothing reads the shared cluster-internal token or -a per-solution registration secret any more: the credential this runtime +value here. Nothing reads the legacy `CODEFLY_INTERNAL_TOKEN` or a per-solution registration +secret. The broker perimeter token comes only from its declared workspace secret; the credential this runtime presents attests which workload it is, and there is no fallback to one that attests nothing. @@ -451,14 +455,13 @@ admitted. ## One credential per execution `Serve` obtains this execution's credential once, before the listener exists, -through the SDK's mint client (`sdk-go/workcontext`): the projected +through the native module broker client: the projected service-account token is presented to the host's mint endpoint, and the host — which establishes the principal, the installation and the build from that token and its own records, never from anything this process reports — answers with a credential sealed to this build incarnation and this installation. -The runtime holds no mint of its own, and it runs **no registration -heartbeat** — which is the claim this cutover is about. It does run other +The runtime runs **no registration heartbeat** — which is the claim this cutover is about. It does run other timers, and saying otherwise here was wrong twice: the inbound and outbound trust watchers re-verify an established peer once a second, the credential acquisition holds off after a failure (capped, jittered), and the listener has @@ -480,32 +483,24 @@ runtime replaced, because the heartbeat at least minted on a timer. **A refused first mint fails the boot and is never retried. An unavailable one is waited for, within a bound.** The two are different answers: -- a **refusal** (`ErrMintRefused`, which the SDK latches for exactly two - statuses, `401` and `403`) is the issuer judging this workload: this build is not the one - its presence document approved, the pod's identity does not match, the token - attests to another subject. No number of attempts changes any of those - answers, so the boot reports the refusal class and exits non-zero. A loop - around it is what this change exists to delete: the old runtime answered every - refusal by beating again, minting a fresh single-use token each time, so one - undeployable solution produced an audited mint every 15 seconds for as long as - it ran. -- an **unavailable** mint (`ErrMintUnavailable`: the host's `503`, `429`, any - other `5xx`, **and a `404`**) is not a judgement. A workload may legitimately start before its - presence generation has been applied, and then there is no incarnation to mint - against yet; the issuer may also be unable to reach the Kubernetes API or its - own policy log, and a host that issues nothing in those cases is behaving - correctly. So the boot asks again, backing off, for up to **two minutes**, and - then exits non-zero. - - A `404` is in this set, and that is a deliberate change rather than an - oversight: the rule here was "`429` and `5xx` retry, everything else is - terminal", and sdk-go v0.3.0 inverted it after measuring the cost — a `408` - from a proxy, with a valid credential in hand, latched for the life of the - process, and a `404` from an ingress mid-rollout would do the same. `401` and - `403` are the two statuses the host signs. So **an old runtime against a host - of this generation waits out its bounded window and exits non-zero for the - orchestrator rather than failing on the first answer.** It still does not - recover; what changed is the shape, not the outcome. +- A **refusal** (`401` or `403`, wrapping `ErrMintRefused`) is terminal and + retained. It is never retried. +- An **unavailable** mint (`503`, timeout, connection refused or connection + reset, wrapping `ErrMintUnavailable`) is retried with backoff inside the + **two-minute** boot deadline. Renewal uses the same classification and keeps + an unexpired held credential available during transient failures. +- Every other HTTP status, including `400`, `404`, `429`, `500`, and redirects, + is a **terminal error**. TLS, permanent DNS, cancellation, unclassified + transport failures, invalid responses and unusable projections are terminal + too. They stop renewal and fail health. Status, status class and transport + category survive; raw body, headers, parser diagnostics and causes do not. + +Every mint makes a fresh TLS 1.3 handshake, presents this workload's identity, +checks `MINT_PEERS_FILE` and the trust anchor before and after the handshake, +and refuses a server that did not request a client certificate. Redirects, +proxies and connection reuse are disabled. The SDK still owns certificate +reloading and authority resolution. `CredentialSource` remains SDK-compatible +for consumers supplying their own issuer. The bound is what keeps the second case from being the heartbeat again: it is a boot waiting for a dependency, with no steady state, nothing minted on success @@ -563,7 +558,7 @@ logs can receive them. Other source failures get a fixed failure message. There is **one** implementation of a Work Context — Core's (`core/workcontext`: deterministic proto, Ed25519) — and this runtime is not it. It obtains a -credential through the SDK's client, carries it as the string it travels as, +credential through the module broker client, carries it as the string it travels as, and lets whoever consumes it decide. It declares none of a capability's fields and reads none of them: the capability a handler's gateway presents to a module is a string this package never decoded, and the one check that must happen @@ -873,7 +868,7 @@ request, and `ForModule` refuses one without an organization and a session. This execution's own credential is sealed to an installation by the host, from the projected service-account token; the runtime names none on that mint and -cannot, since the SDK's `MintOptions` carries no such field. What it does is +cannot: the broker request contains only the declared prefix. What it does is refuse a credential sealed to no installation at all, and refuse a *renewal* sealed to a different one than the credential this process has been acting under: that is a different identity arriving through the renewal path, not a diff --git a/boot_test.go b/boot_test.go index 4e4a641..dc134f0 100644 --- a/boot_test.go +++ b/boot_test.go @@ -64,6 +64,7 @@ const ( // authorityValues provisions the module-authority group the way the platform // does, through the carriers the SDK reads. func authorityValues(t *testing.T) { + t.Setenv("CODEFLY__WORKSPACE_SECRET_CONFIGURATION__MODULE_AUTHORITY__INTERNAL_TOKEN", "broker-perimeter-token") t.Helper() for key, value := range map[string]string{ AuthorityPrincipalKey: testPrincipal, @@ -357,6 +358,13 @@ func (m *hostMint) serve(w http.ResponseWriter, r *http.Request) { }) return } + if r.URL.Path == "/modules/_work-context" { + var body map[string]string + if r.Method != http.MethodPost || r.Header.Get("Content-Type") != "application/json" || r.Header.Get("Accept") != "application/json" || r.Header.Get("X-Codefly-Internal-Token") != "broker-perimeter-token" || !strings.HasPrefix(r.Header.Get("Authorization"), "Bearer ") || json.NewDecoder(r.Body).Decode(&body) != nil || len(body) != 1 || body["prefix"] != testSolutionID { + http.Error(w, "invalid broker request", http.StatusBadRequest) + return + } + } atomic.AddInt64(&m.mints, 1) // recoverAfter 0 means this host never issues; a positive one means it // issues once that many asks have been answered with status. @@ -390,7 +398,11 @@ func (m *hostMint) serve(w http.ResponseWriter, r *http.Request) { http.Error(w, "stand-in issuer: "+err.Error(), http.StatusInternalServerError) return } - writeJSON(w, http.StatusOK, map[string]string{"work_context": token}) + if r.URL.Path == "/modules/_work-context" { + writeJSON(w, http.StatusOK, map[string]string{"token": token, "expiresAt": time.Now().Add(m.ttl).UTC().Format(time.RFC3339), "principalId": corework.FixturePrincipal, "tenant": corework.FixtureTenant}) + } else { + writeJSON(w, http.StatusOK, map[string]string{"work_context": token}) + } } func (m *hostMint) count() int64 { return atomic.LoadInt64(&m.mints) } @@ -1083,7 +1095,7 @@ func TestABootWithAnUnusableAdmissionFileIsRefused(t *testing.T) { writeFile(t, path, tc.contents) } server := New(Manifest{ID: testSolutionID}) - server.cfg = config{ + server.cfg = config{brokerInternalToken: "broker-perimeter-token", port: "8080", gatewayURL: "https://gateway:42152", mintURL: "https://gateway:42152" + credentialMintPath, identityCertFile: certFile, @@ -1497,7 +1509,7 @@ func TestTheListenerDoesNotNegotiateHTTP2(t *testing.T) { c := newCell(t) certFile, keyFile, bundleFile, _, _ := c.workload(t, testPrincipal) server := New(Manifest{ID: testSolutionID}) - server.cfg = config{identityCertFile: certFile, identityKeyFile: keyFile, trustBundleFile: bundleFile, + server.cfg = config{brokerInternalToken: "broker-perimeter-token", identityCertFile: certFile, identityKeyFile: keyFile, trustBundleFile: bundleFile, allowedCallersFile: identitiesFile(t, testGatewayPrincipal)} server.principal = testPrincipal config, err := server.serverIdentity() @@ -1522,10 +1534,8 @@ func TestTheListenerDoesNotNegotiateHTTP2(t *testing.T) { // see. The executed round validated these in scratch; they belong here. // // The issuer answers correctly for the first credential and switches shape -// afterwards. A credential whose whole remaining lifetime sits inside the -// renewal lead cannot be installed at all (the SDK floors that lead at five -// seconds), so the TTL is just above it: the first install succeeds and the -// next ask renews. +// afterwards. The broker renews at one fifth of the issued lifetime (capped +// at thirty seconds), so the first install and control call precede renewal. func TestABootedRenewalToAnotherExecutionEndsTheProcess(t *testing.T) { for _, tc := range []struct { name string @@ -1535,24 +1545,9 @@ func TestABootedRenewalToAnotherExecutionEndsTheProcess(t *testing.T) { {"a renewal sealing a different build", &hostMint{incarnation: 99}}, } { t.Run(tc.name, func(t *testing.T) { - // The TTL sets the margin between the boot and the renewal, and - // six seconds was not a margin at all. - // - // The SDK floors the renewal lead at five seconds - // (minRenewalLeadTime), so a six-second credential is due for - // renewal ONE second after it is issued. The control call below - // has to land before that, and on a loaded CI runner the boot — - // cert generation, the TLS listener, the first mint — does not - // finish within a second, so the control call itself triggered the - // renewal, the switched issuer answered it, the process went - // terminal mid-request and the control saw a closed connection. - // It failed in CI with "the route answered 0 before any renewal" - // and passes six times out of six locally, which is what a margin - // that depends on machine speed looks like. - // - // Twelve seconds puts the renewal at ~7s and leaves the control - // call seven seconds of room. The wall-clock cost is the price of - // a test that does not depend on the boot being fast. + // Twelve seconds leaves roughly nine seconds for the control call + // before the native broker's renewal lead. Keep that margin: an + // earlier version depended on a loaded runner booting in a second. mint := newHostMint(t, &hostMint{ ttl: 12 * time.Second, switchAfter: 1, @@ -1650,8 +1645,28 @@ func TestABootedRenewalToAnotherExecutionEndsTheProcess(t *testing.T) { if !errors.Is(err, workcontext.ErrMintRefused) { t.Errorf("serve ended with %v, which does not wrap ErrMintRefused: a transient answer must not end this process, and only a terminal one may", err) } - if !strings.Contains(err.Error(), "execution") { - t.Errorf("serve ended with %q, which does not say the execution was replaced", err) + if tc.after.executionFree { + if !strings.Contains(err.Error(), "seals no execution") { + t.Errorf("serve ended with %q, which does not identify the missing execution", err) + } + } else { + // The native broker retains the terminal classification and + // redacts the old/new seals, unlike the former SDK wrapper's + // detailed diagnostic. Do not require those internals to leak. + want := fmt.Sprintf("solution %q stopped serving: %s", testSolutionID, workcontext.ErrMintRefused) + if err.Error() != want { + t.Errorf("serve ended with %q, want the redacted terminal refusal %q", err, want) + } + } + // The renewal landed exactly once and remains terminal after the + // listener stopped; a later caller cannot trigger another mint. + for range 2 { + if _, err := solution.server.acquireCredential(context.Background()); !errors.Is(err, workcontext.ErrMintRefused) { + t.Errorf("the terminal execution refusal was not retained: %v", err) + } + } + if got := mint.count(); got != 2 { + t.Errorf("the host saw %d mints, want the boot and one refused renewal", got) } }) } diff --git a/broker_credential.go b/broker_credential.go new file mode 100644 index 0000000..85a1f90 --- /dev/null +++ b/broker_credential.go @@ -0,0 +1,294 @@ +package solution + +import ( + "bytes" + "context" + "crypto/tls" + "crypto/x509" + "encoding/json" + "errors" + "io" + "net" + "net/http" + "net/url" + "strings" + "sync" + "sync/atomic" + "time" + + corework "github.com/codefly-dev/core/workcontext" + "github.com/codefly-dev/sdk-go/workcontext" +) + +// executionCredential is private so the public SDK-based CredentialSource +// remains source compatible. Core alone inspects capability bytes; this runtime +// never decodes a signed message or treats inspection as verification. +type executionCredential interface { + Token() string + Seal() *workcontext.SealedValues + ExpiresAt() time.Time + Attach(*http.Request) error +} + +type brokerCredential struct { + token string + inspected *corework.Inspected + expires time.Time +} + +func (c brokerCredential) Token() string { return c.token } +func (c brokerCredential) Seal() *workcontext.SealedValues { return c.inspected.Seal() } +func (c brokerCredential) ExpiresAt() time.Time { return c.expires } +func (c brokerCredential) Attach(r *http.Request) error { return workcontext.Attach(r, c.token) } + +type brokerMintOptions struct { + URL, Prefix, InternalToken, ProjectionAudience string + Audience workcontext.AuthorityValue + Authority workcontext.AuthorityPin + ProjectedToken workcontext.ProjectedTokenSource + TrustAnchor func() (*x509.CertPool, error) + ClientCertificate func() (*tls.Certificate, error) + AdmittedPeers func() ([]string, error) +} + +type brokerCredentialSource struct { + options brokerMintOptions + client *http.Client + mu sync.Mutex // the runtime's controller bounds waits and shares acquisition + held executionCredential + renewAt time.Time + terminal error + identity authorityHeldSource + tokenReading atomic.Bool + transportReading atomic.Bool +} + +func newBrokerCredentialSource(options brokerMintOptions) (*brokerCredentialSource, error) { + u, err := url.Parse(options.URL) + if err != nil || u.Scheme != "https" || u.Host == "" || u.User != nil || u.RawQuery != "" || u.Fragment != "" || u.Path != credentialMintPath || u.RawPath != "" { + return nil, errors.New("credential mint URL must be an https module broker endpoint at /modules/_work-context without userinfo, query or fragment") + } + if options.Prefix == "" || options.InternalToken == "" || options.ProjectedToken == nil || options.Authority == nil || options.TrustAnchor == nil || options.ClientCertificate == nil || options.AdmittedPeers == nil { + return nil, errors.New("module broker requires a declared solution prefix, module-authority/INTERNAL_TOKEN, projected token, frozen authority and TLS identity readers") + } + if options.ProjectionAudience != "accounts" { + return nil, errors.New("module-authority/PROJECTION_AUDIENCE must be accounts for the module broker") + } + c := &brokerCredentialSource{options: options} + c.client = c.httpClient(u) + return c, nil +} + +func (c *brokerCredentialSource) Credential(ctx context.Context) (executionCredential, error) { + c.mu.Lock() + defer c.mu.Unlock() + if c.terminal != nil { + return workcontext.Credential{}, c.terminal + } + credential, err := c.obtain(ctx) + if terminalCredentialFailure(err) { + c.terminal = err + } + return credential, err +} + +func (c *brokerCredentialSource) obtain(ctx context.Context) (executionCredential, error) { + fail := func() (executionCredential, error) { + return workcontext.Credential{}, &brokerMintError{class: brokerInvalidAnswer} + } + if err := c.options.Authority.Recheck(ctx); err != nil { + return workcontext.Credential{}, workcontext.ErrMintRefused + } + if c.held != nil && time.Now().Before(c.renewAt) { + return c.held, nil + } + // A FIFO or a consumer reader can ignore cancellation. Bound the wait and + // allow at most one outstanding read, discarding any late result. + projected, err := boundedBrokerRead(ctx, &c.tokenReading, c.options.ProjectedToken.ProjectedToken) + if err != nil { + var projection *projectedTokenError + if errors.As(err, &projection) { + return workcontext.Credential{}, &brokerMintError{class: brokerProjection} + } + return workcontext.Credential{}, classifyBrokerTransport(err) + } + if strings.TrimSpace(projected) == "" { + return workcontext.Credential{}, &brokerMintError{class: brokerProjection} + } + payload, _ := json.Marshal(map[string]string{"prefix": c.options.Prefix}) + req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.options.URL, bytes.NewReader(payload)) + if err != nil { + return workcontext.Credential{}, classifyBrokerTransport(err) + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Accept", "application/json") + req.Header.Set("Authorization", "Bearer "+projected) + req.Header.Set("X-Codefly-Internal-Token", c.options.InternalToken) + resp, err := c.client.Do(req) + if err != nil { + return workcontext.Credential{}, classifyBrokerTransport(err) + } + // The response is classified and bounded below; closing is cleanup only. + // Do not drain an untrusted body to reuse a connection: every mint closes it. + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode != http.StatusOK { + return workcontext.Credential{}, brokerStatusError(resp.StatusCode) + } + raw, err := io.ReadAll(io.LimitReader(resp.Body, (64<<10)+1)) + if err != nil { + return workcontext.Credential{}, classifyBrokerTransport(err) + } + if len(raw) > 64<<10 { + return fail() + } + // These are the broker's HTTP fields, not fields of a signed capability. + var issued struct { + Token string `json:"token"` + ExpiresAt string `json:"expiresAt"` + PrincipalID string `json:"principalId"` + Tenant string `json:"tenant"` + } + if json.Unmarshal(raw, &issued) != nil { + return fail() + } + expiry, err := time.Parse(time.RFC3339, issued.ExpiresAt) + if err != nil || !expiry.After(time.Now()) { + return fail() + } + inspected, err := corework.Inspect(issued.Token) + if err != nil { + return fail() + } + audience, err := c.options.Authority.Value(c.options.Audience.Name, c.options.Audience.Key) + if err != nil || inspected.Context().GetAudience() != audience { + return fail() + } + if time.Now().Before(inspected.NotBefore().Add(-corework.DefaultSkew)) { + return fail() + } + // The broker's expiry may shorten, never extend, the signed lifetime. + if inspected.ExpiresAt().Before(expiry) { + expiry = inspected.ExpiresAt() + } + credential := brokerCredential{token: issued.Token, inspected: inspected, expires: expiry} + if err := usableCredential(credential); err != nil { + if errors.Is(err, workcontext.ErrMintRefused) { + return workcontext.Credential{}, err + } + return workcontext.Credential{}, &brokerMintError{class: brokerInvalidAnswer} + } + if err := c.options.Authority.Recheck(ctx); err != nil { + return workcontext.Credential{}, workcontext.ErrMintRefused + } + if err := c.identity.holdSealedIdentity(credential.Seal()); err != nil { + return workcontext.Credential{}, workcontext.ErrMintRefused + } + lead := min(30*time.Second, expiry.Sub(inspected.NotBefore())/5) + c.renewAt = expiry.Add(-lead) + if !time.Now().Before(c.renewAt) { + return fail() + } + c.held = credential + return credential, nil +} + +// This transport is private, uses a fresh mutually authenticated handshake for +// every mint and rechecks trust after it, before any HTTP bytes can be sent. +func (c *brokerCredentialSource) httpClient(endpoint *url.URL) *http.Client { + o := c.options + trust := func() (*x509.CertPool, error) { + pool, err := o.TrustAnchor() + if err != nil || pool == nil { + return nil, errors.New("broker trust anchor unavailable") + } + return pool, nil + } + // Build an owned transport; never inherit mutable global transport hooks. + transport := &http.Transport{DisableKeepAlives: true} + port := endpoint.Port() + if port == "" { + port = "443" + } + address := net.JoinHostPort(endpoint.Hostname(), port) + transport.DialTLSContext = func(ctx context.Context, network, addr string) (net.Conn, error) { + if addr != address { + return nil, errors.New("broker origin mismatch") + } + ctx, cancel := context.WithTimeout(ctx, platformHandshakeTimeout) + defer cancel() + return boundedBrokerRead(ctx, &c.transportReading, func() (net.Conn, error) { + anchor, err := trust() + if err != nil || anchor == nil { + return nil, errors.New("broker trust anchor unavailable") + } + presented := false + config := &tls.Config{MinVersion: tls.VersionTLS13, RootCAs: anchor, ServerName: endpoint.Hostname(), SessionTicketsDisabled: true} + config.GetClientCertificate = func(*tls.CertificateRequestInfo) (*tls.Certificate, error) { + pair, err := o.ClientCertificate() + if err != nil || pair == nil || len(pair.Certificate) == 0 || pair.PrivateKey == nil { + return nil, errors.New("broker client identity unavailable") + } + presented = true + return pair, nil + } + admitOnlyPlatform(config, o.AdmittedPeers) + raw, err := (&net.Dialer{}).DialContext(ctx, network, addr) + if err != nil { + return nil, err + } + conn := tls.Client(raw, config) + if err := conn.HandshakeContext(ctx); err != nil { + _ = conn.Close() + return nil, err + } + if !presented { + _ = conn.Close() + return nil, errors.New("broker did not request a client identity") + } + if err := peerStillTrusted(conn.ConnectionState(), endpoint.Hostname(), trust, o.AdmittedPeers); err != nil { + _ = conn.Close() + return nil, err + } + return conn, nil + }) + } + return &http.Client{Transport: transport, Timeout: platformRequestTimeout, CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }} +} + +func boundedBrokerRead[T any](ctx context.Context, busy *atomic.Bool, read func() (T, error)) (T, error) { + var zero T + if err := ctx.Err(); err != nil { + return zero, err + } + if !busy.CompareAndSwap(false, true) { + return zero, context.DeadlineExceeded + } + type result struct { + value T + err error + } + answer := make(chan result) + go func() { + defer busy.Store(false) + value, err := read() + select { + case answer <- result{value, err}: + case <-ctx.Done(): + if closer, ok := any(value).(io.Closer); ok && closer != nil { + _ = closer.Close() + } + } + }() + select { + case got := <-answer: + if err := ctx.Err(); err != nil { + if closer, ok := any(got.value).(io.Closer); ok && closer != nil { + _ = closer.Close() + } + return zero, err + } + return got.value, got.err + case <-ctx.Done(): + return zero, ctx.Err() + } +} diff --git a/broker_credential_test.go b/broker_credential_test.go new file mode 100644 index 0000000..e538da3 --- /dev/null +++ b/broker_credential_test.go @@ -0,0 +1,423 @@ +package solution + +import ( + "context" + "crypto/tls" + "crypto/x509" + "encoding/json" + "errors" + "fmt" + "io" + "net" + "net/http" + "net/http/httptest" + "net/url" + "path/filepath" + "reflect" + "strings" + "sync/atomic" + "syscall" + "testing" + "time" + + codefly "github.com/codefly-dev/sdk-go" + "github.com/codefly-dev/sdk-go/workcontext" +) + +// The fake broker runs as an HTTP handler through an in-memory transport. This +// tests the actual request/response boundary even where listening is forbidden. +func brokerFixture(t *testing.T) (*Server, *hostMint, string) { + t.Helper() + path := filepath.Join(t.TempDir(), "projection") + writeFile(t, path, "projected-first") + host := &hostMint{ttl: time.Hour} + host.prepareAuthority(t) + s := New(Manifest{ID: testSolutionID}) + s.cfg.mintURL = "https://broker.example/modules/_work-context" + s.cfg.brokerInternalToken = "broker-perimeter-token" + s.cfg.projectedTokenPath = path + options := s.mintOptions("accounts", func() (*x509.CertPool, error) { return x509.NewCertPool(), nil }, func(*tls.CertificateRequestInfo) (*tls.Certificate, error) { return &tls.Certificate{}, nil }) + options.Authority = fixedAuthority{audience: testAudience} + c, err := newBrokerCredentialSource(options) + if err != nil { + t.Fatal(err) + } + c.client.Transport = mintRoundTrip(func(r *http.Request) (*http.Response, error) { + response := httptest.NewRecorder() + host.serve(response, r) + return response.Result(), nil + }) + s.brokerCredential = c + return s, host, path +} + +func TestBrokerExecutionProtocolConformance(t *testing.T) { + s, host, path := brokerFixture(t) + c := s.brokerCredential + wire := c.client.Transport + expectedProjection := "projected-first" + c.client.Transport = mintRoundTrip(func(r *http.Request) (*http.Response, error) { + if r.Method != "POST" || r.URL.String() != "https://broker.example/modules/_work-context" { + t.Fatalf("wrong broker method or endpoint: %s %s", r.Method, r.URL) + } + for key, want := range map[string]string{"Authorization": "Bearer " + expectedProjection, "X-Codefly-Internal-Token": "broker-perimeter-token", "Content-Type": "application/json", "Accept": "application/json"} { + if r.Header.Get(key) != want { + t.Fatalf("wrong broker header %s", key) + } + } + if r.Header.Get("X-Codefly-Module-Secret") != "" || r.Header.Get("X-Codefly-Module-Registration") != "" { + t.Fatal("legacy credential header reached broker") + } + raw, err := io.ReadAll(r.Body) + if err != nil { + t.Fatal(err) + } + var body map[string]any + if json.Unmarshal(raw, &body) != nil || !reflect.DeepEqual(body, map[string]any{"prefix": "widgets-go"}) { + t.Fatalf("broker body must contain exactly the declared prefix: %s", raw) + } + r.Body = io.NopCloser(strings.NewReader(string(raw))) + return wire.RoundTrip(r) + }) + if err := s.openCredential(context.Background()); err != nil { + t.Fatal(err) + } + first, err := s.acquireCredential(context.Background()) + if err != nil || first.Token() == "" { + t.Fatalf("broker response was not accepted: %v", err) + } + for range 20 { + held, err := s.acquireCredential(context.Background()) + if err != nil || held.Token() != first.Token() { + t.Fatalf("cached execution changed: %v", err) + } + } + if host.count() != 1 { + t.Fatalf("cached execution minted %d times", host.count()) + } + // The controller's viewer gateway and health gate both see the broker source. + if s.gatewayFor(http.Header{}).workload == nil { + t.Fatal("viewer mint lost execution attestation") + } + expectedProjection = "projected-rotated" + writeFile(t, path, expectedProjection) + c.renewAt = time.Time{} + renewed, err := s.acquireCredential(context.Background()) + if err != nil || renewed.Token() == first.Token() || host.count() != 2 { + t.Fatalf("renewal did not re-read projection: %v", err) + } +} + +type unreadBrokerBody struct{} + +func (unreadBrokerBody) Read([]byte) (int, error) { panic("non-200 broker body was read") } +func (unreadBrokerBody) Close() error { return nil } + +func TestBrokerStatusClassesAndRetryAllowlist(t *testing.T) { + for _, status := range []int{201, 204, 301, 307, 400, 401, 403, 404, 408, 409, 422, 429, 500, 502, 503, 504} { + t.Run(fmt.Sprint(status), func(t *testing.T) { + s, _, _ := brokerFixture(t) + var calls atomic.Int64 + s.brokerCredential.client.Transport = mintRoundTrip(func(*http.Request) (*http.Response, error) { + calls.Add(1) + return &http.Response{StatusCode: status, Body: unreadBrokerBody{}, Header: http.Header{"Location": {"https://elsewhere.example/private-token"}}}, nil + }) + _, firstErr := s.brokerCredential.Credential(context.Background()) + s.firstMintWindow = 60 * time.Millisecond + err := s.openCredential(context.Background()) + var broker *brokerMintError + if !errors.As(firstErr, &broker) || broker.status != status { + t.Fatalf("lost status: %v", err) + } + retry := status == 503 + refused := status == 401 || status == 403 + if broker.retryable() != retry || errors.Is(firstErr, workcontext.ErrMintRefused) != refused || errors.Is(firstErr, workcontext.ErrMintUnavailable) != retry || errors.Is(err, workcontext.ErrMintUnavailable) != retry { + t.Fatalf("incorrect classification: %v", err) + } + if strings.Contains(err.Error(), "private-token") { + t.Fatal("broker diagnostics escaped") + } + if retry { + if calls.Load() < 2 { + t.Fatalf("503 not retried: %d", calls.Load()) + } + } else { + if calls.Load() != 1 { + t.Fatalf("terminal HTTP %d retried %d times", status, calls.Load()) + } + // A renewal uses the same classification and records it even with no waiter. + for range 3 { + _, _ = s.acquireCredential(context.Background()) + } + if calls.Load() != 1 || s.terminalErr.Load() == nil { + t.Fatal("terminal result was not retained by renewal controller") + } + } + }) + } +} + +func TestBrokerTransportCategoriesAndRetryAllowlist(t *testing.T) { + for _, tc := range []struct { + name string + cause error + category brokerTransportCategory + retry bool + }{ + {"deadline", context.DeadlineExceeded, brokerTransportTimeout, true}, + {"timeout", &net.DNSError{Err: "private-token", IsTimeout: true}, brokerTransportTimeout, true}, + {"refused", syscall.ECONNREFUSED, brokerTransportRefused, true}, + {"reset", syscall.ECONNRESET, brokerTransportReset, true}, + {"cancelled", context.Canceled, brokerTransportOther, false}, + {"tls", x509.UnknownAuthorityError{}, brokerTransportTLS, false}, + {"dns", &net.DNSError{Err: "private-token", IsNotFound: true}, brokerTransportOther, false}, + {"parser", errors.New("private-token: malformed HTTP response"), brokerTransportOther, false}, + {"eof", io.ErrUnexpectedEOF, brokerTransportOther, false}, + } { + t.Run(tc.name, func(t *testing.T) { + s, _, _ := brokerFixture(t) + var calls atomic.Int64 + raw := &url.Error{Op: "POST", URL: "https://private-token.example", Err: fmt.Errorf("private-token: %w", tc.cause)} + s.brokerCredential.client.Transport = mintRoundTrip(func(*http.Request) (*http.Response, error) { calls.Add(1); return nil, raw }) + _, firstErr := s.brokerCredential.Credential(context.Background()) + s.firstMintWindow = 60 * time.Millisecond + err := s.openCredential(context.Background()) + var broker *brokerMintError + if !errors.As(firstErr, &broker) || broker.transport != tc.category || broker.retryable() != tc.retry || errors.Is(err, workcontext.ErrMintUnavailable) != tc.retry { + t.Fatalf("incorrect transport class: %v", err) + } + if strings.Contains(err.Error(), "private-token") || errors.Is(err, raw) || errors.Is(err, tc.cause) { + t.Fatalf("raw transport cause escaped: %v", err) + } + if tc.retry && calls.Load() < 2 || !tc.retry && calls.Load() != 1 { + t.Fatalf("retry=%v, calls=%d", tc.retry, calls.Load()) + } + }) + } +} + +func TestBrokerInvalidResponsesAreTerminal(t *testing.T) { + for _, raw := range []string{`{}`, `null`, `{"work_context":"old-protocol"}`, `{"token":"invalid","expiresAt":"2099-01-01T00:00:00Z","principalId":"principal","tenant":"tenant"}`, `{"token":"private-token","expiresAt":"private-token"}`, `{"token":`, strings.Repeat("x", (64<<10)+1)} { + s, _, _ := brokerFixture(t) + calls := 0 + s.brokerCredential.client.Transport = mintRoundTrip(func(*http.Request) (*http.Response, error) { + calls++ + return &http.Response{StatusCode: 200, Body: io.NopCloser(strings.NewReader(raw))}, nil + }) + for range 2 { + _, err := s.brokerCredential.Credential(context.Background()) + if !terminalCredentialFailure(err) || errors.Is(err, workcontext.ErrMintUnavailable) || strings.Contains(err.Error(), "private-token") { + t.Fatalf("invalid broker answer not safely terminal: %v", err) + } + } + if calls != 1 { + t.Fatalf("invalid response retried: %d", calls) + } + } +} + +func TestBrokerConfigurationUsesDeclaredSecretAndRefusesMissingValues(t *testing.T) { + authorityValues(t) + t.Setenv("CODEFLY_INTERNAL_TOKEN", "legacy-token-must-not-be-used") + cfg := loadConfig(context.Background(), testSolutionID, nil) + if cfg.brokerInternalToken != "broker-perimeter-token" { + t.Fatal("broker did not read declared workspace secret") + } + t.Setenv("CODEFLY__WORKSPACE_SECRET_CONFIGURATION__MODULE_AUTHORITY__INTERNAL_TOKEN", "") + if err := codefly.LoadEnvironmentVariables(); err != nil { + t.Fatal(err) + } + cfg = loadConfig(context.Background(), testSolutionID, nil) + cfg.port, cfg.gatewayURL, cfg.mintURL, cfg.profile = "8080", "https://broker.example", "https://broker.example/modules/_work-context", localProfile + if err := cfg.validate(); err == nil || !strings.Contains(err.Error(), "module-authority/INTERNAL_TOKEN") { + t.Fatalf("missing declared perimeter token was accepted: %v", err) + } + s, _, _ := brokerFixture(t) + options := s.brokerCredential.options + for _, field := range []string{"prefix", "secret", "audience", "path"} { + bad := options + switch field { + case "prefix": + bad.Prefix = "" + case "secret": + bad.InternalToken = "" + case "audience": + bad.ProjectionAudience = "different" + case "path": + bad.URL = "https://broker.example/platform/_credential" + } + if _, err := newBrokerCredentialSource(bad); err == nil { + t.Fatalf("accepted missing/wrong %s", field) + } + } +} + +type brokerTokenRead func() (string, error) + +func (read brokerTokenRead) ProjectedToken() (string, error) { return read() } + +func TestBrokerProjectionWaitIsBoundedAndDoesNotAccumulateReaders(t *testing.T) { + s, host, _ := brokerFixture(t) + c := s.brokerCredential + release := make(chan struct{}) + var calls atomic.Int64 + c.options.ProjectedToken = brokerTokenRead(func() (string, error) { + calls.Add(1) + <-release + return "late-projection", nil + }) + for range 3 { + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond) + _, err := c.Credential(ctx) + cancel() + if !errors.Is(err, workcontext.ErrMintUnavailable) { + t.Fatalf("blocked read lost its timeout category: %v", err) + } + } + if calls.Load() != 1 || host.count() != 0 { + t.Fatal("blocked reads accumulated or disclosed a late projection") + } + close(release) + deadline := time.Now().Add(time.Second) + for c.tokenReading.Load() && time.Now().Before(deadline) { + time.Sleep(time.Millisecond) + } + if c.tokenReading.Load() { + t.Fatal("reader slot not released") + } + if host.count() != 0 { + t.Fatal("late token sent a mint") + } +} + +func TestBrokerTransientRenewalKeepsHeldCredentialAndBacksOff(t *testing.T) { + s, host, _ := brokerFixture(t) + if err := s.openCredential(context.Background()); err != nil { + t.Fatal(err) + } + initial, _ := s.heldCredential() + s.brokerCredential.renewAt = time.Time{} + host.status = 503 + for range 20 { + held, err := s.acquireCredential(context.Background()) + if err != nil || held.Token() != initial.Token() { + t.Fatalf("transient renewal discarded held credential: %v", err) + } + } + if host.count() != 2 || s.terminalErr.Load() != nil { + t.Fatalf("transient renewal did not back off: %d mints", host.count()) + } +} + +func TestBrokerTerminalRenewalDiscardsHeldAuthority(t *testing.T) { + s, host, _ := brokerFixture(t) + if err := s.openCredential(context.Background()); err != nil { + t.Fatal(err) + } + s.brokerCredential.renewAt = time.Time{} + host.status = 429 + for range 3 { + _, err := s.acquireCredential(context.Background()) + var broker *brokerMintError + if !errors.Is(err, ErrCredentialRefused) || !errors.As(err, &broker) || broker.status != 429 { + t.Fatalf("terminal renewal served held authority or lost classification: %v", err) + } + } + if host.count() != 2 { + t.Fatalf("terminal renewal retried: %d", host.count()) + } + if err := s.actingForAViewer(context.Background()); err == nil { + t.Fatal("viewer gate ignored terminal broker failure") + } + recorder := httptest.NewRecorder() + s.handleHealth(recorder, httptest.NewRequest("GET", "/health", nil)) + if recorder.Code == http.StatusOK { + t.Fatal("health accepted terminal broker failure") + } +} + +type brokerAuthorityPin struct { + fixedAuthority + drift atomic.Bool +} + +func (p *brokerAuthorityPin) Recheck(context.Context) error { + if p.drift.Load() { + return errors.New("private-authority-diagnostic") + } + return nil +} + +func TestBrokerHoldsAuthorityAndExecutionAcrossRenewals(t *testing.T) { + for _, when := range []string{"before", "during"} { + t.Run("authority-"+when, func(t *testing.T) { + s, host, _ := brokerFixture(t) + c := s.brokerCredential + pin := &brokerAuthorityPin{fixedAuthority: fixedAuthority{audience: testAudience}} + c.options.Authority = pin + if _, err := c.Credential(context.Background()); err != nil { + t.Fatal(err) + } + c.renewAt = time.Time{} + if when == "before" { + pin.drift.Store(true) + } else { + wire := c.client.Transport + c.client.Transport = mintRoundTrip(func(r *http.Request) (*http.Response, error) { + response, err := wire.RoundTrip(r) + pin.drift.Store(true) + return response, err + }) + } + for range 2 { + if _, err := c.Credential(context.Background()); !errors.Is(err, workcontext.ErrMintRefused) || strings.Contains(err.Error(), "private-authority") { + t.Fatalf("authority drift was accepted or exposed: %v", err) + } + } + want := int64(1) + if when == "during" { + want = 2 + } + if host.count() != want { + t.Fatalf("drift minted %d times, want %d", host.count(), want) + } + }) + } + for _, tc := range []struct { + name string + after *hostMint + }{ + {"execution missing", &hostMint{executionFree: true}}, + {"execution changed", &hostMint{incarnation: 99}}, + } { + t.Run(tc.name, func(t *testing.T) { + s, host, _ := brokerFixture(t) + host.switchTo = tc.after + host.switchTo.prepareAuthority(t) + host.switchAfter = 1 + if err := s.openCredential(context.Background()); err != nil { + t.Fatal(err) + } + s.brokerCredential.renewAt = time.Time{} + for range 2 { + if _, err := s.acquireCredential(context.Background()); !errors.Is(err, workcontext.ErrMintRefused) || !errors.Is(err, ErrCredentialRefused) { + t.Fatalf("execution change accepted or not terminal: %v", err) + } + } + if host.count() != 2 { + t.Fatalf("execution change retried: %d", host.count()) + } + // Exercise the native broker through the controller without a + // socket. The booted test separately proves serve observes this + // signal and returns without cancellation by the test. + select { + case <-s.credentialRefusedC(): + default: + t.Fatal("the execution refusal did not signal serve to stop") + } + if terminal := s.terminalErr.Load(); terminal == nil || !errors.Is(*terminal, workcontext.ErrMintRefused) { + t.Fatal("the execution refusal was not recorded for serve") + } + }) + } +} diff --git a/contract_test.go b/contract_test.go index b8fec53..5756baf 100644 --- a/contract_test.go +++ b/contract_test.go @@ -21,7 +21,7 @@ func contractServer(t *testing.T, profile string, contract ModuleContract, modul modules = []ConsumedModule{passthroughModule()} } server := New(Manifest{ID: testSolutionID}).Consumes(modules...).Contract(contract) - server.cfg = config{profile: profile, consumes: newProjection(consumesThings)} + server.cfg = config{brokerInternalToken: "broker-perimeter-token", profile: profile, consumes: newProjection(consumesThings)} server.principal = testPrincipal return server } @@ -276,7 +276,7 @@ func TestAnUnusableProfileNameIsRefused(t *testing.T) { }}, passthroughModule()); err == nil { t.Error("a traversing profile name was rendered") } - cfg := config{ + cfg := config{brokerInternalToken: "broker-perimeter-token", port: "8080", gatewayURL: "https://gateway:42152", mintURL: "https://gateway:42152" + credentialMintPath, identityCertFile: "c", identityKeyFile: "k", projectedTokenPath: "t", trustBundleFile: "b", profile: "../local", } @@ -706,7 +706,7 @@ func TestTheDeclaredCeilingIsFrozenAtDeclaration(t *testing.T) { Contract(ModuleContract{Ceilings: map[string]map[string][]Scope{ localProfile: {"things": {{ResourceKind: "things", Actions: ceilingActions}}}, }}) - server.cfg = config{gatewayURL: gw.URL, profile: localProfile, consumes: newProjection(consumesThings)} + server.cfg = config{brokerInternalToken: "broker-perimeter-token", gatewayURL: gw.URL, profile: localProfile, consumes: newProjection(consumesThings)} server.principal = testPrincipal contract, err := server.resolveContract() if err != nil { diff --git a/credential.go b/credential.go index edf0be7..4c5db26 100644 --- a/credential.go +++ b/credential.go @@ -15,20 +15,9 @@ import ( "github.com/codefly-dev/sdk-go/workcontext" ) -// CredentialSource is where this execution's credential comes from. -// -// It is the SDK mint client's own method, so *workcontext.MintClient satisfies -// it directly and this runtime holds no mint of its own: obtaining a credential -// from a projected service-account token, sealing it to a build incarnation and -// an installation, re-reading the rotated projection and rechecking the -// boot-read authority before each renewal are all the SDK's, and this runtime -// decides only when a credential is needed and what happens when there is none. -// -// Every use calls it. The client mints the first credential, hands back the one -// it holds while that one is current, and renews it once it has entered its -// renewal lead — so there is no path through this runtime on which a lapsed -// credential is presented, and no timer here that is not the credential's own -// expiry. +// CredentialSource supplies an execution credential from a consumer's issuer. +// The platform broker uses an internal source; this SDK-compatible hook remains +// available to existing consumers without changing their return type. type CredentialSource interface { Credential(ctx context.Context) (workcontext.Credential, error) } @@ -71,9 +60,8 @@ const ( // against, as the host names it. AuthorityAudienceKey = "AUDIENCE" // AuthorityProjectionAudienceKey is the audience the projected - // service-account token was itself minted for. It is sent with the mint so - // the host can refuse a projection aimed at something else, rather than - // reviewing whatever it is handed. + // service-account token was itself minted for. The broker requires accounts; + // this value is checked locally and is not a broker request field. AuthorityProjectionAudienceKey = "PROJECTION_AUDIENCE" ) @@ -94,7 +82,7 @@ func readAuthority(ctx context.Context) (*codefly.Authority, error) { return authority, nil } -// platformCredentialSource is the default source: the SDK's mint client, +// platformCredentialSource is the native module broker client, // pointed at the host's mint endpoint, presenting the service-account token the // platform projects for this workload. // @@ -103,7 +91,7 @@ func readAuthority(ctx context.Context) (*codefly.Authority, error) { // projected, is not proxied, and does not follow redirects: the request carries // the projected token that attests which workload this process is, and a // Location would hand that to whatever answered at the mint URL. -func (s *Server) platformCredentialSource() (CredentialSource, error) { +func (s *Server) platformCredentialSource() (*brokerCredentialSource, error) { projectionAudience, err := s.authority.Value(AuthorityGroup, AuthorityProjectionAudienceKey) if err != nil { return nil, err @@ -147,11 +135,11 @@ func (s *Server) platformCredentialSource() (CredentialSource, error) { return nil, fmt.Errorf("this workload would present a certificate the host has not approved it under to the credential mint: %w", err) } } - client, err := workcontext.NewMintClient(s.mintOptions(projectionAudience, s.peerTrustAnchor(identity), held)) + client, err := newBrokerCredentialSource(s.mintOptions(projectionAudience, s.peerTrustAnchor(identity), held)) if err != nil { return nil, fmt.Errorf("configure this workload's credential mint at %s: %w", s.cfg.mintURL, err) } - return &redactedMintSource{client: client}, nil + return client, nil } // heldToTheFrozenAuthority rechecks the authority whenever a source hands back @@ -290,63 +278,24 @@ type sealedIdentity struct { incarnation uint64 } -// mintOptions is how this runtime asks the SDK's mint client for a credential. -// -// A function of its own so it can be read in a test. Every field here is a -// decision, and the one most easily lost is Authority: the SDK rechecks it -// before every renewal, which is the single moment a drifted authority value -// would otherwise be laundered into a credential nobody approved. Dropping it -// leaves the boot and the first mint working perfectly and only renewals wrong, -// which is not a shape a test over the happy path can see. +// mintOptions binds the declared prefix and perimeter credential to the frozen +// authority and live TLS readers. No environment is read after loadConfig. func (s *Server) mintOptions( projectionAudience string, anchor func() (*x509.CertPool, error), leaf func(*tls.CertificateRequestInfo) (*tls.Certificate, error), -) workcontext.MintOptions { - return workcontext.MintOptions{ - URL: s.cfg.mintURL, - // The pinned value the audience is READ from, not an audience this - // runtime resolved and passed beside the pin. The SDK made that a - // typed reference for a reason worth keeping in mind here: a string - // passed alongside the pin meant the drift check guarded a value the - // mint did not use. +) brokerMintOptions { + return brokerMintOptions{ + URL: s.cfg.mintURL, + Prefix: s.manifest.ID, + InternalToken: s.cfg.brokerInternalToken, Audience: workcontext.AuthorityValue{Name: AuthorityGroup, Key: AuthorityAudienceKey}, ProjectedToken: projectedTokenFile(s.cfg.projectedTokenPath), ProjectionAudience: projectionAudience, - // The frozen reader, not a value read again here: it is what rechecks - // the authority-bearing values before each mint, including the first. - Authority: s.authority, - // The transport's three judgements, as READERS rather than values. - // - // This is sdk-go#51, and it is the shape this runtime argued for from - // both ends. The SDK still builds and owns the transport — a supplied - // *http.Client is a hole it cannot inspect, since a nil Transport - // means the global mutable default, a DialTLSContext bypasses - // TLSClientConfig entirely, and a caller holding the same transport - // can turn verification off after construction — but it no longer - // takes a pool by value and fixes it for the client's lifetime. - // - // Every one of these is called DURING the handshake, so withdrawal - // takes effect on the next handshake rather than on the next process: - // - // - TrustAnchor is the same reader the listener and the gateway dial - // use, read before and after each handshake and never cached, so a - // removed root stops this process dialling the mint. - // - ClientCertificate makes the mint request present this workload's - // X.509-SVID again. The projected service-account token is no - // longer the whole of what attests who is asking. - // - AdmittedPeers is MINT_PEERS_FILE, read per handshake, so the - // mint endpoint is held to WHICH party it is and not merely to a - // certificate from the right CA. Every workload in the trust - // domain holds one of those, which is why authentication was never - // authorisation here. - TrustAnchor: anchor, - // Asked with an empty request rather than nil: a consumer's own - // GetClientCertificate may read the acceptable CAs off it, and handing - // it nil would be this runtime choosing a dereference on somebody - // else's code path. - ClientCertificate: func() (*tls.Certificate, error) { return leaf(&tls.CertificateRequestInfo{}) }, - AdmittedPeers: s.admittedMint(), + Authority: s.authority, + TrustAnchor: anchor, + ClientCertificate: func() (*tls.Certificate, error) { return leaf(&tls.CertificateRequestInfo{}) }, + AdmittedPeers: s.admittedMint(), } } @@ -382,25 +331,17 @@ func (s *Server) mintOptions( // orchestrator whose job is to restart it, and a restart is cheaper to read // than a solution that serves nothing while logging forever. func (s *Server) openCredential(ctx context.Context) error { - if s.credential == nil { + if s.credential == nil && s.brokerCredential == nil { source, err := s.platformCredentialSource() if err != nil { return err } - s.credential = source + s.brokerCredential = source + } + // Consumer sources receive the same authority hold as the native broker. + if s.credential != nil { + s.credential = heldToTheFrozenAuthority(s.credential, s.authority) } - // Whatever the source is, the authority recheck is this runtime's promise - // and not the source's. - // - // A supplied source replaces the mint client, and with it every option - // mintOptions carries — including Authority, which the SDK rechecks before - // each renewal. So the one guarantee that exists to stop a drifted - // authority value being laundered into a differently-sealed credential - // applied to the platform path only, while this package's own - // documentation promises it of the process. Wrapping here makes the promise - // true of every source, and it costs the platform source nothing: the SDK - // does it too, and a recheck of three frozen values is a comparison. - s.credential = heldToTheFrozenAuthority(s.credential, s.authority) window := s.firstMintWindow if window == 0 { window = firstMintWait @@ -423,7 +364,7 @@ func (s *Server) openCredential(ctx context.Context) error { wait = quarter } for attempt := 1; ; attempt++ { - credential, err := s.credential.Credential(deadline) + credential, err := s.nextCredential(deadline) switch { case err == nil && usableCredential(credential) != nil: // A source may hand back the zero credential or an expired one, @@ -475,7 +416,11 @@ func (s *Server) openCredential(ctx context.Context) error { return fmt.Errorf("obtain this execution's credential from %s: still unavailable after %s and %d attempt(s): %w — the issuer answers this way while a presence generation has not been applied yet, or while it cannot reach the Kubernetes API or its policy log; all three are transient, so this is an exit for the orchestrator to retry rather than a judgement on this build", s.cfg.mintURL, window, attempt, err) } - if remaining := time.Until(mustDeadline(deadline)); wait > remaining { + remaining := time.Until(mustDeadline(deadline)) + if remaining <= 0 { + return fmt.Errorf("obtain this execution's credential from %s: still unavailable after %s (transient): %w", s.cfg.mintURL, window, err) + } + if wait > remaining { wait = remaining } log.Printf("solution %q: this execution's credential is not available yet (attempt %d, retrying in %s): %v", @@ -575,7 +520,7 @@ func (s *Server) openAuthority(ctx context.Context) error { // the held-credential fallback that every other caller goes through did not // apply to the viewer's mint, which is the hottest path there is. One // controller means one controller. -type credentialAcquirer func(context.Context) (workcontext.Credential, error) +type credentialAcquirer func(context.Context) (executionCredential, error) // attestWorkloadReporting is attestWorkload with somewhere to report a // *terminal* failure, which is the half that is easy to get backwards. @@ -676,31 +621,22 @@ func (r *attestationReport) recovered(id string) { } } -// terminalCredentialFailure reports whether the issuer's answer is a judgement -// no retry can change. -// -// ErrMintRefused only, and the correction is worth recording because the -// it is easy to get from the wrong direction. It also treated ErrRevoked -// as terminal here, on the reasoning that live state moving under a sound -// credential is as final as a refused build. Two things are wrong with that: -// the mint client does not return ErrRevoked at all (it comes back from a -// *callee* that was shown a capability sealed to state that has moved), and the -// SDK's own contract for it is a refresh and one retry — so treating it as -// terminal would have ended the process on a condition the client is built to -// recover from, if the path had ever been reachable. It was not, which is why -// no test noticed; a dead branch that would do the wrong thing if it ever woke -// up is worse than no branch. -// -// A callee's ErrRevoked arrives on a viewer's own request and is answered -// there, as a 409 to that viewer. Nothing is inferred from it about the -// capability or about this execution's credential: a 409 means "the state you -// were sealed to has moved" and it means "that page already exists", and -// nothing on the wire says which (see the RoundTrip comment in solution.go). -// -// Everything else, ErrMintUnavailable above all, is transient by construction -// and must not end the process: an issuer that cannot reach its own policy log -// is an issuer behaving correctly, and exiting on it would turn a dependency -// blip into a crash loop. +// terminalCredentialFailure preserves refusals separately from terminal broker +// protocol/configuration errors. Neither may cause another execution mint. func terminalCredentialFailure(err error) bool { - return errors.Is(err, workcontext.ErrMintRefused) + var broker *brokerMintError + return errors.Is(err, workcontext.ErrMintRefused) || + (errors.As(err, &broker) && !broker.retryable()) +} + +// Both sources enter the same acquisition controller; the public SDK source +// keeps its original signature while the broker carries Core's inspected value. +func (s *Server) nextCredential(ctx context.Context) (executionCredential, error) { + if s.credential != nil { + return s.credential.Credential(ctx) + } + if s.brokerCredential != nil { + return s.brokerCredential.Credential(ctx) + } + return workcontext.Credential{}, errors.New("no credential source") } diff --git a/credential_test.go b/credential_test.go index 9daf4ae..26f972a 100644 --- a/credential_test.go +++ b/credential_test.go @@ -37,21 +37,6 @@ func TestARefusedMintFailsTheBootAndIsNeverRetried(t *testing.T) { }{ {"the host refuses this workload's build or identity", http.StatusForbidden}, {"the projected token is not acceptable", http.StatusUnauthorized}, - // 404 is NOT here any more, and that is sdk-go v0.3.0 inverting the - // default rather than this test being relaxed. - // - // The rule was "429 and 5xx retry, everything else is terminal", which - // made every status anything between the host and this process might - // invent into a permanent stop — measured: a 408 from a proxy, with a - // valid credential in hand, latched for the life of the process, and a - // 404 from an ingress mid-rollout would do the same. Only 401 and 403 - // latch now, because those are the two the host signs: the projected - // token is not acceptable, or the build is not approved. - // - // An old runtime against a new host still does not recover — it waits - // out its bounded window and exits non-zero for the orchestrator - // instead of failing on the first answer. Covered by the unavailable - // cases, and AGENTS.md says so now. } { t.Run(tc.name, func(t *testing.T) { mint := newHostMint(t, &hostMint{status: tc.status}) @@ -358,6 +343,8 @@ func TestABootedRuntimeRegistersNothing(t *testing.T) { "GATEWAY_MODULE_REGISTER_URL": legacy.URL + "/modules/_register", "GATEWAY_MODULE_REGISTRATION_TOKEN_URL": legacy.URL + "/modules/_registration-token", "GATEWAY_SOLUTION_REGISTRATION_TOKEN_URL": legacy.URL + "/solutions/_registration-token", + "CODEFLY__SOLUTION_REGISTRATION_SECRET": "legacy-solution-secret", + "CODEFLY__MODULE_REGISTRATION_SECRETS": `{"widgets-go":"legacy-module-secret"}`, "CODEFLY_INTERNAL_TOKEN": "internal-token-that-must-not-travel", "CODEFLY__SOLUTION_REGISTRATION_INTERVAL": "1s", "CODEFLY_HOST_FRONTEND": "frontend", @@ -470,8 +457,12 @@ func TestABootedRuntimeRegistersNothing(t *testing.T) { if got := mint.count(); got != 1 { t.Errorf("the host received %d requests, want exactly 1 (the mint): anything more is a registration that survived", got) } + // The broker requires a perimeter header, but its value must come from the + // declared workspace secret, never the nonempty legacy variable above. + if mint.headers[0].Get("X-Codefly-Internal-Token") != "broker-perimeter-token" { + t.Error("broker perimeter header did not use the declared secret") + } for _, header := range []string{ - "x-codefly-internal-token", "x-codefly-solution-registration", "x-codefly-module-registration", removedClaimShape(t, "header"), @@ -652,7 +643,7 @@ func (s stubCredentialSource) Credential(context.Context) (workcontext.Credentia func mintClientFor(t *testing.T, mint *hostMint, tokenFile string) CredentialSource { t.Helper() client, err := workcontext.NewMintClient(workcontext.MintOptions{ - URL: mint.URL + credentialMintPath, + URL: mint.URL + "/platform/_credential", // The audience is named as a pinned value now, not passed as a string // beside the pin: a string alongside meant the drift check guarded a // value the mint did not use. @@ -1066,7 +1057,7 @@ func TestEverySourceIsHeldToTheFrozenAuthority(t *testing.T) { // The control: with the authority as this process froze it, a renewal is // handed over. - if _, err := server.credential.Credential(ctx); err != nil { + if _, err := server.nextCredential(ctx); err != nil { t.Fatalf("a renewal was refused while the authority had not drifted: %v", err) } @@ -1077,7 +1068,7 @@ func TestEverySourceIsHeldToTheFrozenAuthority(t *testing.T) { if err := codefly.LoadEnvironmentVariables(); err != nil { t.Fatal(err) } - _, err := server.credential.Credential(ctx) + _, err := server.nextCredential(ctx) if err == nil { t.Fatal("a renewal proceeded after an authority-bearing value drifted from the one this process froze: that renewal is the one moment the drift would be laundered into a differently-sealed credential") } @@ -1418,7 +1409,7 @@ func TestAWithdrawnAuthorityIsCaughtBeforeTheMint(t *testing.T) { if err := codefly.LoadEnvironmentVariables(); err != nil { t.Fatal(err) } - if _, err := server.credential.Credential(ctx); err == nil { + if _, err := server.nextCredential(ctx); err == nil { t.Fatal("an ask proceeded under a drifted authority") } // And the source was never reached, so nothing was minted under it. @@ -1702,7 +1693,7 @@ func TestDriftDuringAnAskIsCaughtAfterIt(t *testing.T) { server.authority = authorityFor(t, ctx) server.credential = heldToTheFrozenAuthority(source, server.authority) - _, err := server.credential.Credential(ctx) + _, err := server.nextCredential(ctx) <-drifted if err == nil { t.Fatal("a credential obtained while an authority-bearing value drifted mid-ask was handed over: the recheck before the ask cannot see a drift that lands after it, so the one after the ask is what catches this — and gating that on the token changing means a first credential, an unchanged one, and a mid-ask drift all go unchecked") @@ -2086,7 +2077,7 @@ func expiredCredential(t *testing.T, mint *hostMint, tokenFile string) workconte } mint.ttl = 10 * time.Minute client, err := workcontext.NewMintClient(workcontext.MintOptions{ - URL: mint.URL + credentialMintPath, + URL: mint.URL + "/platform/_credential", Audience: workcontext.AuthorityValue{Name: AuthorityGroup, Key: AuthorityAudienceKey}, Authority: fixedAuthority{audience: testAudience}, ProjectedToken: workcontext.ProjectedTokenFile(tokenFile), @@ -2385,7 +2376,13 @@ func TestTheMintHopIsHeldToTheSameThingsTheGatewayHopIs(t *testing.T) { // test that re-asked would pass against a client that never re-checked // anything. Refresh is the call that re-mints, so it is the call that reaches // the readers under test. -func freshMint(t *testing.T, source CredentialSource, held workcontext.Credential) (workcontext.Credential, error) { +func freshMint(t *testing.T, source any, held executionCredential) (executionCredential, error) { + if broker, ok := source.(*brokerCredentialSource); ok { + broker.mu.Lock() + broker.renewAt = time.Time{} + broker.mu.Unlock() + return broker.Credential(context.Background()) + } t.Helper() // These checks inspect the SDK transport directly, below error redaction. if redacted, ok := source.(*redactedMintSource); ok { @@ -2395,7 +2392,7 @@ func freshMint(t *testing.T, source CredentialSource, held workcontext.Credentia if !ok { t.Fatalf("the source under test is %T, not the SDK's mint client, so this would not exercise the transport at all", source) } - return client.Refresh(context.Background(), held) + return client.Refresh(context.Background(), held.(workcontext.Credential)) } // mintClientWith is a mint client against the fake endpoint, with one or more @@ -2403,7 +2400,7 @@ func freshMint(t *testing.T, source CredentialSource, held workcontext.Credentia func mintClientWith(t *testing.T, mint *hostMint, tokenFile string, with ...func(*workcontext.MintOptions)) CredentialSource { t.Helper() options := workcontext.MintOptions{ - URL: mint.URL + credentialMintPath, + URL: mint.URL + "/platform/_credential", Audience: workcontext.AuthorityValue{Name: AuthorityGroup, Key: AuthorityAudienceKey}, Authority: fixedAuthority{audience: testAudience}, ProjectedToken: workcontext.ProjectedTokenFile(tokenFile), @@ -2529,14 +2526,15 @@ func TestTheMintRefusesToPresentARotatedIdentity(t *testing.T) { server := New(Manifest{ID: testSolutionID}) server.cfg = config{ - identityCertFile: certFile, - identityKeyFile: keyFile, - trustBundleFile: bundleFile, - allowedCallersFile: identitiesFile(t, testGatewayPrincipal), - mintPeersFile: identitiesFile(t, testGatewayPrincipal), - gatewayPeersFile: identitiesFile(t, testGatewayPrincipal), - mintURL: mint.URL + credentialMintPath, - projectedTokenPath: tokenFile, + identityCertFile: certFile, + identityKeyFile: keyFile, + trustBundleFile: bundleFile, + allowedCallersFile: identitiesFile(t, testGatewayPrincipal), + mintPeersFile: identitiesFile(t, testGatewayPrincipal), + gatewayPeersFile: identitiesFile(t, testGatewayPrincipal), + brokerInternalToken: "broker-perimeter-token", + mintURL: mint.URL + credentialMintPath, + projectedTokenPath: tokenFile, } server.principal = testPrincipal // The frozen authority, read through the SDK the way a boot reads it. @@ -2571,44 +2569,27 @@ func TestTheMintRefusesToPresentARotatedIdentity(t *testing.T) { } } - // THE FINDING: a valid, same-CA certificate for another workload. + // An ordinary rotation remains usable by this same client. + rotate(testPrincipal) + if _, err := freshMint(t, source, held); err != nil { + t.Fatalf("ordinary identity rotation refused: %v", err) + } + // A valid same-CA certificate for another workload is refused before HTTP. rotate(rivalPrincipal) before := mint.count() - if _, err := freshMint(t, source, held); err == nil { - t.Fatal("the mint was dialled presenting a certificate issued for another workload: the host holds this process to the identity its presence document approved, and this hop is the one that carries the projected service-account token") - } else if !strings.Contains(err.Error(), "another workload") && !strings.Contains(err.Error(), "issued for") { - t.Errorf("the refusal %q does not say the certificate names another workload", err) + if _, err := freshMint(t, source, held); !terminalCredentialFailure(err) { + t.Fatalf("wrong workload identity was not terminal: %v", err) } if got := mint.count(); got != before { - t.Errorf("the issuer served %d more requests while this workload was presenting another identity: the refusal has to come before the projected service-account token is on the wire, not after", got-before) + t.Fatalf("wrong identity sent %d requests", got-before) } - - // And a rotation that is ordinary — a fresh certificate for the same - // principal — still works, so the check is about the identity and not - // about the pair having changed. + // The new retry allowlist makes this permanent TLS/configuration failure + // terminal, even if the files are repaired. Only a restarted process retries. rotate(testPrincipal) - // Through the SDK's hold-off, not around it. - // - // The refusal above is a failed ask, and the mint client backs off after - // one — "held off until …" — so the next attempt is refused by the backoff - // rather than by anything about the certificate. That is correct behaviour - // and it is also the reason this step waits: asserting the recovery - // against a brand-new client would prove that a fresh client works, which - // is not what is in question. - var recovered error - deadline := time.Now().Add(30 * time.Second) - for { - if _, recovered = freshMint(t, source, held); recovered == nil { - break - } - if !strings.Contains(recovered.Error(), "held off until") || time.Now().After(deadline) { - t.Fatalf("an ordinary rotation to a fresh certificate for this workload was refused, which would make every projected-pair renewal fatal: %v", recovered) - } - time.Sleep(250 * time.Millisecond) - } - if got := mint.count(); got <= before { - t.Errorf("the issuer served %d requests after a valid rotation, want more than %d", got, before) + if _, err := freshMint(t, source, held); !terminalCredentialFailure(err) || mint.count() != before { + t.Fatalf("terminal identity failure was retried: %v", err) } + } // TestEachSealedFieldIsComparedOnItsOwn isolates the three comparisons that diff --git a/environment_boundary_test.go b/environment_boundary_test.go index d4ab3e0..e69dd3d 100644 --- a/environment_boundary_test.go +++ b/environment_boundary_test.go @@ -4,7 +4,6 @@ import ( "go/ast" "go/parser" "go/token" - "go/types" "os" "path/filepath" "sort" @@ -213,19 +212,6 @@ var readerNames = map[string]map[string]bool{ "syscall": {"Getenv": true, "Environ": true}, } -// readerPackageOf is the import path a reader package goes by, since SSA names -// a package by path and the syntax names it by identifier. -func readerPackageOf(path string) string { return path } - -// sdkReaderNames are the SDK's own accessors, by method name. They are matched -// by name because they are methods on the SDK's query value rather than -// package functions, and the older gate's round-four finding was precisely a -// closure reading configuration through one of them. -var sdkReaderNames = map[string]bool{ - "WorkspaceConfiguration": true, "WorkspaceSecret": true, - "Environment": true, "Endpoint": true, "Secret": true, -} - // keylessReaders read the whole environment rather than one named variable, so // there is no key to resolve and nothing to constrain: the read itself is the // finding when it happens after boot. @@ -405,15 +391,6 @@ func isPostBootRoot(fn *ssa.Function) bool { return ast.IsExported(fn.Name()) } -// staticCall reports whether an edge is a call that names its callee, rather -// than one CHA resolved from a function type. -func staticCall(edge *callgraph.Edge, callee *ssa.Function) bool { - if edge.Site == nil { - return false - } - return edge.Site.Common().StaticCallee() == callee -} - // addressTaken is every function of the package whose value is used as a // value rather than called: stored in a variable or a struct, handed to a // registration call, closed over. SSA makes this visible — a function @@ -467,29 +444,6 @@ func isBootResolution(fn *ssa.Function) bool { fn.Pkg.Pkg.Path() == "github.com/codefly-dev/solution-runtime-go" } -// environmentReader reports whether an SSA function is one of the readers, by -// the SAME definition the key gate uses: package and name for the os and -// syscall readers, method name for the SDK's accessors, and this package's own -// env helper. -func environmentReader(fn *ssa.Function) (string, bool) { - if fn.Pkg != nil { - path := fn.Pkg.Pkg.Path() - if readerNames[path][fn.Name()] { - return path + "." + fn.Name(), true - } - if path == "github.com/codefly-dev/solution-runtime-go" && fn.Name() == "env" { - return "env", true - } - } - // The SDK's own accessors, matched by the package they belong to as well - // as the name: a bare name match reported grpc's resolver `Endpoint` as a - // configuration read, which it is not. - if fn.Pkg != nil && sdkReaderNames[fn.Name()] && strings.HasPrefix(fn.Pkg.Pkg.Path(), "github.com/codefly-dev/sdk-go") { - return "the SDK's " + fn.Name(), true - } - return "", false -} - // unparen removes every layer of parentheses, so `(env)(...)` is a call to env // rather than a call to something this gate has never heard of. func unparen(expr ast.Expr) ast.Expr { @@ -502,23 +456,6 @@ func unparen(expr ast.Expr) ast.Expr { } } -// isFuncTyped reports whether a type is a function type. -func isFuncTyped(t types.Type) bool { - if t == nil { - return false - } - _, ok := types.Unalias(t).Underlying().(*types.Signature) - return ok -} - -// pkgOf is the import path of the package a function belongs to. -func pkgOf(fn *types.Func) string { - if fn.Pkg() == nil { - return "" - } - return fn.Pkg().Path() -} - // And the read has to happen at boot, not merely be written there. // // Static reachability is not the claim AGENTS.md makes, and the difference was diff --git a/facade_claim_test.go b/facade_claim_test.go index de68a18..2b5e13a 100644 --- a/facade_claim_test.go +++ b/facade_claim_test.go @@ -5,7 +5,6 @@ import ( "go/constant" "go/token" "go/types" - "os" "path/filepath" "sort" "strconv" @@ -40,35 +39,28 @@ import ( // implement. There is no list of expression forms to finish here: the compiler // has already folded every constant, conversions and concatenation alike, so // the value is taken from it. That fixture is a compiled negative case below. -// The removed vocabulary is archived in CHANGELOG.md. Keeping the gate's -// inputs there lets configuration and source trees stay free of those names -// without weakening the compiled regression fixtures. +// Pin canonical wire spellings independently of documentation and fixture +// generation. Archive edits must never redefine the boundary. func removedClaimShape(t *testing.T, kind string) string { t.Helper() - raw, err := os.ReadFile("CHANGELOG.md") - if err != nil { - t.Fatal(err) + switch kind { + case "header": + return "X-Codefly-Module-Secret" + case "carrier": + return "CODEFLY__MODULE_REGISTRATION_SECRETS" } - for _, line := range strings.Split(string(raw), "\n") { - parts := strings.Split(line, "|") - if len(parts) == 4 && strings.TrimSpace(parts[1]) == kind { - value := strings.Trim(strings.TrimSpace(parts[2]), "`") - if value != "" { - return value - } - } - } - t.Fatalf("missing removed claim %q in the migration archive", kind) + t.Fatalf("unknown removed claim kind %q", kind) return "" } func claimShapes(t *testing.T) map[string]string { return map[string]string{ - "/modules/_register": "the gateway's module route-claim path", - "/modules/_registration-token": "the exchange that credentials a module route claim", - "X-Codefly-Module-Registration": "the header carrying a module route-claim credential", - removedClaimShape(t, "header"): "the removed module credential header", - removedClaimShape(t, "carrier"): "the removed module credential carrier", + "CODEFLY__SOLUTION_REGISTRATION_SECRET": "the removed solution credential carrier", + "/modules/_register": "the gateway's module route-claim path", + "/modules/_registration-token": "the exchange that credentials a module route claim", + "X-Codefly-Module-Registration": "the header carrying a module route-claim credential", + removedClaimShape(t, "header"): "the removed module credential header", + removedClaimShape(t, "carrier"): "the removed module credential carrier", } } @@ -226,6 +218,10 @@ func TestTheFacadeGatesCatchTheEscapingFixture(t *testing.T) { import "os" +func LegacySolutionCredential() string { + return os.Getenv("CODEFLY__SOLUTION_REGISTRATION_SECRET") +} + // A constant conversion splicing the carrier's name. func FacadeCredentialViaConversion() string { const key = __CARRIER_CONVERSION__ @@ -324,7 +320,9 @@ func joinedKey(parts []string) string { } ` - carrier, header := removedClaimShape(t, "carrier"), removedClaimShape(t, "header") + // Independent of claimShapes: changing the gate's vocabulary must fail + // against these canonical wire inputs, even if the archive changes too. + carrier, header := "CODEFLY__MODULE_REGISTRATION_SECRETS", "X-Codefly-Module-Secret" conversion := func(value string) string { return strconv.Quote(value[:1]) + " + string(rune(" + strconv.Itoa(int(value[1])) + ")) + " + strconv.Quote(value[2:]) } @@ -345,6 +343,7 @@ func joinedKey(parts []string) string { keys = strings.Join(unresolvedKeyFindings(t, pkg), "\n") } for _, want := range []string{ + "CODEFLY__SOLUTION_REGISTRATION_SECRET", carrier, "/modules/_register", header, diff --git a/identity_test.go b/identity_test.go index 7c1ce25..5ec9923 100644 --- a/identity_test.go +++ b/identity_test.go @@ -56,7 +56,7 @@ func TestABootWithoutAWorkloadIdentityIsRefusedByName(t *testing.T) { } { t.Run(tc.name, func(t *testing.T) { server := New(Manifest{ID: testSolutionID}) - server.cfg = config{ + server.cfg = config{brokerInternalToken: "broker-perimeter-token", port: "8080", gatewayURL: "https://gateway:42152", mintURL: "https://gateway:42152" + credentialMintPath, @@ -105,7 +105,7 @@ func TestACredentialBearingDestinationMustBeHTTPS(t *testing.T) { {"a plaintext mint", "https://gateway:42152", "http://gateway:42152" + credentialMintPath, "service-account token", CredentialMintURLEnvironmentVariable}, } { t.Run(tc.name, func(t *testing.T) { - cfg := config{port: "8080", gatewayURL: tc.gateway, mintURL: tc.mint, profile: localProfile} + cfg := config{brokerInternalToken: "broker-perimeter-token", port: "8080", gatewayURL: tc.gateway, mintURL: tc.mint, profile: localProfile} err := cfg.validate() if err == nil { t.Fatal("validate accepted a plaintext credential-bearing destination") @@ -125,7 +125,7 @@ func TestACredentialBearingDestinationMustBeHTTPS(t *testing.T) { // projected file paths as well would be requiring provisioning nothing reads. // What no source changes is the authority identity. func TestASuppliedSourceNeedsNoProvisioningItDoesNotRead(t *testing.T) { - base := config{ + base := config{brokerInternalToken: "broker-perimeter-token", port: "8080", gatewayURL: "https://gateway:42152", mintURL: "https://gateway:42152" + credentialMintPath, @@ -187,7 +187,7 @@ func TestAnIdentitySourceReachesTheSamePostureAsTheProjectedOne(t *testing.T) { } { t.Run(tc.name, func(t *testing.T) { server := New(Manifest{ID: testSolutionID}).Identity(staticIdentity{config: tc.config}) - server.cfg = config{port: listenOn(t, server)} + server.cfg = config{brokerInternalToken: "broker-perimeter-token", port: listenOn(t, server)} server.principal = testPrincipal ln, err := server.listen() if ln != nil { @@ -204,7 +204,7 @@ func TestAnIdentitySourceReachesTheSamePostureAsTheProjectedOne(t *testing.T) { t.Run("a source that reaches the posture is accepted", func(t *testing.T) { server := New(Manifest{ID: testSolutionID}).Identity(staticIdentity{config: usable}) - server.cfg = config{port: listenOn(t, server)} + server.cfg = config{brokerInternalToken: "broker-perimeter-token", port: listenOn(t, server)} server.principal = testPrincipal ln, err := server.listen() if err != nil { @@ -222,7 +222,7 @@ func TestAnIdentitySourceReachesTheSamePostureAsTheProjectedOne(t *testing.T) { func TestAListenerIsHeldToItsOwnFrozenPrincipal(t *testing.T) { certFile, keyFile, bundleFile, _, _ := workloadIdentity(t, "spiffe://codefly.test/ns/solutions/sa/another-workload") server := New(Manifest{ID: testSolutionID}) - server.cfg = config{port: listenOn(t, server), identityCertFile: certFile, identityKeyFile: keyFile, trustBundleFile: bundleFile} + server.cfg = config{brokerInternalToken: "broker-perimeter-token", port: listenOn(t, server), identityCertFile: certFile, identityKeyFile: keyFile, trustBundleFile: bundleFile} server.principal = testPrincipal ln, err := server.listen() if ln != nil { @@ -549,7 +549,7 @@ func TestASourcesPerConnectionCallbackCannotLowerThePosture(t *testing.T) { } { t.Run(tc.name, func(t *testing.T) { server := New(Manifest{ID: testSolutionID}).Identity(staticIdentity{config: downgrading(tc.change)}) - server.cfg = config{port: listenOn(t, server), allowedCallersFile: identitiesFile(t, testGatewayPrincipal)} + server.cfg = config{brokerInternalToken: "broker-perimeter-token", port: listenOn(t, server), allowedCallersFile: identitiesFile(t, testGatewayPrincipal)} server.principal = testPrincipal // The two failures listen() returns as one are separated here. // Refusing the *identity* at boot is an acceptable answer — what @@ -615,7 +615,7 @@ func TestASourcesPerConnectionCallbackCannotLowerThePosture(t *testing.T) { // and read as a success anyway: in TLS 1.3 the client finishes before // it learns whether the server accepted its certificate, so a // server-side refusal is invisible to tls.Dial. - server.cfg = config{port: listenOn(t, server), allowedCallersFile: identitiesFile(t, testGatewayPrincipal)} + server.cfg = config{brokerInternalToken: "broker-perimeter-token", port: listenOn(t, server), allowedCallersFile: identitiesFile(t, testGatewayPrincipal)} server.principal = testPrincipal ln, err := server.listen() if err != nil { @@ -943,7 +943,7 @@ func TestARotatedLeafIsStillHeldToTheFrozenPrincipal(t *testing.T) { c := newCell(t) certFile, keyFile, bundleFile, _, _ := c.workload(t, testPrincipal) server := New(Manifest{ID: testSolutionID}) - server.cfg = config{port: listenOn(t, server), identityCertFile: certFile, identityKeyFile: keyFile, + server.cfg = config{brokerInternalToken: "broker-perimeter-token", port: listenOn(t, server), identityCertFile: certFile, identityKeyFile: keyFile, trustBundleFile: bundleFile, allowedCallersFile: identitiesFile(t, testGatewayPrincipal), mintPeersFile: identitiesFile(t, testGatewayPrincipal), gatewayPeersFile: identitiesFile(t, testGatewayPrincipal)} server.principal = testPrincipal @@ -1201,7 +1201,7 @@ func TestASourcesConfigurationIsNotMutated(t *testing.T) { } server := New(Manifest{ID: testSolutionID}).Identity(staticIdentity{config: shared}) - server.cfg = config{allowedCallersFile: identitiesFile(t, testGatewayPrincipal)} + server.cfg = config{brokerInternalToken: "broker-perimeter-token", allowedCallersFile: identitiesFile(t, testGatewayPrincipal)} server.principal = testPrincipal config, err := server.serverIdentity() if err != nil { diff --git a/mcp_test.go b/mcp_test.go index cb37fa2..480bd1c 100644 --- a/mcp_test.go +++ b/mcp_test.go @@ -625,7 +625,7 @@ func TestTheMCPSeamRefusesWithoutAServeMCPDeclaration(t *testing.T) { // the host's issuer and must not start failing to boot because this runtime now // resolves one. func TestValidateRefusesAnUnresolvedIssuerOnlyWhenMCPIsServed(t *testing.T) { - cfg := config{mcp: true} + cfg := config{brokerInternalToken: "broker-perimeter-token", mcp: true} err := cfg.validateMCP() if err == nil { t.Fatal("an unresolved issuer was accepted while serving MCP") @@ -633,7 +633,7 @@ func TestValidateRefusesAnUnresolvedIssuerOnlyWhenMCPIsServed(t *testing.T) { if !strings.Contains(err.Error(), mcpConfigurationValue(MCPIssuerURLKey)) { t.Errorf("refusal %q does not name %s, the configuration that fixes it", err, mcpConfigurationValue(MCPIssuerURLKey)) } - if err := (config{}).validateMCP(); err != nil { + if err := (config{brokerInternalToken: "broker-perimeter-token"}).validateMCP(); err != nil { t.Errorf("a solution serving no MCP was refused: %v", err) } } @@ -649,7 +649,7 @@ func TestValidateRefusesAnUnresolvedIssuerOnlyWhenMCPIsServed(t *testing.T) { // derived identifier is unusable only because PUBLIC_URL is. func TestValidateRefusesAnUnpairablePublicURL(t *testing.T) { for _, public := range []string{"https://host.test/solutions/wiki", "https://host.test/mcp/", "/solutions/wiki/mcp"} { - cfg := config{mcp: true, mcpIssuerURL: testIssuer, mcpPublicURL: public, mcpPublicExplicit: true} + cfg := config{brokerInternalToken: "broker-perimeter-token", mcp: true, mcpIssuerURL: testIssuer, mcpPublicURL: public, mcpPublicExplicit: true} err := cfg.validateMCP() if err == nil { t.Errorf("%s %q was accepted, want a refusal", mcpConfigurationValue(MCPPublicURLKey), public) @@ -669,7 +669,7 @@ func TestValidateRefusesAnUnpairablePublicURL(t *testing.T) { t.Errorf("refusal %q for a derived identifier does not send the operator to PUBLIC_URL", err) } } - cfg := config{mcp: true, mcpIssuerURL: testIssuer, mcpPublicURL: "https://host.test/solutions/wiki" + MCPPath} + cfg := config{brokerInternalToken: "broker-perimeter-token", mcp: true, mcpIssuerURL: testIssuer, mcpPublicURL: "https://host.test/solutions/wiki" + MCPPath} if err := cfg.validateMCP(); err != nil { t.Errorf("a pairable public URL was refused: %v", err) } @@ -732,7 +732,7 @@ func TestValidateRefusesADeployedMCPThatNobodyAddressed(t *testing.T) { const inCluster = "http://frontend.saas.svc.cluster.local:3000" const public = "https://app.example.com/solutions/wiki" + MCPPath - deployed := config{mcp: true, runtimeContext: "kubernetes", mcpIssuerURL: inCluster} + deployed := config{brokerInternalToken: "broker-perimeter-token", mcp: true, runtimeContext: "kubernetes", mcpIssuerURL: inCluster} err := deployed.validateMCP() if err == nil { t.Fatal("a deployed MCP surface with no public URL was accepted") @@ -781,7 +781,7 @@ func TestValidateRefusesADeployedMCPThatNobodyAddressed(t *testing.T) { // A local run resolves a host it can actually reach and is unaffected: the // refusal is about a deployment, never about the environment's name. for _, local := range []string{"", "native", "nix", "container", "free"} { - cfg := config{mcp: true, runtimeContext: local, mcpIssuerURL: "http://localhost:3000"} + cfg := config{brokerInternalToken: "broker-perimeter-token", mcp: true, runtimeContext: local, mcpIssuerURL: "http://localhost:3000"} if err := cfg.validateMCP(); err != nil { t.Errorf("runtime context %q was refused: %v", local, err) } @@ -1183,7 +1183,7 @@ func TestAnMCPToolDoesNotRunWhileThisExecutionHoldsNoCredential(t *testing.T) { } server := New(Manifest{ID: mcpServerName, Title: "Wiki"}). ServeMCP(mcpServerName, "v1.2.3", register). - Credential(refusingSource{err: fmt.Errorf("%w: the issuer said so, at https://mint.internal.example/platform/_credential (dial tcp 10.4.1.9:8443: connection refused)", tc.err)}) + Credential(refusingSource{err: fmt.Errorf("%w: the issuer said so, at https://mint.internal.example/modules/_work-context (dial tcp 10.4.1.9:8443: connection refused)", tc.err)}) handler, err := server.mcpHandler(MCPEnvironment{IssuerURL: testIssuer}) if err != nil { @@ -1375,7 +1375,7 @@ func TestAToolsOwnFailureIsSanitizedOnTheWayOut(t *testing.T) { func TestTheMethodLayerGateRefusesOnItsOwn(t *testing.T) { server := New(Manifest{ID: mcpServerName, Title: "Wiki"}). ServeMCP(mcpServerName, "v1.2.3", readCollectionTool). - Credential(refusingSource{err: fmt.Errorf("%w: at https://mint.internal.example/platform/_credential", workcontext.ErrMintUnavailable)}) + Credential(refusingSource{err: fmt.Errorf("%w: at https://mint.internal.example/modules/_work-context", workcontext.ErrMintUnavailable)}) var reached atomic.Bool next := func(context.Context, string, mcp.Request) (mcp.Result, error) { @@ -1455,7 +1455,7 @@ func TestNoMCPRefusalEverPrintsASecret(t *testing.T) { for _, runtimeContext := range []string{"", "kubernetes"} { for _, value := range withSecret { for _, which := range []string{"issuer", "public"} { - cfg := config{mcp: true, runtimeContext: runtimeContext, + cfg := config{brokerInternalToken: "broker-perimeter-token", mcp: true, runtimeContext: runtimeContext, mcpIssuerURL: "https://login.example.com", mcpIssuerExplicit: true, mcpPublicURL: "https://app.example.com/wiki" + MCPPath, mcpPublicExplicit: true} switch which { @@ -1504,7 +1504,7 @@ func TestNoMCPRefusalEverPrintsASecret(t *testing.T) { func TestNoExportedGatewayMethodRelaysTheRuntimesError(t *testing.T) { gw := newWorkContextGateway(t, &workContextGateway{}) server := New(Manifest{ID: testSolutionID}).Credential(attestingSource(t)) - server.cfg = config{gatewayURL: gw.URL} + server.cfg = config{brokerInternalToken: "broker-perimeter-token", gatewayURL: gw.URL} header := http.Header{} header.Set("authorization", viewerBearer()) @@ -1696,9 +1696,9 @@ func TestAToolsOwnRefusalIsLeftAlone(t *testing.T) { // was looking in one of the two places the SDK sends. func TestAStructuredToolResultNeverCarriesTheRuntimesInternals(t *testing.T) { server := New(Manifest{ID: testSolutionID}) - server.cfg = config{ + server.cfg = config{brokerInternalToken: "broker-perimeter-token", gatewayURL: "https://gateway.internal.test", - mintURL: "https://mint.internal.test/platform/_credential", + mintURL: "https://mint.internal.test/modules/_work-context", } server.cfg.dialled = dialledAddresses(server.cfg.gatewayURL, server.cfg.mintURL) @@ -1707,7 +1707,7 @@ func TestAStructuredToolResultNeverCarriesTheRuntimesInternals(t *testing.T) { IsError: true, Content: []mcp.Content{&mcp.TextContent{Text: "the call did not complete"}}, StructuredContent: map[string]any{ - "error": `Post "https://mint.internal.test/platform/_credential": dial tcp 127.0.0.1:1 connect: connection refused`, + "error": `Post "https://mint.internal.test/modules/_work-context": dial tcp 127.0.0.1:1 connect: connection refused`, }, }, "tools/call") call, ok := result.(*mcp.CallToolResult) @@ -1767,7 +1767,9 @@ func TestAStructuredToolResultNeverCarriesTheRuntimesInternals(t *testing.T) { // through Marshal, so a search over the serialized bytes for // "https://host" misses — and the client decodes the address straight // back out. The decoded strings are what get checked now. - escaped := json.RawMessage(`{"error":"https:\/\/mint.internal.test\/platform\/_credential failed"}`) + // Escape the configured endpoint so a protocol migration cannot leave + // this probe testing a URL the runtime no longer dials. + escaped := json.RawMessage(`{"error":"` + strings.ReplaceAll(server.cfg.mintURL, "/", `\/`) + ` failed"}`) result := server.withoutRuntimeAddresses(&mcp.CallToolResult{ IsError: true, Content: []mcp.Content{&mcp.TextContent{Text: "the call did not complete"}}, @@ -1799,7 +1801,7 @@ func TestAStructuredToolResultNeverCarriesTheRuntimesInternals(t *testing.T) { result := server.withoutRuntimeAddresses(&mcp.CallToolResult{ IsError: true, Content: []mcp.Content{&mcp.TextContent{Text: "the call did not complete"}}, - StructuredContent: map[string]any{"https://mint.internal.test/platform/_credential": "refused"}, + StructuredContent: map[string]any{"https://mint.internal.test/modules/_work-context": "refused"}, }, "tools/call") if call, ok := result.(*mcp.CallToolResult); !ok || call.StructuredContent != nil { t.Errorf("an address in a structured KEY survived the filter: %+v", result) @@ -1813,7 +1815,7 @@ func TestAStructuredToolResultNeverCarriesTheRuntimesInternals(t *testing.T) { // mint's address afterwards passed the check and disclosed on the // second call. No concurrency needed — marshalling is not a pure // function of a value this runtime did not write. - shifty := &shiftingMarshaler{later: `{"error":"https://mint.internal.test/platform/_credential"}`} + shifty := &shiftingMarshaler{later: `{"error":"https://mint.internal.test/modules/_work-context"}`} result := server.withoutRuntimeAddresses(&mcp.CallToolResult{ IsError: true, Content: []mcp.Content{&mcp.TextContent{Text: "the call did not complete"}}, @@ -1843,7 +1845,7 @@ func TestAStructuredToolResultNeverCarriesTheRuntimesInternals(t *testing.T) { result := server.withoutRuntimeAddresses(&mcp.CallToolResult{ IsError: true, Content: []mcp.Content{text}, - StructuredContent: sabotagingMarshaler{text: text, to: "https://mint.internal.test/platform/_credential"}, + StructuredContent: sabotagingMarshaler{text: text, to: "https://mint.internal.test/modules/_work-context"}, }, "tools/call") published, err := json.Marshal(result) if err != nil { diff --git a/mint_errors.go b/mint_errors.go index 68cec71..39e6322 100644 --- a/mint_errors.go +++ b/mint_errors.go @@ -2,14 +2,20 @@ package solution import ( "context" + "crypto/tls" + "crypto/x509" "errors" + "fmt" + "net" + "net/http" + "syscall" "github.com/codefly-dev/sdk-go/workcontext" ) -// The pinned execution client predates the broker's redacted error contract. -// Preserve its classifications, but discard its diagnostic and entire cause -// chain before any runtime caller can return or log response or transport text. +// Consumer-supplied SDK sources keep their public sentinel contract, but their +// diagnostics and causes are discarded. The native broker preserves its own +// more precise status and transport categories below. type redactedMintSource struct{ client CredentialSource } func (s *redactedMintSource) Credential(ctx context.Context) (workcontext.Credential, error) { @@ -50,3 +56,120 @@ func (*projectedTokenError) Error() string { } func (*projectedTokenError) Unwrap() error { return workcontext.ErrMintUnavailable } + +// Only safe metadata survives the broker boundary. No body, header, URL, +// parser diagnostic, or underlying transport error is retained. +type brokerErrorClass uint8 + +const ( + brokerOtherStatus brokerErrorClass = iota + brokerRefused + brokerNotEstablished + brokerTransport + brokerInvalidAnswer + brokerProjection +) + +type brokerTransportCategory uint8 + +const ( + brokerTransportOther brokerTransportCategory = iota + brokerTransportTimeout + brokerTransportRefused + brokerTransportReset + brokerTransportTLS +) + +type brokerMintError struct { + status int + class brokerErrorClass + transport brokerTransportCategory +} + +func (e *brokerMintError) Error() string { + switch e.class { + case brokerRefused: + return fmt.Sprintf("module broker: HTTP %d: request refused", e.status) + case brokerNotEstablished: + return "module broker: HTTP 503: execution not established" + case brokerOtherStatus: + return fmt.Sprintf("module broker: HTTP %d: request failed", e.status) + case brokerInvalidAnswer: + return "module broker: invalid credential response" + case brokerProjection: + return "projected service-account token is missing, empty, unreadable or exceeds 65536 bytes" + } + switch e.transport { + case brokerTransportTimeout: + return "module broker: transport or response read timed out" + case brokerTransportRefused: + return "module broker: connection refused" + case brokerTransportReset: + return "module broker: connection reset" + case brokerTransportTLS: + return "module broker: TLS failure" + default: + return "module broker: transport or response read failed" + } +} +func (e *brokerMintError) retryable() bool { + return e.class == brokerNotEstablished || e.class == brokerTransport && + (e.transport == brokerTransportTimeout || e.transport == brokerTransportRefused || e.transport == brokerTransportReset) +} +func (e *brokerMintError) Unwrap() error { + if e.class == brokerRefused { + return workcontext.ErrMintRefused + } + if e.retryable() { + return workcontext.ErrMintUnavailable + } + return nil +} +func brokerStatusError(status int) *brokerMintError { + e := &brokerMintError{status: status, class: brokerOtherStatus} + switch status { + case http.StatusUnauthorized, http.StatusForbidden: + e.class = brokerRefused + case http.StatusServiceUnavailable: + e.class = brokerNotEstablished + } + return e +} +func classifyBrokerTransport(err error) *brokerMintError { + e := &brokerMintError{class: brokerTransport} + var verification *tls.CertificateVerificationError + var record tls.RecordHeaderError + var alert tls.AlertError + var unknownAuthority x509.UnknownAuthorityError + var hostname x509.HostnameError + var invalid x509.CertificateInvalidError + var roots x509.SystemRootsError + switch { + case errors.Is(err, context.Canceled): // cancellation is not an outage + case errors.Is(err, context.DeadlineExceeded) || brokerTimedOut(err): + e.transport = brokerTransportTimeout + case errors.Is(err, syscall.ECONNREFUSED): + e.transport = brokerTransportRefused + case errors.Is(err, syscall.ECONNRESET): + e.transport = brokerTransportReset + case errors.As(err, &verification), errors.As(err, &record), errors.As(err, &alert), errors.As(err, &unknownAuthority), errors.As(err, &hostname), errors.As(err, &invalid), errors.As(err, &roots): + e.transport = brokerTransportTLS + } + return e +} +func brokerTimedOut(err error) bool { + if timed, ok := err.(net.Error); ok && timed.Timeout() { + return true + } + switch e := err.(type) { + case interface{ Unwrap() error }: + return brokerTimedOut(e.Unwrap()) + case interface{ Unwrap() []error }: + for _, cause := range e.Unwrap() { + if brokerTimedOut(cause) { + return true + } + } + } + return false +} diff --git a/mint_migration_test.go b/mint_migration_test.go index 9d2576e..acb6ab9 100644 --- a/mint_migration_test.go +++ b/mint_migration_test.go @@ -56,16 +56,17 @@ func TestRuntimeProjectedTokenReaderReadsFreshAndFailsClosed(t *testing.T) { } // Exercise the production source, including its TLS readers and error boundary. -func productionMintSource(t *testing.T, mint *hostMint, tokenFile string) CredentialSource { +func productionMintSource(t *testing.T, mint *hostMint, tokenFile string) *brokerCredentialSource { t.Helper() cert, key, bundle, _, _, _ := bootIdentityIn(t, mint, testPrincipal) s := New(Manifest{ID: testSolutionID}) s.cfg = config{ identityCertFile: cert, identityKeyFile: key, trustBundleFile: bundle, - allowedCallersFile: identitiesFile(t, testGatewayPrincipal), - mintPeersFile: identitiesFile(t, testGatewayPrincipal), - gatewayPeersFile: identitiesFile(t, testGatewayPrincipal), - mintURL: mint.URL + credentialMintPath, projectedTokenPath: tokenFile, + allowedCallersFile: identitiesFile(t, testGatewayPrincipal), + mintPeersFile: identitiesFile(t, testGatewayPrincipal), + gatewayPeersFile: identitiesFile(t, testGatewayPrincipal), + brokerInternalToken: "broker-perimeter-token", + mintURL: mint.URL + credentialMintPath, projectedTokenPath: tokenFile, } s.principal = testPrincipal s.authority = authorityFor(t, context.Background()) @@ -94,7 +95,7 @@ func TestUnusableProjectedTokenNeverReachesTheMint(t *testing.T) { mint := newHostMint(t, &hostMint{}) source := productionMintSource(t, mint, path) credential, err := source.Credential(context.Background()) - if credential.Token() != "" || !errors.Is(err, workcontext.ErrMintUnavailable) { + if credential.Token() != "" || !terminalCredentialFailure(err) { t.Fatalf("unusable projection did not fail closed: %v", err) } if !strings.Contains(err.Error(), "projected service-account token") || strings.Contains(err.Error(), path) { @@ -161,7 +162,8 @@ func TestViewerMintDiscardsResponseAndTransportDiagnostics(t *testing.T) { for _, kind := range []string{"401", "403", "503", "redirect", "malformed-json", "transport"} { t.Run(kind, func(t *testing.T) { gw := newGateway("https://gateway.example", "viewer", "org", "session") - gw.workload = attestingSource(t).Credential + source := attestingSource(t) + gw.workload = func(ctx context.Context) (executionCredential, error) { return source.Credential(ctx) } calls := 0 gw.transport = mintRoundTrip(func(r *http.Request) (*http.Response, error) { calls++ diff --git a/projection_boundary_test.go b/projection_boundary_test.go index f67bbe3..4309b5c 100644 --- a/projection_boundary_test.go +++ b/projection_boundary_test.go @@ -30,9 +30,12 @@ func bootableEnvironment(t *testing.T) { t.Helper() t.Setenv("PORT", "8090") t.Setenv("GATEWAY_URL", "https://gateway.cell.test:42152") - t.Setenv(CredentialMintURLEnvironmentVariable, "https://mint.cell.test/mint") + t.Setenv(CredentialMintURLEnvironmentVariable, "https://mint.cell.test/modules/_work-context") t.Setenv(ContractProfileEnvironmentVariable, "test") t.Setenv("CODEFLY__RUNTIME_CONTEXT", "") + // The broker requires its declared workspace secret even when nothing is + // consumed. Resolve it through the SDK just as the real boot does. + authorityValues(t) } // bootConfig is the config a boot resolves in that environment, asserted to be @@ -278,6 +281,8 @@ func TestTheProjectionCannotBeHalfConstructed(t *testing.T) { refused.consumes = malformed if err := refused.validate(); err == nil { t.Error("a config holding a malformed projection passed validate()") + } else if !strings.Contains(err.Error(), manifest.APIConsumesEnvironmentVariable) { + t.Errorf("the malformed projection was refused for another reason: %v", err) } accepted := loadConfig(context.Background(), "solution-under-test", nil) accepted.consumes = empty diff --git a/solution.go b/solution.go index e1969b8..ee5650c 100644 --- a/solution.go +++ b/solution.go @@ -303,10 +303,11 @@ type Server struct { // source: one resolution serving the listener and the outbound client, so // the two cannot be different identities. identityConfig *tls.Config - // credential is where this execution's credential comes from. Nil until - // the boot opens the platform's mint endpoint; a test or a consumer on - // another issuer supplies its own with Credential. - credential CredentialSource + // credential is the optional consumer-supplied SDK source. The native + // broker is opened at boot when no consumer source was supplied. Both + // feed the same controller through nextCredential. + credential CredentialSource + brokerCredential *brokerCredentialSource // authority is the frozen set of authority-bearing values this process runs // under, read once at boot and rechecked before every credential renewal. authority *codefly.Authority @@ -323,7 +324,7 @@ type Server struct { credentialMu sync.Mutex credentialAsk *credentialFlight credentialAskErr error - credentialHeld workcontext.Credential + credentialHeld executionCredential credentialQuietUntil time.Time // credentialAskFailures counts consecutive failed acquisitions, which is // what the backoff grows on and a success resets. @@ -374,13 +375,10 @@ type config struct { // telling an MCP client an address this runtime dialled. dialled []string assetsDir string - // mintURL is the host endpoint that mints this execution's credential. The - // audience the projected token must be bound to is read through the SDK's - // authority reader and frozen (see openAuthority), not carried here: there - // used to be a mintAudience field beside this one, unread by anything, and - // its comment claimed both were "resolved, never defaulted" while mintURL - // does carry a default path. - mintURL string + // mintURL is the explicitly configured module broker endpoint. The prefix + // comes from Manifest.ID; authority audiences remain frozen separately. + mintURL string + brokerInternalToken string // projectedTokenPath is the file the platform projects this workload's // service-account token into. It is re-read before every renewal, because // the projection is rotated under the running process and a token read once @@ -698,21 +696,13 @@ func loadConfig(ctx context.Context, id string, environmentLoadErr error) config projectedTokenPath = "/var/run/secrets/codefly/token" } + internalToken, _ := codefly.For(ctx).WorkspaceSecret("module-authority", "INTERNAL_TOKEN") cfg := config{ - port: port, - gatewayURL: gatewayURL, - assetsDir: env("ASSETS_DIR", "../fe-remote/dist"), - // No default, because there is nothing to default to. Whether the mint - // stays brokered through the gateway is not settled — the host prefers - // the workload to post directly to the issuer with its projected token - // bound to the issuer's own audience, and neither endpoint exists yet. - // Deriving a path on the resolved gateway and labelling it a guess is - // not an answer: this is the address - // this runtime POSTs the projected service-account token to, the - // strongest statement it can make about which workload it is, and a - // guessed address for that is the one thing fail-closed does not - // permit. It is refused at boot until something resolves it, which is - // how this package already treats a token-exchange URL it cannot pair. + brokerInternalToken: strings.TrimSpace(internalToken), + port: port, + gatewayURL: gatewayURL, + assetsDir: env("ASSETS_DIR", "../fe-remote/dist"), + // The broker path is published, but its origin remains explicit. mintURL: strings.TrimSpace(env(CredentialMintURLEnvironmentVariable, "")), dialled: dialledAddresses(gatewayURL, strings.TrimSpace(env(CredentialMintURLEnvironmentVariable, ""))), projectedTokenPath: projectedTokenPath, @@ -900,12 +890,9 @@ const CredentialMintURLEnvironmentVariable = "CODEFLY__CREDENTIAL_MINT_URL" // machine's — the gap codefly-dev/core#687 named and core v0.7.1 closed. const ContractProfileEnvironmentVariable = "CODEFLY__CONTRACT_PROFILE" -// credentialMintPath is the path this package's own tests mount their fake mint -// on. It is NOT a default: nothing in loadConfig derives an address from it, -// because the host has not settled brokered-versus-direct and a guessed mint -// address is refused rather than assumed. It exists so the suite's fixtures -// agree with each other. -const credentialMintPath = "/platform/_credential" +// credentialMintPath is the host's published module-broker execution endpoint. +// The origin remains explicitly resolved, never inferred from a solution name. +const credentialMintPath = "/modules/_work-context" // workloadPath resolves one of the identity paths above: an explicit // environment override first, then the workspace configuration group the @@ -1010,6 +997,12 @@ func (c config) validate() error { required.name, required.override) } } + if u, _ := url.Parse(c.mintURL); u.Path != credentialMintPath || u.RawPath != "" { + return fmt.Errorf("%s must name the broker endpoint /modules/_work-context", CredentialMintURLEnvironmentVariable) + } + if c.brokerInternalToken == "" { + return fmt.Errorf("missing module-authority/INTERNAL_TOKEN: provision the broker perimeter credential as a workspace secret") + } if err := resources.ValidateConfigurationProfileName(c.profile); err != nil { return fmt.Errorf("unusable contract profile %q: %w — it is the Codefly environment's own name unless %s overrides it", c.profile, err, ContractProfileEnvironmentVariable) @@ -1658,7 +1651,7 @@ func (s *Server) handleHealth(w http.ResponseWriter, r *http.Request) { // and hands the same one back until its renewal point, so this is a // mutex and a comparison except at the renewal the credential's own expiry // dictates. Nothing here runs on a timer this runtime chose. - if s.credential != nil { + if s.credential != nil || s.brokerCredential != nil { // Through credentialWithin, so the deadline is real: passing a context // to the SDK's client does not bound it, because what blocks is a // mutex held across the mint. @@ -1708,7 +1701,7 @@ func (s *Server) actingForAViewer(ctx context.Context) error { if err := s.terminalErr.Load(); err != nil { return fmt.Errorf("%w: %w", ErrCredentialRefused, *err) } - if s.credential == nil { + if s.credential == nil && s.brokerCredential == nil { return nil } _, err := s.credentialWithin(ctx, routeCredentialTimeout) @@ -1736,7 +1729,7 @@ func (s *Server) actingForAViewer(ctx context.Context) error { // outright would turn a slow renewal — which happens while a perfectly valid // credential is still in hand, since renewal starts inside a lead before expiry // — into a 503 for every viewer. -func (s *Server) credentialWithin(ctx context.Context, d time.Duration) (workcontext.Credential, error) { +func (s *Server) credentialWithin(ctx context.Context, d time.Duration) (executionCredential, error) { // Held and unexpired is the answer whenever the issuer is not answering, // and this is the branch that was missing. // @@ -1791,7 +1784,7 @@ func (s *Server) credentialWithin(ctx context.Context, d time.Duration) (workcon // holdCredential records a credential the controller did not itself acquire — // the boot's — so the first caller after boot sees a process that holds one. -func (s *Server) holdCredential(credential workcontext.Credential) { +func (s *Server) holdCredential(credential executionCredential) { s.credentialMu.Lock() s.credentialHeld = credential s.credentialQuietUntil = time.Time{} @@ -1802,10 +1795,10 @@ func (s *Server) holdCredential(credential workcontext.Credential) { // expired. The zero credential never counts as held: a source that hands back // nothing has given this process no authority, which is the case a wrapper // keyed on the token's *value* could not see. -func (s *Server) heldCredential() (workcontext.Credential, bool) { +func (s *Server) heldCredential() (executionCredential, bool) { s.credentialMu.Lock() defer s.credentialMu.Unlock() - if s.credentialHeld.Token() == "" { + if s.credentialHeld == nil || s.credentialHeld.Token() == "" { return workcontext.Credential{}, false } if !time.Now().Before(s.credentialHeld.ExpiresAt()) { @@ -1819,11 +1812,11 @@ func (s *Server) heldCredential() (workcontext.Credential, bool) { // // All three under one lock, because a caller that reads them separately can act // on a mixture of two different moments. -func (s *Server) credentialState() (held workcontext.Credential, quiet bool, last error) { +func (s *Server) credentialState() (held executionCredential, quiet bool, last error) { s.credentialMu.Lock() defer s.credentialMu.Unlock() held = s.credentialHeld - if held.Token() == "" || !time.Now().Before(held.ExpiresAt()) { + if held == nil || held.Token() == "" || !time.Now().Before(held.ExpiresAt()) { held = workcontext.Credential{} } last = s.credentialAskErr @@ -1842,7 +1835,7 @@ func (s *Server) credentialState() (held workcontext.Credential, quiet bool, las // SDK's client holds a sync.Mutex across the network mint and sync.Mutex.Lock // ignores contexts (see the drafted sdk-go issue, which is the durable half of // this). -func (s *Server) askOnce(ctx context.Context, d time.Duration) (workcontext.Credential, error) { +func (s *Server) askOnce(ctx context.Context, d time.Duration) (executionCredential, error) { s.credentialMu.Lock() inflight := s.credentialAsk if inflight == nil { @@ -1888,7 +1881,7 @@ func (s *Server) askOnce(ctx context.Context, d time.Duration) (workcontext.Cred func (s *Server) resolveCredential(ctx context.Context, flight *credentialFlight) { asked, cancel := context.WithTimeout(context.WithoutCancel(ctx), platformRequestTimeout) defer cancel() - credential, err := s.credential.Credential(asked) + credential, err := s.nextCredential(asked) if err == nil { err = usableCredential(credential) } @@ -1930,7 +1923,7 @@ func (s *Server) resolveCredential(ctx context.Context, flight *credentialFlight // credentialFlight is one acquisition and the answer it produced. type credentialFlight struct { done chan struct{} - credential workcontext.Credential + credential executionCredential err error } @@ -1993,8 +1986,8 @@ func credentialAskQuiet(failures int) time.Duration { // // The authority wrapper could not see either: it keys on the token *changing*, // and a token that is always "" never changes. -func usableCredential(credential workcontext.Credential) error { - if credential.Token() == "" { +func usableCredential(credential executionCredential) error { + if credential == nil || credential.Token() == "" { return fmt.Errorf("%w: the credential source returned no credential at all (empty token), so this process holds nothing that attests which workload it is", workcontext.ErrMintRefused) } @@ -2058,7 +2051,7 @@ func usableCredential(credential workcontext.Credential) error { // // Nil when no source was supplied, so attestWorkloadReporting still refuses a // mint rather than treating an absent source as nothing to attest with. -func (s *Server) acquireCredential(ctx context.Context) (workcontext.Credential, error) { +func (s *Server) acquireCredential(ctx context.Context) (executionCredential, error) { return s.credentialWithin(ctx, routeCredentialTimeout) } @@ -2416,7 +2409,7 @@ func (s *Server) gatewayFor(header http.Header) *Gateway { gw.id, gw.report = s.manifest.ID, &s.attestation // Left nil when no source was supplied, so a mint is refused for want of // one rather than panicking inside the controller. - if s.credential != nil { + if s.credential != nil || s.brokerCredential != nil { gw.workload = s.acquireCredential } gw.terminal = s.credentialRefused diff --git a/solution_test.go b/solution_test.go index dca9ed0..e027add 100644 --- a/solution_test.go +++ b/solution_test.go @@ -755,7 +755,7 @@ func serveHandler(t *testing.T, gatewayURL string, handler Handler) *httptest.Se func serveHandlerWith(t *testing.T, gatewayURL string, source CredentialSource, handler Handler) *httptest.Server { t.Helper() s := New(Manifest{ID: "notes", Title: "Notes"}).Credential(source) - s.cfg = config{gatewayURL: gatewayURL} + s.cfg = config{brokerInternalToken: "broker-perimeter-token", gatewayURL: gatewayURL} server := httptest.NewServer(s.wrapRequest(func(r *http.Request, gw *Gateway) (any, error) { return handler(r.Context(), gw) })) @@ -1169,7 +1169,7 @@ func TestPlatformTrafficIsNeverProxiedAndPresentsThisWorkload(t *testing.T) { certFile, keyFile, bundleFile, _, _ := workloadIdentity(t, testPrincipal) server := New(Manifest{ID: testSolutionID}) - server.cfg = config{identityCertFile: certFile, identityKeyFile: keyFile, trustBundleFile: bundleFile, + server.cfg = config{brokerInternalToken: "broker-perimeter-token", identityCertFile: certFile, identityKeyFile: keyFile, trustBundleFile: bundleFile, mintPeersFile: identitiesFile(t, testGatewayPrincipal), gatewayPeersFile: identitiesFile(t, testGatewayPrincipal), // This test inspects the transport rather than dialling, so any @@ -1231,7 +1231,7 @@ func TestAHandlersGatewayDialsThroughTheBootsAuthenticatedTransport(t *testing.T host := newPlatformHost(t, c, testGatewayPrincipal) server := New(Manifest{ID: testSolutionID}) - server.cfg = config{identityCertFile: certFile, identityKeyFile: keyFile, trustBundleFile: bundleFile, + server.cfg = config{brokerInternalToken: "broker-perimeter-token", identityCertFile: certFile, identityKeyFile: keyFile, trustBundleFile: bundleFile, mintPeersFile: identitiesFile(t, testGatewayPrincipal), gatewayPeersFile: identitiesFile(t, testGatewayPrincipal), gatewayURL: host.URL} server.principal = testPrincipal @@ -1575,7 +1575,7 @@ func optionalIdentitiesFile(t *testing.T, identities string) string { // guessed address for that is what fail-closed forbids, and this package // already refuses rather than guesses when it cannot pair a token-exchange URL. func TestABootWithoutAResolvedMintURLIsRefused(t *testing.T) { - cfg := config{port: "8080", gatewayURL: "https://gateway:42152", profile: localProfile} + cfg := config{brokerInternalToken: "broker-perimeter-token", port: "8080", gatewayURL: "https://gateway:42152", profile: localProfile} err := cfg.validate() if err == nil { t.Fatal("the boot accepted a configuration with no mint URL, so this runtime would POST its projected token to an address nobody resolved") @@ -1623,7 +1623,7 @@ func TestNoProductionCodeDerivesTheMintAddress(t *testing.T) { func TestTwoAdmissionSourcesAreRefusedRatherThanRanked(t *testing.T) { base := func(t *testing.T) config { t.Helper() - return config{ + return config{brokerInternalToken: "broker-perimeter-token", port: "8080", gatewayURL: "https://gateway:42152", mintURL: "https://gateway:42152" + credentialMintPath, profile: localProfile, @@ -1792,12 +1792,12 @@ func TestAnAdmissionFileIsReadWholeOrRefused(t *testing.T) { // sent, so it is never used for anything except being logged. func TestACredentialBearingURLCarriesNothingButADestination(t *testing.T) { for _, tc := range []struct{ name, mint, names string }{ - {"userinfo", "https://ops:s3cr3t@mint.cell/platform/_credential", "userinfo"}, - {"a query", "https://mint.cell/platform/_credential?token=s3cr3t", "query"}, - {"a fragment", "https://mint.cell/platform/_credential#part", "fragment"}, + {"userinfo", "https://ops:s3cr3t@mint.cell/modules/_work-context", "userinfo"}, + {"a query", "https://mint.cell/modules/_work-context?token=s3cr3t", "query"}, + {"a fragment", "https://mint.cell/modules/_work-context#part", "fragment"}, } { t.Run(tc.name, func(t *testing.T) { - cfg := config{port: "8080", gatewayURL: "https://gateway:42152", mintURL: tc.mint, profile: localProfile} + cfg := config{brokerInternalToken: "broker-perimeter-token", port: "8080", gatewayURL: "https://gateway:42152", mintURL: tc.mint, profile: localProfile} err := cfg.validate() if err == nil { t.Fatalf("a credential mint URL carrying %s was accepted", tc.names) @@ -1825,17 +1825,17 @@ func TestACredentialBearingURLCarriesNothingButADestination(t *testing.T) { log.SetOutput(&captured) t.Cleanup(func() { log.SetOutput(os.Stderr) }) - cfg := config{port: "8080", gatewayURL: "https://gateway:42152", profile: localProfile, - mintURL: "https://ops:s3cr3t-password@mint.cell/platform/_credential"} + cfg := config{brokerInternalToken: "broker-perimeter-token", port: "8080", gatewayURL: "https://gateway:42152", profile: localProfile, + mintURL: "https://ops:s3cr3t-password@mint.cell/modules/_work-context"} // validate() refuses this URL, and the log line still must not carry // the secret: the refusal and the logging are independent, which is // the whole reason the log redacts regardless. _ = cfg.validate() - cfg.mintURL = "https://mint.cell/platform/_credential" + cfg.mintURL = "https://mint.cell/modules/_work-context" if err := cfg.validate(); err != nil { t.Fatalf("a conforming configuration was refused: %v", err) } - cfg.mintURL = "https://ops:s3cr3t-password@mint.cell/platform/_credential" + cfg.mintURL = "https://ops:s3cr3t-password@mint.cell/modules/_work-context" cfg.logResolved() if got := captured.String(); strings.Contains(got, "s3cr3t-password") { t.Errorf("the boot log carries the secret from the mint URL:\n%s", got) @@ -1862,7 +1862,7 @@ func TestTheEnvironmentLoadErrorReachesTheRefusals(t *testing.T) { }) t.Run("an unresolved destination says which it is", func(t *testing.T) { - cfg := config{port: "8080", profile: localProfile, environmentLoadErr: loadErr} + cfg := config{brokerInternalToken: "broker-perimeter-token", port: "8080", profile: localProfile, environmentLoadErr: loadErr} err := cfg.validate() if err == nil { t.Fatal("a configuration with no gateway was accepted") @@ -1873,7 +1873,7 @@ func TestTheEnvironmentLoadErrorReachesTheRefusals(t *testing.T) { }) t.Run("a missing path says which it is", func(t *testing.T) { - cfg := config{environmentLoadErr: loadErr} + cfg := config{brokerInternalToken: "broker-perimeter-token", environmentLoadErr: loadErr} err := cfg.requirePath("workload identity certificate", "", "OVERRIDE", "CERT_FILE") if err == nil { t.Fatal("an unresolved path was accepted") @@ -1889,9 +1889,9 @@ func TestTheEnvironmentLoadErrorReachesTheRefusals(t *testing.T) { // many of them are carried — came through the redaction intact. func TestAUsernameCredentialIsNotLogged(t *testing.T) { for _, raw := range []string{ - "https://s3cr3t-token@mint.cell/platform/_credential", - "https://s3cr3t-token:@mint.cell/platform/_credential", - "https://user:s3cr3t-token@mint.cell/platform/_credential", + "https://s3cr3t-token@mint.cell/modules/_work-context", + "https://s3cr3t-token:@mint.cell/modules/_work-context", + "https://user:s3cr3t-token@mint.cell/modules/_work-context", } { if got := redactedURL(raw); strings.Contains(got, "s3cr3t-token") { t.Errorf("redactedURL(%q) = %q, which still carries the secret: Redacted() only masks the password", raw, got) @@ -1958,7 +1958,7 @@ func TestAnAskCannotBeEditedAfterItsCeilingIsChecked(t *testing.T) { Contract(ModuleContract{Ceilings: map[string]map[string][]Scope{ localProfile: {"things": {{ResourceKind: "things", Actions: []string{"read"}}}}, }}) - server.cfg = config{gatewayURL: gw.URL, profile: localProfile, consumes: newProjection(consumesThings)} + server.cfg = config{brokerInternalToken: "broker-perimeter-token", gatewayURL: gw.URL, profile: localProfile, consumes: newProjection(consumesThings)} server.principal = testPrincipal contract, err := server.resolveContract() if err != nil {