diff --git a/cmd/run/launcher.go b/cmd/run/launcher.go index 982f3c51..40aacde8 100644 --- a/cmd/run/launcher.go +++ b/cmd/run/launcher.go @@ -2,9 +2,12 @@ package run import ( "context" + "encoding/json" + "errors" "fmt" "os" "path/filepath" + "reflect" "slices" "sort" "strings" @@ -57,14 +60,16 @@ import ( // "not established yet", which the workload retries. type launcher struct { - vouch *identity.Launcher - workspace *resources.Workspace - env *environments.Environment - flow launcherFlow - fixture string - stateDir string - identities map[string]gitops.NativeIdentity - signer solutionhost.Signer + vouch *identity.Launcher + workspace *resources.Workspace + env *environments.Environment + flow launcherFlow + fixture string + stateDir string + identities map[string]gitops.NativeIdentity + projections map[string][]string + verifiers map[string]bool + signer solutionhost.Signer hostModule, hostService string @@ -82,11 +87,14 @@ type launcher struct { // delivered is the last manifest delivered per module, a withdrawal // included. delivered map[string]*solutionhost.ModuleManifest - // pendingWithdrawals are modules whose withdrawal the host has not applied - // yet: retried on the next composition refresh, even with no gone roots. + // pendingWithdrawals are modules whose stopped workloads have not been + // reconciled with the host: a reduced manifest or full withdrawal is + // retried on the next composition refresh, even with no gone roots. pendingWithdrawals map[string]bool envelope *gitops.EnvelopeWriter stop chan struct{} + closeOnce sync.Once + refresh sync.WaitGroup // schedule bounds each workload's wait for the host; waiters are the // deliveries in flight, one per workload. @@ -103,6 +111,9 @@ var ( // attachLauncher wires the run as the composition's cluster when its // environment declares a host and the run is native. It is a no-op otherwise. func attachLauncher(ctx context.Context, flow *orchestration.Flow, workspace *resources.Workspace, env *environments.Environment, fixture string) error { + if err := env.ValidateNativeExecution(runtimeContext); err != nil { + return err + } if env == nil || env.Host == nil || runtimeContext == resources.RuntimeContextContainer { return nil } @@ -110,13 +121,15 @@ func attachLauncher(ctx context.Context, flow *orchestration.Flow, workspace *re launcherMu.Lock() defer launcherMu.Unlock() if launcherActive != nil { - launcherActive.close() + if err := launcherActive.close(); err != nil { + return err + } launcherActive = nil } if err := env.Host.Validate(); err != nil { return err } - identities, err := gitops.ResolveNativeIdentities(ctx, workspace, env) + plan, err := gitops.ResolveNativeIdentityPlan(ctx, workspace, env) if err != nil { return err } @@ -131,8 +144,9 @@ func attachLauncher(ctx context.Context, flow *orchestration.Flow, workspace *re } cluster := &launcher{ vouch: launcherIdentity, workspace: workspace, env: env, flow: flow, fixture: fixture, - stateDir: stateDir, - identities: identities, + stateDir: stateDir, + identities: plan.Identities, + projections: plan.Projections, verifiers: plan.Verifiers, signer: signer, running: map[string]bool{}, delivered: map[string]*solutionhost.ModuleManifest{}, @@ -147,37 +161,50 @@ func attachLauncher(ctx context.Context, flow *orchestration.Flow, workspace *re cluster.hostModule, cluster.hostService = parts[0], parts[1] // The run vouches for exactly what it composes: each identity workload, // each module's delivery carrier, and the applied observer it acts as. - cluster.declare(identities) + cluster.declare(plan.Identities) + cluster.declareProjections() if err := cluster.projectHost(); err != nil { return err } world := flow.World() world.RenderOwnedGroups = cluster.renderOwnedGroups world.RuntimeOverrides = cluster.runtimeOverrides + world.RuntimeStopped = func(service *resources.Service) error { + return cluster.stopWorkload(resources.WithUnique(service).Unique()) + } flow.WithRunningHook(func(service string, _ int) { go cluster.onRunning(context.WithoutCancel(ctx), service) }) - go cluster.refreshTokens() + cluster.refresh.Add(1) + go func() { defer cluster.refresh.Done(); cluster.refreshTokens() }() w.Info("native run: the run vouches for the composition's workloads", wool.Field("anchor", launcherIdentity.AnchorFingerprint()[:16])) launcherActive = cluster return nil } -func stopLauncher() { +func stopLauncher() error { launcherMu.Lock() defer launcherMu.Unlock() if launcherActive != nil { - launcherActive.close() + if err := launcherActive.close(); err != nil { + return err + } launcherActive = nil } + return nil } -func (cluster *launcher) close() { - select { - case <-cluster.stop: - default: - close(cluster.stop) +func (cluster *launcher) close() error { + cluster.closeOnce.Do(func() { + if cluster.stop != nil { + close(cluster.stop) + } + }) + cluster.refresh.Wait() + if cluster.vouch != nil { + return cluster.vouch.Close() } + return nil } // declare records identity workloads the run vouches for, with their delivery @@ -265,10 +292,18 @@ func (cluster *launcher) identityOf(service *resources.Service) (string, gitops. // component is projected, delivered and enveloped through the same path as // any workload reaching Running. It returns the uniques admitted, sorted. func (cluster *launcher) admit(ctx context.Context, workspace *resources.Workspace) ([]string, error) { - identities, err := gitops.ResolveNativeIdentities(ctx, workspace, cluster.env) + reloadedEnv, err := environments.Select(workspace, cluster.env.Name) if err != nil { return nil, err } + if !reflect.DeepEqual(reloadedEnv.NativeIdentity, cluster.env.NativeIdentity) { + return nil, fmt.Errorf("native-identity declaration changed; stop the run before changing projections or verifiers") + } + plan, err := gitops.ResolveNativeIdentityPlan(ctx, workspace, cluster.env) + if err != nil { + return nil, err + } + identities := plan.Identities cluster.composition.Lock() defer cluster.composition.Unlock() knownUniques := make([]string, 0, len(cluster.identities)) @@ -299,7 +334,9 @@ func (cluster *launcher) admit(ctx context.Context, workspace *resources.Workspa newcomers[unique] = identities[unique] } cluster.workspace = workspace + cluster.projections, cluster.verifiers = plan.Projections, plan.Verifiers cluster.declare(newcomers) + cluster.declareProjections() if cluster.vouch != nil && cluster.hostIdentityAnchor != "" && len(added) > 0 { // The host re-reads the anchor: a newcomer is vouched for from now on. if err := cluster.vouch.WriteAnchor(cluster.hostIdentityAnchor); err != nil { @@ -309,30 +346,59 @@ func (cluster *launcher) admit(ctx context.Context, workspace *resources.Workspa return added, nil } +func (cluster *launcher) declareProjections() { + if cluster.vouch == nil { + return + } + for unique := range cluster.projections { + module, service, _ := strings.Cut(unique, "/") + cluster.vouch.Declare(cluster.vouch.Workload(module, service)) + } +} + // runtimeOverrides is the process environment a cell gives a workload and a // laptop does not: the host finds its API server, a module-identity service // finds its projected token. Where a workload reaches the host's gateway is // not here: it is the workspace value the composition points at the gateway's // `workload` endpoint, which the run resolves like every other // `${endpoint:…}` reference, at the address this run allocated. -func (cluster *launcher) runtimeOverrides(service *resources.Service) map[string]string { +func (cluster *launcher) runtimeOverrides(service *resources.Service) (map[string]string, error) { unique, identity, ok := cluster.identityOf(service) out := map[string]string{} if ok { - // A module-identity service reads its token from the path a cell's - // render mounts unless told otherwise; this is the otherwise. credential, err := cluster.project(unique, identity) if err != nil { - wool.Get(context.Background()).In("launcher").Warn("cannot project the module's credential", wool.Field("service", unique), wool.ErrField(err)) - return out + return nil, fmt.Errorf("project %s host credential: %w", unique, err) } out[workloadTokenFileVariable] = credential.File } - if unique == resources.ServiceUnique(cluster.hostModule, cluster.hostService) { + cluster.composition.RLock() + audiences := append([]string(nil), cluster.projections[unique]...) + verifier := cluster.verifiers[unique] + cluster.composition.RUnlock() + if len(audiences) > 0 { + module, name, _ := strings.Cut(unique, "/") + paths := map[string]string{} + for _, audience := range audiences { + credential, err := cluster.vouch.ProjectForAudience(unique, cluster.vouch.Workload(module, name), audience) + if err != nil { + return nil, fmt.Errorf("project %s for %q: %w", unique, audience, err) + } + paths[audience] = credential.File + } + mapping, err := json.Marshal(paths) + if err != nil { + return nil, err + } + out[environments.NativeIdentityCredentials] = string(mapping) + } + if verifier || unique == resources.ServiceUnique(cluster.hostModule, cluster.hostService) { out[nativeidentity.AnchorEnvironment] = cluster.hostIdentityAnchor + } + if unique == resources.ServiceUnique(cluster.hostModule, cluster.hostService) { out["SOLUTION_HOST_LOCAL_TRUST_ANCHOR"] = cluster.hostAnchorDir } - return out + return out, nil } // renderOwnedGroups projects the authority a cell's render would project for @@ -485,7 +551,9 @@ func (cluster *launcher) refreshTokens() { case <-cluster.stop: return case <-ticker.C: - cluster.vouch.Refresh() + if err := cluster.vouch.Refresh(); err != nil { + wool.Get(context.Background()).In("launcher").Error("cannot refresh native credentials", wool.ErrField(err)) + } } } } @@ -501,6 +569,15 @@ func (cluster *launcher) onRunning(ctx context.Context, unique string) { return } cluster.mu.Lock() + // A queued Running hook can arrive after Stop. Only a still-projected + // workload may rejoin delivery; Stop removes its credentials before + // clearing running under this same lock. + if cluster.vouch != nil { + if _, projected := cluster.vouch.Projected(unique); !projected { + cluster.mu.Unlock() + return + } + } cluster.running[unique] = true cluster.mu.Unlock() waiter := cluster.waitFor(unique) @@ -564,40 +641,54 @@ func (cluster *launcher) newDeliverer(ctx context.Context) (*gitops.NativeDelive // workload is forgotten. The run's authority, the envelope and the host's // process are untouched. Returned are the modules withdrawn. func (cluster *launcher) withdraw(ctx context.Context, uniques []string) ([]string, error) { - modules := map[string]bool{} + var failures []error for _, unique := range uniques { if identity, ok := cluster.identity(unique); ok && cluster.delivers(identity) { - modules[identity.Module] = true + cluster.mu.Lock() + if cluster.pendingWithdrawals == nil { + cluster.pendingWithdrawals = map[string]bool{} + } + cluster.pendingWithdrawals[identity.Module] = true + cluster.mu.Unlock() + } + if err := cluster.forget(unique); err != nil { + failures = append(failures, err) + } + } + if cluster.vouch != nil { + if err := cluster.vouch.RetryCleanup(); err != nil { + failures = append(failures, err) } - cluster.forget(unique) } cluster.mu.Lock() + pending := make([]string, 0, len(cluster.pendingWithdrawals)) for module := range cluster.pendingWithdrawals { - modules[module] = true - } - cluster.mu.Unlock() - pending := make([]string, 0, len(modules)) - for module := range modules { pending = append(pending, module) } + cluster.mu.Unlock() sort.Strings(pending) var withdrawn []string for _, module := range pending { if len(cluster.startedIdentities(module)) > 0 { if err := cluster.deliver(ctx, module); err != nil { - return withdrawn, fmt.Errorf("%s: deliver the workloads that remain: %w", module, err) + failures = append(failures, fmt.Errorf("%s: deliver the workloads that remain: %w", module, err)) + continue } + cluster.mu.Lock() + delete(cluster.pendingWithdrawals, module) + cluster.mu.Unlock() continue } done, err := cluster.withdrawModule(ctx, module) if err != nil { - return withdrawn, fmt.Errorf("%s: withdraw: %w", module, err) + failures = append(failures, fmt.Errorf("%s: withdraw: %w", module, err)) + continue } if done { withdrawn = append(withdrawn, module) } } - return withdrawn, nil + return withdrawn, errors.Join(failures...) } // withdrawModule delivers a module's withdrawal and waits for the host to @@ -638,16 +729,33 @@ func (cluster *launcher) withdrawModule(ctx context.Context, module string) (boo // forget drops everything this run keeps for one workload: the identity the // composition gave it, its projection and whether it runs. Nothing of the // run's authority is touched. -func (cluster *launcher) forget(unique string) { +func (cluster *launcher) forget(unique string) error { cluster.composition.Lock() delete(cluster.identities, unique) cluster.composition.Unlock() + return cluster.stopWorkload(unique) +} + +// stopWorkload retires credentials without forgetting the declaration: a +// supported individual restart projects fresh credentials through Init/Start. +func (cluster *launcher) stopWorkload(unique string) error { + cluster.waitersMu.Lock() + if waiter := cluster.waiters[unique]; waiter != nil { + close(waiter.superseded) + delete(cluster.waiters, unique) + } + cluster.waitersMu.Unlock() + var err error if cluster.vouch != nil { - cluster.vouch.Forget(unique) + err = cluster.vouch.Forget(unique) } cluster.mu.Lock() delete(cluster.running, unique) cluster.mu.Unlock() + if err != nil { + return fmt.Errorf("remove stopped workload %s credentials: %w", unique, err) + } + return nil } // workloadTokenFileVariable names the projected token's path to a process diff --git a/cmd/run/launcher_test.go b/cmd/run/launcher_test.go index e0661a1b..09493b47 100644 --- a/cmd/run/launcher_test.go +++ b/cmd/run/launcher_test.go @@ -2,7 +2,7 @@ package run import ( "context" - "github.com/codefly-dev/core/workloadidentity" + "encoding/json" "os" "path/filepath" "strings" @@ -12,6 +12,7 @@ import ( "github.com/codefly-dev/cli/pkg/gitops" "github.com/codefly-dev/cli/pkg/run/identity" "github.com/codefly-dev/core/resources" + "github.com/codefly-dev/core/workloadidentity" "github.com/codefly-dev/core/workloadidentity/native" "github.com/stretchr/testify/require" ) @@ -238,3 +239,127 @@ func writeNonGoWorkloadDir(t *testing.T, name string) string { } return root } + +func TestNativeAudienceMappingsAndVerifierIsolation(t *testing.T) { + workspace, env := writeHostedNativeWorkspace(t) + env.NativeIdentity = &environments.EnvironmentNativeIdentity{ + Projections: []environments.NativeIdentityProjection{ + {Service: "model/model", Audiences: []string{env.Host.Audience, "other-audience"}}, + }, Verifiers: []string{"ledger/store"}, + } + plan, err := gitops.ResolveNativeIdentityPlan(t.Context(), workspace, env) + require.NoError(t, err) + vouch, err := identity.Start(env.Host.TrustDomain, env.Host.Audience, t.TempDir()) + require.NoError(t, err) + cluster := &launcher{vouch: vouch, workspace: workspace, env: env, identities: plan.Identities, + projections: plan.Projections, verifiers: plan.Verifiers, hostIdentityAnchor: filepath.Join(t.TempDir(), "anchor.json"), running: map[string]bool{}} + cluster.declare(plan.Identities) + cluster.declareProjections() + require.NoError(t, vouch.WriteAnchor(cluster.hostIdentityAnchor)) + t.Setenv(native.AnchorEnvironment, cluster.hostIdentityAnchor) + attester, err := native.LoadAttester() + require.NoError(t, err) + service, err := workspace.LoadService(t.Context(), &resources.ServiceWithModule{Module: "model", Name: "model"}) + require.NoError(t, err) + overrides, err := cluster.runtimeOverrides(service) + require.NoError(t, err) + require.NotContains(t, overrides, native.AnchorEnvironment) + var mapping map[string]string + require.NoError(t, json.Unmarshal([]byte(overrides[environments.NativeIdentityCredentials]), &mapping)) + require.Len(t, mapping, 2) + require.Equal(t, overrides[workloadTokenFileVariable], mapping[env.Host.Audience]) + for audience, path := range mapping { + token, err := os.ReadFile(path) + require.NoError(t, err) + _, err = attester.Attest(t.Context(), string(token), audience) + require.NoError(t, err) + } + verifier, err := workspace.LoadService(t.Context(), &resources.ServiceWithModule{Module: "ledger", Name: "store"}) + require.NoError(t, err) + overrides, err = cluster.runtimeOverrides(verifier) + require.NoError(t, err) + require.Equal(t, map[string]string{native.AnchorEnvironment: cluster.hostIdentityAnchor}, overrides) + unrelated := &resources.Service{Name: "other"} + unrelated.WithModule("example") + overrides, err = cluster.runtimeOverrides(unrelated) + require.NoError(t, err) + require.Empty(t, overrides) + require.NoError(t, cluster.forget("model/model")) + require.NoError(t, vouch.Refresh()) + for _, path := range mapping { + require.NoFileExists(t, path) + } +} + +func TestNativeProjectionDoesNotOptIntoModuleDelivery(t *testing.T) { + workspace, env := writeHostedNativeWorkspace(t) + env.NativeIdentity = &environments.EnvironmentNativeIdentity{ + Projections: []environments.NativeIdentityProjection{{Service: "ledger/store", Audiences: []string{"second"}}}, + } + plan, err := gitops.ResolveNativeIdentityPlan(t.Context(), workspace, env) + require.NoError(t, err) + require.NotContains(t, plan.Identities, "ledger/store") + require.Equal(t, []string{"second"}, plan.Projections["ledger/store"]) + vouch, err := identity.Start(env.Host.TrustDomain, env.Host.Audience, t.TempDir()) + require.NoError(t, err) + cluster := &launcher{vouch: vouch, env: env, identities: plan.Identities, projections: plan.Projections} + cluster.declare(plan.Identities) + cluster.declareProjections() + service, err := workspace.LoadService(t.Context(), &resources.ServiceWithModule{Module: "ledger", Name: "store"}) + require.NoError(t, err) + overrides, err := cluster.runtimeOverrides(service) + require.NoError(t, err) + require.Len(t, overrides, 1) + require.Contains(t, overrides, environments.NativeIdentityCredentials) + require.False(t, vouch.Anchor().Vouches(vouch.Workload("ledger", workloadidentity.DeliveryService))) + + env.NativeIdentity.Projections[0].Service = "absent/service" + _, err = gitops.ResolveNativeIdentityPlan(t.Context(), workspace, env) + require.ErrorContains(t, err, "unknown service") + env.NativeIdentity.Projections[0].Service = "ledger/store" + env.ManagedServices = map[string]environments.EnvironmentManagedService{"ledger/store": {Kind: "external", ExternalName: "example.test"}} + _, err = gitops.ResolveNativeIdentityPlan(t.Context(), workspace, env) + require.ErrorContains(t, err, "managed") +} + +func TestNativeProjectionFailureIsReturned(t *testing.T) { + workspace, env := writeHostedNativeWorkspace(t) + blocked := filepath.Join(t.TempDir(), "file") + require.NoError(t, os.WriteFile(blocked, []byte("not a directory"), 0o600)) + vouch, err := identity.Start(env.Host.TrustDomain, env.Host.Audience, blocked) + require.NoError(t, err) + identities, err := gitops.ResolveNativeIdentities(t.Context(), workspace, env) + require.NoError(t, err) + cluster := &launcher{vouch: vouch, env: env, identities: identities} + cluster.declare(identities) + service, err := workspace.LoadService(t.Context(), &resources.ServiceWithModule{Module: "model", Name: "model"}) + require.NoError(t, err) + overrides, err := cluster.runtimeOverrides(service) + require.ErrorContains(t, err, "project model/model") + require.Nil(t, overrides) +} + +func TestNativeDeclarationChangeRequiresNewRun(t *testing.T) { + workspace, env := writeHostedNativeWorkspace(t) + cluster := &launcher{workspace: workspace, env: env} + file := filepath.Join(workspace.Dir(), resources.WorkspaceConfigurationName) + contents, err := os.ReadFile(file) + require.NoError(t, err) + contents = append(contents, []byte(" native-identity:\n verifiers: [ledger/store]\n")...) + require.NoError(t, os.WriteFile(file, contents, 0o600)) + reloaded, err := resources.LoadWorkspaceFromDir(t.Context(), workspace.Dir()) + require.NoError(t, err) + _, err = cluster.admit(t.Context(), reloaded) + require.ErrorContains(t, err, "native-identity declaration changed") + require.Same(t, workspace, cluster.workspace) +} + +func TestNativeIdentityRefusesIncompatibleAttachBeforeAnyLaunch(t *testing.T) { + previous := runtimeContext + t.Cleanup(func() { runtimeContext = previous }) + env := &environments.Environment{Host: &environments.EnvironmentHost{}, NativeIdentity: &environments.EnvironmentNativeIdentity{Verifiers: []string{"example/api"}}} + for _, runtime := range []string{resources.RuntimeContextContainer, resources.RuntimeContextFree} { + runtimeContext = runtime + require.ErrorContains(t, attachLauncher(t.Context(), nil, nil, env, ""), "native-identity requires") + } +} diff --git a/cmd/run/native_identity_lifecycle_test.go b/cmd/run/native_identity_lifecycle_test.go new file mode 100644 index 00000000..e9947712 --- /dev/null +++ b/cmd/run/native_identity_lifecycle_test.go @@ -0,0 +1,201 @@ +package run + +import ( + "context" + "crypto/ed25519" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "sync" + "testing" + + "github.com/codefly-dev/cli/pkg/gitops" + "github.com/codefly-dev/cli/pkg/run/identity" + "github.com/codefly-dev/core/solutionhost" + "github.com/stretchr/testify/require" +) + +type withdrawalFlow struct { + fakeLauncherFlow + address string +} + +func (f *withdrawalFlow) GetAddressForEndpoint(context.Context, string, string, string) (string, error) { + return f.address, nil +} + +func TestWithdrawalCleanupFailureStillDeliversAndRetriesWithoutStoppedRoots(t *testing.T) { + workspace, env := writeHostedNativeWorkspace(t) + env.Host.Gateway = "model/model" + issuer, err := identity.Start(env.Host.TrustDomain, env.Host.Audience, t.TempDir()) + require.NoError(t, err) + signer, err := solutionhost.NewInMemorySigner("dev-0123456789abcdef", ed25519.NewKeyFromSeed(make([]byte, 32))) + require.NoError(t, err) + var mu sync.Mutex + posted := map[string]*solutionhost.ModuleManifest{} + refuse := true + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + mu.Lock() + defer mu.Unlock() + if r.Method == http.MethodPost { + body, err := io.ReadAll(r.Body) + if err != nil { + t.Error(err) + w.WriteHeader(500) + return + } + signed, err := solutionhost.ParseSigned(body) + if err != nil { + t.Error(err) + w.WriteHeader(400) + return + } + manifest, err := solutionhost.ManifestFromVerified(signed.Document) + if err != nil { + t.Error(err) + w.WriteHeader(400) + return + } + posted[manifest.Binding] = manifest + if refuse { + w.WriteHeader(503) + return + } + _ = json.NewEncoder(w).Encode(map[string]any{"disposition": "accepted", "generation": manifest.Generation}) + return + } + manifest := posted[filepath.Base(r.URL.Path)] + if manifest == nil { + w.WriteHeader(404) + return + } + digest, err := manifest.Digest() + if err != nil { + t.Error(err) + w.WriteHeader(500) + return + } + _ = json.NewEncoder(w).Encode(map[string]any{"binding": manifest.Binding, "generation": manifest.Generation, "digest": digest, "release": manifest.Release.Identity(), "removed": manifest.Removed}) + })) + defer server.Close() + cluster := &launcher{vouch: issuer, workspace: workspace, env: env, stateDir: t.TempDir(), signer: signer, + flow: &withdrawalFlow{address: server.URL}, hostModule: "host", identities: map[string]gitops.NativeIdentity{}, + running: map[string]bool{}, delivered: map[string]*solutionhost.ModuleManifest{}, pendingWithdrawals: map[string]bool{}} + paths := map[string]string{} + for _, module := range []string{"first", "second"} { + unique := module + "/api" + cluster.identities[unique] = gitops.NativeIdentity{Module: module, Service: "api"} + cluster.running[unique] = true + cluster.delivered[module] = &solutionhost.ModuleManifest{Schema: solutionhost.SchemaModuleManifestV1, Binding: "acme.local." + module, Alias: module, Kind: solutionhost.KindModule, Generation: 1, Host: solutionhost.HostTarget{Coordinate: env.Host.Coordinate, Domain: env.Host.Domain}, Release: solutionhost.Release{Publisher: "acme", Name: module, Version: "1.0.0"}, Workloads: []solutionhost.Workload{{Service: "api", Container: "api"}}} + issuer.DeclareModule(module, "api") + for _, audience := range []string{env.Host.Audience, "second-audience"} { + credential, err := issuer.ProjectForAudience(unique, issuer.Workload(module, "api"), audience) + require.NoError(t, err) + paths[unique+audience] = credential.File + } + } + issuer.Declare(issuer.Observer()) + bad := paths["first/api"+env.Host.Audience] + require.NoError(t, os.Remove(bad)) + require.NoError(t, os.Mkdir(bad, 0o700)) + blocker := filepath.Join(bad, "busy") + require.NoError(t, os.WriteFile(blocker, []byte("fixture"), 0o600)) + _, err = cluster.withdraw(t.Context(), []string{"first/api", "second/api"}) + require.ErrorContains(t, err, "remove native credential") + require.ErrorContains(t, err, "503") + require.Len(t, cluster.pendingWithdrawals, 2) + mu.Lock() + require.Len(t, posted, 2, "file cleanup failure must not skip either host reconciliation") + for _, manifest := range posted { + require.True(t, manifest.Removed) + } + refuse = false + mu.Unlock() + require.NoError(t, issuer.Refresh()) + for _, path := range paths { + if path != bad { + require.NoFileExists(t, path) + } + } + require.NoError(t, os.Remove(blocker)) + withdrawn, err := cluster.withdraw(t.Context(), nil) + require.NoError(t, err) + require.Equal(t, []string{"first", "second"}, withdrawn) + require.Empty(t, cluster.pendingWithdrawals) + for _, path := range paths { + _, err := os.Stat(path) + require.ErrorIs(t, err, os.ErrNotExist) + } + require.FileExists(t, filepath.Join(cluster.stateDir, "delivery", "generations.json"), "credential cleanup preserves delivery state") +} + +func TestStopLauncherJoinsRefreshRemovesAllAndReportsCleanupFailure(t *testing.T) { + issuer, err := identity.Start("example.test", "host", t.TempDir()) + require.NoError(t, err) + workload := issuer.Workload("example", "api") + issuer.Declare(workload) + var paths []string + for _, audience := range []string{"host", "second"} { + credential, err := issuer.ProjectForAudience("example/api", workload, audience) + require.NoError(t, err) + paths = append(paths, credential.File) + } + cluster := &launcher{vouch: issuer, stop: make(chan struct{})} + cluster.refresh.Add(1) + finished := make(chan struct{}) + go func() { defer cluster.refresh.Done(); defer close(finished); cluster.refreshTokens() }() + require.NoError(t, os.Remove(paths[0])) + require.NoError(t, os.Mkdir(paths[0], 0o700)) + blocker := filepath.Join(paths[0], "busy") + require.NoError(t, os.WriteFile(blocker, []byte("fixture"), 0o600)) + launcherMu.Lock() + previous := launcherActive + launcherActive = cluster + launcherMu.Unlock() + t.Cleanup(func() { launcherMu.Lock(); launcherActive = previous; launcherMu.Unlock() }) + require.ErrorContains(t, stopLauncher(), "remove native credential") + select { + case <-finished: + default: + t.Fatal("shutdown returned before refresh joined") + } + require.NoFileExists(t, paths[1]) + require.NoError(t, issuer.Refresh()) + require.NoFileExists(t, paths[1]) + _, err = issuer.Project("example/api", workload) + require.ErrorContains(t, err, "closed") + require.NoError(t, os.Remove(blocker)) + require.NoError(t, stopLauncher(), "failed shutdown cleanup remains retryable") + require.Nil(t, launcherActive) + _, err = os.Stat(paths[0]) + require.ErrorIs(t, err, os.ErrNotExist) +} + +func TestStoppedWorkloadIgnoresQueuedRunningHookAndKeepsDeclaration(t *testing.T) { + cluster, flow, unique := launcherWithFakeFlow(t, func(string) bool { return true }) + issuer, err := identity.Start("example.test", "host", t.TempDir()) + require.NoError(t, err) + cluster.vouch = issuer + workload := issuer.Workload("model", "api") + issuer.Declare(workload) + credential, err := issuer.Project(unique, workload) + require.NoError(t, err) + cluster.running[unique] = true + require.NoError(t, cluster.stopWorkload(unique)) + cluster.onRunning(t.Context(), unique) + require.Empty(t, cluster.running) + require.Zero(t, flow.attempts) + _, declared := cluster.identity(unique) + require.True(t, declared, "individual stop preserves the composition for restart") + require.NoError(t, issuer.Refresh()) + require.NoFileExists(t, credential.File) + _, err = issuer.Project(unique, workload) + require.NoError(t, err) + cluster.onRunning(t.Context(), unique) + require.True(t, cluster.running[unique]) + require.Equal(t, 1, flow.attempts, "restart can deliver through the original lifecycle") + require.NoError(t, cluster.close()) +} diff --git a/cmd/run/service.go b/cmd/run/service.go index d8526869..de4ce689 100644 --- a/cmd/run/service.go +++ b/cmd/run/service.go @@ -224,8 +224,7 @@ func runServiceCommand(cmd *cobra.Command, args []string) (returnErr error) { shutdownCtx, shutdownCancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second) err := stopService(shutdownCtx, run.flow) shutdownCancel() - stopLauncher() - return err + return errors.Join(err, stopLauncher()) } defer func() { returnErr = errors.Join(returnErr, stopFresh()) }() diff --git a/docs/commands.md b/docs/commands.md index c7022e14..b076b258 100644 --- a/docs/commands.md +++ b/docs/commands.md @@ -194,6 +194,10 @@ service is issued a workload credential when it is projected, its module's signed manifest is delivered to the host once it runs, and the module is offered for installation in the host's own registry. See [module-manifests.md](module-manifests.md#a-native-run-the-run-answers-as-the-cluster). +An environment can opt into additional native audience credentials and explicit +verifier recipients with `native-identity`; see the +[declaration and runtime carrier](module-manifests.md#additional-native-audience-projections-new-cli-contract). +This requires native/nix execution and does not opt a service into module delivery. ### `codefly run service [name...]` diff --git a/docs/configuration-contract.md b/docs/configuration-contract.md index a81d4bd4..e79febd4 100644 --- a/docs/configuration-contract.md +++ b/docs/configuration-contract.md @@ -263,3 +263,16 @@ carrier, and SDK-Go exposes raw and typed document accessors. This is separate from `service-config.values`, whose explicitly declared environment values remain strings. Render qualification does not establish live cloud identity, secret-store access or database connectivity. + +## Native workload identity + +`Environment.NativeIdentity` (`native-identity`) is an opt-in native execution +declaration, separate from cloud `service-identity` attachments. It declares +qualified service/audience projections and qualified verifier services. It is +preserved through environment load/save, not projected into Core runtime context +or new agent protocol fields. The existing runtime override map carries the generated +`CODEFLY_NATIVE_IDENTITY_CREDENTIALS` audience-to-file JSON and verifier anchor. +See [native audience projections](module-manifests.md#additional-native-audience-projections-new-cli-contract) +for its strict validation, recipient isolation, runtime restrictions and assurance +boundary. Importing this declaration does not make a deployment support it; +deployment rendering refuses it. diff --git a/docs/module-manifests.md b/docs/module-manifests.md index 0fb52d89..f45b8afc 100644 --- a/docs/module-manifests.md +++ b/docs/module-manifests.md @@ -1022,6 +1022,70 @@ until the host's read-only applied record holds that exact withdrawal. A withdrawal not yet applied stays pending and is redelivered unchanged (the host replays it) on the next signal. +### Additional native audience projections (new CLI contract) + +An environment may opt into additional workload credentials and explicit +verifiers. This extends the existing native launcher; it does not change Core's +credential format or grant application permission. For example, with the named +services already in the resolved composition: + +```yaml +native-identity: + projections: + - service: example/api + audiences: [host-audience, example-verifier] + verifiers: [example/verifier] +``` + +Place this block beside `host` in the environment declaration. The host supplies +the run's trust domain and default audience. References must be qualified +`/` names of actual, unmanaged composed services. Unknown keys, +unknown services, duplicate service records, duplicate audiences/verifiers, empty +audiences and audiences containing whitespace/control characters are refused. +An audience is otherwise opaque; no service address is inferred from it. + +Each projected workload receives `CODEFLY_NATIVE_IDENTITY_CREDENTIALS`, a JSON +object mapping only its requested audiences to absolute credential-file paths. +The launcher chooses collision-safe paths within its private run directory; +authors cannot choose output paths, subjects or environment-variable names. +The existing `module-identity` host credential remains implicit and reaches its +workload through `CODEFLY__WORKLOAD_TOKEN_FILE`. Requesting that same audience in +the new block reuses that projection. Other audiences have distinct files and +share the declared workload and instance. Each refresh uses the original +audience. Clients must re-read the file before a call. + +Only explicit `verifiers` and the existing delivery host receive +`CODEFLY_NATIVE_IDENTITY_ANCHOR`. A verifier-only service receives no credential +mapping. A projection-only service is declared to Core's native issuer without +becoming a `module-identity` workload or joining module-manifest delivery. The +verifier uses Core's native attester with its expected audience and must separately +authorize the exact caller. Endpoint reach still comes from existing dependency +and configuration-reference checks; a projection creates no route. + +This declaration requires `native` or `nix` execution. Container/free-choice +execution, execution inside a pod, deployment rendering, and a combined +`cluster` or `gitops` declaration refuse it. A per-service non-native runtime override also refuses delivery to a +recipient. An explicit projection or verifier selected with `--remote` refuses +before remote initialization or exposure. Conflicting launcher-owned runtime +overrides and `service-config` values are errors. Projection errors stop initialization/start. Refresh errors +are reported; existing credentials expire at Core's normal short lifetime. +Stopping a workload retires every audience from refresh, then removes its files. +Removal failures are reported and kept for retry; they never keep credentials +refreshing. Composition withdrawal still attempts host reconciliation for every +stopped workload and retries pending cleanup and reconciliation on the next +refresh even when no new roots stopped. Ordinary shutdown joins the refresh loop, +removes all projections and returns cleanup failures to its caller, preserving +unrelated delivery state. An individual restart uses the existing Init/Start +projection path to write fresh credentials. None of these file operations revoke +credential bytes already copied before the stop; those expire normally. Changing +the declaration during composition reload requires stopping and starting the run; it is never silently ignored. + +This is native laptop assurance: the run owns one ephemeral issuer, and files are +private to its operating-system user. It provides no isolation from another +process with that same user identity and makes no deployed authentication or +witness qualification claim. The declaration/carrier is a new CLI public contract; +consumers must adopt it explicitly before it can qualify their integration. + ## Dev deployments `codefly deploy dev` re-pins a service's image inside an already-rendered diff --git a/pkg/environments/environment.go b/pkg/environments/environment.go index d03cbc26..555db9a1 100644 --- a/pkg/environments/environment.go +++ b/pkg/environments/environment.go @@ -1343,6 +1343,8 @@ type Environment struct { // CLI-side; not serialized to proto. Host *EnvironmentHost `yaml:"host,omitempty"` + NativeIdentity *EnvironmentNativeIdentity `yaml:"native-identity,omitempty"` + // ManagedServices keys a replacement by the identity of the service it // replaces: "/", or a bare "" when exactly one // module in the workspace declares that name. Composed modules routinely ship diff --git a/pkg/environments/native_identity.go b/pkg/environments/native_identity.go new file mode 100644 index 00000000..7df9cc25 --- /dev/null +++ b/pkg/environments/native_identity.go @@ -0,0 +1,149 @@ +package environments + +import ( + "fmt" + "strings" + "unicode" + + "github.com/codefly-dev/core/resources" + "github.com/codefly-dev/core/workloadidentity/native" + "gopkg.in/yaml.v3" +) + +// NativeIdentityCredentials is the JSON audience-to-file mapping delivered only +// to a workload declaring native projections. Paths are owned by the launcher. +const NativeIdentityCredentials = "CODEFLY_NATIVE_IDENTITY_CREDENTIALS" + +// NativeIdentityCarrier identifies launcher-owned process values. +func NativeIdentityCarrier(key string) bool { + return key == NativeIdentityCredentials || key == native.AnchorEnvironment || key == "CODEFLY__WORKLOAD_TOKEN_FILE" || key == "SOLUTION_HOST_LOCAL_TRUST_ANCHOR" +} + +// EnvironmentNativeIdentity extends the native run's host identity projection; +// it neither grants application permission nor changes module-manifest delivery. +type EnvironmentNativeIdentity struct { + Projections []NativeIdentityProjection `yaml:"projections,omitempty"` + Verifiers []string `yaml:"verifiers,omitempty"` +} + +type NativeIdentityProjection struct { + Service string `yaml:"service"` + Audiences []string `yaml:"audiences"` +} + +func (n *EnvironmentNativeIdentity) UnmarshalYAML(node *yaml.Node) error { + if err := rejectUnknownKeys(node, "native-identity", "projections", "verifiers"); err != nil { + return err + } + type plain EnvironmentNativeIdentity + return node.Decode((*plain)(n)) +} + +func (p *NativeIdentityProjection) UnmarshalYAML(node *yaml.Node) error { + if err := rejectUnknownKeys(node, "native-identity projection", "service", "audiences"); err != nil { + return err + } + type plain NativeIdentityProjection + return node.Decode((*plain)(p)) +} + +func (env *Environment) validateNativeIdentity() error { + n := env.NativeIdentity + if n == nil { + return nil + } + if env.Host == nil { + return fmt.Errorf("native-identity requires an environment host declaring the run's trust domain and default audience") + } + if env.Cluster != nil || env.Gitops != nil { + return fmt.Errorf("native-identity is native-only and cannot be combined with cluster or gitops settings") + } + if len(n.Projections)+len(n.Verifiers) == 0 { + return fmt.Errorf("native-identity declares no projections or verifiers") + } + if env.ServiceConfig != nil { + for service, mapping := range env.ServiceConfig.Services { + for key := range mapping.Values { + if NativeIdentityCarrier(key) { + return fmt.Errorf("service-config %s: %s conflicts with native-identity launcher ownership", service, key) + } + } + } + } + qualified := func(ref string) error { + if !gatewayPattern.MatchString(ref) { + return fmt.Errorf("native-identity service %q must be /", ref) + } + return nil + } + seen := map[string]bool{} + for _, p := range n.Projections { + if err := qualified(p.Service); err != nil { + return err + } + if seen[p.Service] { + return fmt.Errorf("duplicate native-identity projection service %q", p.Service) + } + seen[p.Service] = true + if len(p.Audiences) == 0 { + return fmt.Errorf("native-identity projection %q declares no audiences", p.Service) + } + audiences := map[string]bool{} + for _, audience := range p.Audiences { + if audience == "" || strings.ContainsFunc(audience, func(r rune) bool { return unicode.IsSpace(r) || unicode.IsControl(r) }) { + return fmt.Errorf("native-identity audience %q must be nonempty without whitespace or control characters", audience) + } + if audiences[audience] { + return fmt.Errorf("duplicate native-identity audience %q for %s", audience, p.Service) + } + audiences[audience] = true + } + } + clear(seen) + for _, ref := range n.Verifiers { + if err := qualified(ref); err != nil { + return err + } + if seen[ref] { + return fmt.Errorf("duplicate native-identity verifier %q", ref) + } + seen[ref] = true + } + return nil +} + +// ValidateNativeExecution prevents security declarations from being silently +// discarded by container, free-choice, or deployment execution. +func (env *Environment) ValidateNativeExecution(runtime string) error { + if env != nil && env.NativeIdentity != nil { + if err := env.validateNativeIdentity(); err != nil { + return err + } + if native.InPod() { + return fmt.Errorf("native-identity: %w", native.ErrInPod) + } + if runtime != resources.RuntimeContextNative && runtime != resources.RuntimeContextNix { + return fmt.Errorf("native-identity requires native or nix execution; %q cannot project this run's credentials", runtime) + } + } + return nil +} + +// NativeIdentityRecipient reports explicit recipients, using the same qualified +// references validated against the resolved composition at launcher admission. +func (env *Environment) NativeIdentityRecipient(unique string) bool { + if env == nil || env.NativeIdentity == nil { + return false + } + for _, p := range env.NativeIdentity.Projections { + if p.Service == unique { + return true + } + } + for _, verifier := range env.NativeIdentity.Verifiers { + if verifier == unique { + return true + } + } + return false +} diff --git a/pkg/environments/native_identity_test.go b/pkg/environments/native_identity_test.go new file mode 100644 index 00000000..3f765af9 --- /dev/null +++ b/pkg/environments/native_identity_test.go @@ -0,0 +1,80 @@ +package environments + +import ( + "strings" + "testing" + + "github.com/codefly-dev/core/resources" + "github.com/stretchr/testify/require" + "gopkg.in/yaml.v3" +) + +const nativeEnvironment = `name: local +host: + coordinate: example/local/dev + component: host + domain: example + audience: host + trust_domain: example.test + delivery: host/api/rest +native-identity: + projections: + - service: example/api + audiences: [host, second/audience] + verifiers: [example/verifier] +` + +func TestNativeIdentityDeclarationRoundTrip(t *testing.T) { + var runtime resources.Environment + require.NoError(t, yaml.Unmarshal([]byte(nativeEnvironment), &runtime)) + env, err := FromRuntime(&runtime) + require.NoError(t, err) + require.Equal(t, []string{"host", "second/audience"}, env.NativeIdentity.Projections[0].Audiences) + resource, err := env.Resource() + require.NoError(t, err) + reloaded, err := FromRuntime(resource) + require.NoError(t, err) + require.Equal(t, env.NativeIdentity, reloaded.NativeIdentity) + require.Empty(t, env.Runtime().Extensions, "native declarations never extend the Core wire") + for _, runtime := range []string{resources.RuntimeContextNative, resources.RuntimeContextNix} { + require.NoError(t, env.ValidateNativeExecution(runtime)) + } + for _, runtime := range []string{resources.RuntimeContextContainer, resources.RuntimeContextFree, "deployment", ""} { + require.ErrorContains(t, env.ValidateNativeExecution(runtime), "native-identity requires") + } +} + +func TestNativeIdentityDeclarationRefusesInvalidSettings(t *testing.T) { + cases := map[string]string{ + "unknown block key": strings.Replace(nativeEnvironment, "verifiers:", "verifers:", 1), + "unknown projection key": strings.Replace(nativeEnvironment, "audiences:", "audience:", 1), + "duplicate block key": nativeEnvironment + " verifiers: [example/api]\n", + "duplicate service": strings.Replace(nativeEnvironment, " verifiers:", " - service: example/api\n audiences: [third]\n verifiers:", 1), + "duplicate audience": strings.Replace(nativeEnvironment, "host, second/audience", "host, host", 1), + "duplicate verifier": strings.Replace(nativeEnvironment, "[example/verifier]", "[example/verifier, example/verifier]", 1), + "bare service": strings.Replace(nativeEnvironment, "service: example/api", "service: api", 1), + "path traversal": strings.Replace(nativeEnvironment, "service: example/api", "service: ../api", 1), + "no audience": strings.Replace(nativeEnvironment, "[host, second/audience]", "[]", 1), + "blank audience": strings.Replace(nativeEnvironment, "[host, second/audience]", "[' ']", 1), + "control audience": strings.Replace(nativeEnvironment, "[host, second/audience]", `["second\nline"]`, 1), + "carrier conflict": nativeEnvironment + "service-config:\n services:\n api:\n values:\n CODEFLY_NATIVE_IDENTITY_CREDENTIALS: authored\n", + "cluster conflict": nativeEnvironment + "cluster: {kind: k3d, context: k3d-local}\n", + "gitops conflict": nativeEnvironment + "gitops: {repo-url: example}\n", + } + for name, input := range cases { + t.Run(name, func(t *testing.T) { + var runtime resources.Environment + err := yaml.Unmarshal([]byte(input), &runtime) + if err == nil { + _, err = FromRuntime(&runtime) + } + require.Error(t, err) + }) + } +} + +func TestNativeIdentityRefusesPodExecution(t *testing.T) { + t.Setenv("KUBERNETES_SERVICE_HOST", "cluster.example") + env := &Environment{Host: &EnvironmentHost{}, NativeIdentity: &EnvironmentNativeIdentity{Verifiers: []string{"example/api"}}} + require.ErrorContains(t, env.ValidateNativeExecution(resources.RuntimeContextNative), "pod proves its identity") +} diff --git a/pkg/environments/workspace.go b/pkg/environments/workspace.go index caf00122..dc44a6b5 100644 --- a/pkg/environments/workspace.go +++ b/pkg/environments/workspace.go @@ -100,6 +100,9 @@ func WorkspaceGitops(workspace *resources.Workspace) (*EnvironmentGitops, error) } func (env *Environment) Validate() error { + if err := env.validateNativeIdentity(); err != nil { + return err + } if err := env.admitCarried(); err != nil { return err } diff --git a/pkg/gitops/native_delivery.go b/pkg/gitops/native_delivery.go index 61ed1a12..9f7627c0 100644 --- a/pkg/gitops/native_delivery.go +++ b/pkg/gitops/native_delivery.go @@ -51,25 +51,71 @@ func (identity NativeIdentity) Workload(trustDomain string) workloadidentity.Wor // composition, each in its module's namespace. A native run has no cluster // to require. func ResolveNativeIdentities(ctx context.Context, workspace *resources.Workspace, env *environments.Environment) (map[string]NativeIdentity, error) { + plan, err := ResolveNativeIdentityPlan(ctx, workspace, env) + if err != nil { + return nil, err + } + return plan.Identities, nil +} + +// NativeIdentityPlan uses the resolved service inventory once, keeping explicit +// projections separate from workloads that participate in host delivery. +type NativeIdentityPlan struct { + Identities map[string]NativeIdentity + Projections map[string][]string + Verifiers map[string]bool +} + +func ResolveNativeIdentityPlan(ctx context.Context, workspace *resources.Workspace, env *environments.Environment) (*NativeIdentityPlan, error) { if env.Host == nil { return nil, fmt.Errorf("environment %s declares no host", env.Name) } + if err := env.Validate(); err != nil { + return nil, err + } refs, err := workspace.LoadServiceWithModules(ctx) if err != nil { return nil, err } - identities := map[string]NativeIdentity{} + plan := &NativeIdentityPlan{Identities: map[string]NativeIdentity{}, Projections: map[string][]string{}, Verifiers: map[string]bool{}} + services := map[string]*resources.Service{} for _, ref := range refs { - service, loadErr := workspace.LoadService(ctx, ref) - if loadErr != nil { - return nil, loadErr + service, err := workspace.LoadService(ctx, ref) + if err != nil { + return nil, err } - if !service.ModuleIdentity { - continue + unique := resources.ServiceUnique(ref.Module, ref.Name) + services[unique] = service + if service.ModuleIdentity { + plan.Identities[unique] = NativeIdentity{Module: ref.Module, Service: ref.Name} + } + } + if env.NativeIdentity == nil { + return plan, nil + } + validate := func(ref string) error { + if services[ref] == nil { + return fmt.Errorf("native-identity references unknown service %q", ref) + } + module, service, _ := strings.Cut(ref, "/") + if _, managed := env.ManagedService(module, service); managed { + return fmt.Errorf("native-identity service %q is managed and has no native workload", ref) + } + return nil + } + for _, projection := range env.NativeIdentity.Projections { + if err := validate(projection.Service); err != nil { + return nil, err + } + plan.Projections[projection.Service] = append([]string(nil), projection.Audiences...) + } + for _, verifier := range env.NativeIdentity.Verifiers { + if err := validate(verifier); err != nil { + return nil, err } - identities[resources.ServiceUnique(ref.Module, ref.Name)] = NativeIdentity{Module: ref.Module, Service: ref.Name} + plan.Verifiers[verifier] = true } - return identities, nil + return plan, nil } // NativeWorkload is one module a native run delivers a manifest for: the diff --git a/pkg/gitops/native_identity_test.go b/pkg/gitops/native_identity_test.go new file mode 100644 index 00000000..d3d0ea71 --- /dev/null +++ b/pkg/gitops/native_identity_test.go @@ -0,0 +1,20 @@ +package gitops + +import ( + "context" + "testing" + + "github.com/codefly-dev/cli/pkg/environments" + "github.com/stretchr/testify/require" +) + +func TestRenderRefusesNativeIdentityBeforeGenerating(t *testing.T) { + env := &environments.Environment{Host: &environments.EnvironmentHost{}, NativeIdentity: &environments.EnvironmentNativeIdentity{Verifiers: []string{"example/api"}}} + generated := false + _, err := RenderOwnedTree(t.Context(), &RenderOptions{Target: env, Destination: t.TempDir()}, func(context.Context, string) error { + generated = true + return nil + }) + require.ErrorContains(t, err, "native-identity requires") + require.False(t, generated) +} diff --git a/pkg/gitops/render.go b/pkg/gitops/render.go index 1a4150fd..61438539 100644 --- a/pkg/gitops/render.go +++ b/pkg/gitops/render.go @@ -62,6 +62,9 @@ type projectContract struct { } func RenderOwnedTree(ctx context.Context, opts *RenderOptions, generate func(context.Context, string) error) (RenderResult, error) { + if err := opts.Target.ValidateNativeExecution("deployment"); err != nil { + return RenderResult{}, err + } if opts.Destination == "" { return RenderResult{}, fmt.Errorf("render destination is required") } diff --git a/pkg/orchestration/extend.go b/pkg/orchestration/extend.go index 5999c7f5..048ee4f5 100644 --- a/pkg/orchestration/extend.go +++ b/pkg/orchestration/extend.go @@ -2,6 +2,7 @@ package orchestration import ( "context" + "errors" "fmt" "slices" "sync" @@ -165,6 +166,7 @@ func (flow *Flow) Withdraw(ctx context.Context, workspace *resources.Workspace, managers[manager.Unique()] = manager } var stopped []string + var cleanupFailures []error for i := len(before) - 1; i >= 0; i-- { unique := before[i] manager, managed := managers[unique] @@ -172,6 +174,12 @@ func (flow *Flow) Withdraw(ctx context.Context, workspace *resources.Workspace, continue } if _, stopErr := manager.RunnerDoStop(ctx); stopErr != nil { + var cleanupErr *runtimeStopCleanupError + if errors.As(stopErr, &cleanupErr) { + cleanupFailures = append(cleanupFailures, fmt.Errorf("%s cleanup: %w", unique, stopErr)) + stopped = append(stopped, unique) + continue + } *flow.workspace = originalWorkspace flow.graphWorkspace = originalGraphWorkspace flow.moduleSeeds = originalModuleSeeds @@ -184,7 +192,7 @@ func (flow *Flow) Withdraw(ctx context.Context, workspace *resources.Workspace, if policy, ok := flow.policy.(*RuntimeStartPolicy); ok { policy.SetDependencies(originalDependencies) } - return stopped, fmt.Errorf("%s stop: %w", unique, stopErr) + return stopped, errors.Join(append(cleanupFailures, fmt.Errorf("%s stop: %w", unique, stopErr))...) } stopped = append(stopped, unique) } @@ -195,7 +203,7 @@ func (flow *Flow) Withdraw(ctx context.Context, workspace *resources.Workspace, return service != nil && !kept[resources.WithUnique(service).Unique()] }) w.Info("narrowed the running composition", wool.Field("roots", withdrawn), wool.Field("stopped", stopped)) - return stopped, nil + return stopped, errors.Join(cleanupFailures...) } // admitRoots takes the reloaded composition in and records the roots it adds: diff --git a/pkg/orchestration/extend_test.go b/pkg/orchestration/extend_test.go index f9341f4b..8a5343c0 100644 --- a/pkg/orchestration/extend_test.go +++ b/pkg/orchestration/extend_test.go @@ -217,3 +217,78 @@ func TestWithdrawRetainsRootsWhenAStopRPCFails(t *testing.T) { require.NoError(t, err) require.NotContains(t, flow.Roots(), "wiki/backend") } + +// Exercise Flow -> Manager -> Runner -> Runtime.Stop -> RuntimeStopped, rather +// than a manager stub that cannot distinguish process stop from local cleanup. +func TestWithdrawReconcilesStoppedRunnersDespiteCleanupFailure(t *testing.T) { + for _, failedRPC := range []string{"", "wiki/backend", "documents/documents"} { + t.Run("failed RPC="+failedRPC, func(t *testing.T) { + flow, ctx := multiRootFlow(t, "lastlogin-go/backend", "wiki/backend") + required, err := flow.runClosure(ctx) + require.NoError(t, err) + flow.hub = &Hub{} + flow.playbook = &Playbook{} + flow.policy = &RuntimeStartPolicy{dependencies: flow.world.Dependencies, ExecutorManager: flow} + cleanupErr := errors.New("credential cleanup failed") + rpcErr := errors.New("runtime stop unavailable") + failCleanup := true + flow.world.RuntimeStopped = func(service *resources.Service) error { + if failCleanup && resources.WithUnique(service).Unique() == "wiki/backend" { + return cleanupErr + } + return nil + } + clients := map[string]*restartRuntimeClient{} + for _, unique := range append(uniques(required), "lastlogin-go/backend") { + ref, err := resources.ParseServiceWithOptionalModule(unique) + require.NoError(t, err) + service, err := flow.workspace.LoadService(ctx, ref) + require.NoError(t, err) + client := &restartRuntimeClient{} + if unique == failedRPC { + client.stopErr = rpcErr + } + runner := runnerWithRuntimeClient(client) + runner.instance.Identity, err = service.Identity() + require.NoError(t, err) + runner.instance.Service = service + runner.world = flow.world + flow.hub.add(&Manager{service: service, world: flow.world, Runner: runner}) + flow.services = append(flow.services, service) + clients[unique] = client + } + reloaded, err := resources.LoadWorkspaceFromDir(ctx, "testdata/multi-root") + require.NoError(t, err) + stopped, err := flow.Withdraw(ctx, reloaded, []string{"wiki/backend"}) + require.ErrorIs(t, err, cleanupErr, "cleanup failure stays visible to the caller") + require.Equal(t, 1, clients["wiki/backend"].stopCalls) + if failedRPC == "" { + require.Equal(t, []string{"wiki/backend", "documents/documents"}, stopped, "every stopped workload must reach host reconciliation") + require.Equal(t, 1, clients["documents/documents"].stopCalls, "cleanup failure must not skip subsequent stops") + require.NotContains(t, flow.Roots(), "wiki/backend") + for _, manager := range flow.hub.all() { + require.NotContains(t, stopped, manager.Unique(), "successfully stopped managers leave the flow") + } + } else { + require.ErrorIs(t, err, rpcErr) + require.Contains(t, flow.Roots(), "wiki/backend", "a real RPC failure retains the root for retry") + if failedRPC == "wiki/backend" { + require.Empty(t, stopped, "failed RPC plus cleanup error is still a failed process stop") + require.Zero(t, clients["documents/documents"].stopCalls) + } else { + require.Equal(t, []string{"wiki/backend"}, stopped, "earlier successful process stops still reach reconciliation") + } + failCleanup = false + clients[failedRPC].stopErr = nil + stopped, err = flow.Withdraw(ctx, reloaded, []string{"wiki/backend"}) + require.NoError(t, err) + require.Equal(t, []string{"wiki/backend", "documents/documents"}, stopped) + require.NotContains(t, flow.Roots(), "wiki/backend") + require.Equal(t, 2, clients[failedRPC].stopCalls, "failed process stop is retried") + } + require.Zero(t, clients["lastlogin-go/backend"].stopCalls) + require.Zero(t, clients["saas/frontend"].stopCalls) + require.Zero(t, clients["saas/auth-gateway"].stopCalls) + }) + } +} diff --git a/pkg/orchestration/flow.go b/pkg/orchestration/flow.go index ca03a3c7..2dff63b8 100644 --- a/pkg/orchestration/flow.go +++ b/pkg/orchestration/flow.go @@ -236,8 +236,12 @@ type World struct { // derives for the composition it stands in for (a cluster's in-cluster // address, the host's mint address). Read when a runner initializes, so // a value that depends on another service's address is resolved after - // every service has been loaded. Nil outside such a run. - RuntimeOverrides func(service *resources.Service) map[string]string + // every service has been loaded. Errors refuse Init/Start before the RPC. + // Nil outside such a run. + RuntimeOverrides func(service *resources.Service) (map[string]string, error) + + // RuntimeStopped retires launcher-owned process values after a stop attempt. + RuntimeStopped func(service *resources.Service) error Env *environments.Environment Mode Mode diff --git a/pkg/orchestration/invocation_test.go b/pkg/orchestration/invocation_test.go index 2c2b0b3a..08559552 100644 --- a/pkg/orchestration/invocation_test.go +++ b/pkg/orchestration/invocation_test.go @@ -47,7 +47,7 @@ func TestIsolatedInvocationPropagatesToRunners(t *testing.T) { id := flow.WithIsolatedInvocation() runner := &Runner{world: flow.world} - require.Equal(t, id, runner.runtimeOverrides()[resources.NamingScopePrefix]) + require.Equal(t, id, testRuntimeOverrides(t, runner)[resources.NamingScopePrefix]) } // The scope is a human label the caller owns; codefly must not silently diff --git a/pkg/orchestration/native_identity_lifecycle_test.go b/pkg/orchestration/native_identity_lifecycle_test.go new file mode 100644 index 00000000..7e3bd157 --- /dev/null +++ b/pkg/orchestration/native_identity_lifecycle_test.go @@ -0,0 +1,89 @@ +package orchestration + +import ( + "context" + "encoding/json" + "net" + "os" + "testing" + + "github.com/codefly-dev/cli/pkg/environments" + "github.com/codefly-dev/cli/pkg/run/identity" + agentservices "github.com/codefly-dev/core/agents/services" + runtimev0 "github.com/codefly-dev/core/generated/go/codefly/services/runtime/v0" + "github.com/codefly-dev/core/resources" + "github.com/codefly-dev/core/workloadidentity/native" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" +) + +type nativeLifecyclePeer struct { + realAgent + starts chan map[string]string +} + +func (peer *nativeLifecyclePeer) Start(_ context.Context, request *runtimev0.StartRequest) (*runtimev0.StartResponse, error) { + peer.starts <- request.Overrides + return &runtimev0.StartResponse{Status: &runtimev0.StartStatus{State: runtimev0.StartStatus_STARTED}}, nil +} + +func TestNativeIdentityIndividualStopRestartThroughRuntimePeer(t *testing.T) { + listener, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + peer := &nativeLifecyclePeer{realAgent: realAgent{mode: realAgentInputs}, starts: make(chan map[string]string, 2)} + server := grpc.NewServer() + runtimev0.RegisterRuntimeServer(server, peer) + go func() { _ = server.Serve(listener) }() + t.Cleanup(server.Stop) + conn, err := grpc.NewClient(listener.Addr().String(), grpc.WithTransportCredentials(insecure.NewCredentials())) + require.NoError(t, err) + t.Cleanup(func() { _ = conn.Close() }) + runner, world := gatewayRunner(t, &agentservices.RuntimeAgent{RuntimeClient: runtimev0.NewRuntimeClient(conn)}) + issuer, err := identity.Start("example.test", "host", t.TempDir()) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, issuer.Close()) }) + workload := issuer.Workload("web", "gateway") + issuer.Declare(workload) + attester, err := native.NewAttester(issuer.Anchor()) + require.NoError(t, err) + world.RuntimeOverrides = func(*resources.Service) (map[string]string, error) { + paths := map[string]string{} + for _, audience := range []string{"host", "second"} { + credential, err := issuer.ProjectForAudience("web/gateway", workload, audience) + if err != nil { + return nil, err + } + paths[audience] = credential.File + } + mapping, err := json.Marshal(paths) + return map[string]string{environments.NativeIdentityCredentials: string(mapping)}, err + } + world.RuntimeStopped = func(*resources.Service) error { return issuer.Forget("web/gateway") } + previous := map[string]string{} + for attempt := 0; attempt < 2; attempt++ { + _, err = runner.Init(t.Context()) + require.NoError(t, err) + _, err = runner.Start(t.Context()) + require.NoError(t, err) + mappings := map[string]string{} + require.NoError(t, json.Unmarshal([]byte((<-peer.starts)[environments.NativeIdentityCredentials]), &mappings)) + require.Len(t, mappings, 2) + for audience, path := range mappings { + token, err := os.ReadFile(path) + require.NoError(t, err) + _, err = attester.Attest(t.Context(), string(token), audience) + require.NoError(t, err) + if attempt > 0 { + require.NotEqual(t, previous[audience], string(token)) + } + previous[audience] = string(token) + } + _, err = runner.Stop(t.Context()) + require.NoError(t, err) + require.NoError(t, issuer.Refresh()) + for _, path := range mappings { + require.NoFileExists(t, path) + } + } +} diff --git a/pkg/orchestration/runner.go b/pkg/orchestration/runner.go index b63cf6d1..0830a231 100644 --- a/pkg/orchestration/runner.go +++ b/pkg/orchestration/runner.go @@ -261,6 +261,9 @@ func (runner *Runner) Init(ctx context.Context) (*OutputProperty, error) { w := wool.Get(ctx).In("Runner.Init", wool.ThisField(runner.instance)) if runner.remoteEnvironment != nil { + if err := runner.validateRemoteIdentity(); err != nil { + return nil, err + } return runner.InitRemote(ctx) } @@ -303,6 +306,10 @@ func (runner *Runner) Init(ctx context.Context) (*OutputProperty, error) { // altogether, so a fixture or an override delivered only at Start reaches // every dependency and never the service the suite is about. Start still // carries both, and the agent takes the first non-empty. + overrides, err := runner.runtimeOverridesFor(networkMappings) + if err != nil { + return nil, w.Wrapf(err, "cannot project runtime overrides") + } req := &runtimev0.InitRequest{ RuntimeContext: runtimeContext, ProposedNetworkMappings: networkMappings, @@ -312,7 +319,7 @@ func (runner *Runner) Init(ctx context.Context) (*OutputProperty, error) { WorkspaceConfigurations: workspaceConfigurations, DependenciesConfigurations: dependenciesConfigurations, Fixture: runner.fixture, - Overrides: runner.runtimeOverridesFor(networkMappings), + Overrides: overrides, } err = resources.Validate(req) if err != nil { @@ -907,6 +914,9 @@ func (runner *Runner) Start(ctx context.Context) (*OutputProperty, error) { w.Debug("start") if runner.remoteEnvironment != nil { + if err := runner.validateRemoteIdentity(); err != nil { + return nil, err + } return runner.StartRemote(ctx) } @@ -920,6 +930,10 @@ func (runner *Runner) Start(ctx context.Context) (*OutputProperty, error) { if err != nil { return nil, w.Wrapf(err, "cannot load service instance") } + overrides, err := runner.runtimeOverrides() + if err != nil { + return nil, w.Wrapf(err, "cannot project runtime overrides") + } if runner.outputEnv != "" { runtimeContext, runtimeErr := resources.NewRuntimeContext(runner.runtimeContext) if runtimeErr != nil { @@ -983,7 +997,7 @@ func (runner *Runner) Start(ctx context.Context) (*OutputProperty, error) { identity, runtimeContext, runner.fixture, - runner.runtimeOverrides(), + overrides, endpointMappings, ); appendErr != nil { return nil, w.Wrapf(appendErr, "cannot write runtime environment variables to file") @@ -993,7 +1007,7 @@ func (runner *Runner) Start(ctx context.Context) (*OutputProperty, error) { req := &runtimev0.StartRequest{ DependenciesNetworkMappings: dependenciesNetworkMappings, Fixture: runner.fixture, - Overrides: runner.runtimeOverrides(), + Overrides: overrides, } err = resources.Validate(req) if err != nil { @@ -1395,6 +1409,14 @@ func (runner *Runner) Stop(ctx context.Context) (*OutputProperty, error) { return runner.stop(ctx) } +// runtimeStopCleanupError reports a confirmed process stop whose local cleanup +// failed. Withdrawal must still reconcile that stopped workload with the host. +// It is never returned when the runtime Stop RPC itself failed. +type runtimeStopCleanupError struct{ cause error } + +func (err *runtimeStopCleanupError) Error() string { return err.cause.Error() } +func (err *runtimeStopCleanupError) Unwrap() error { return err.cause } + func (runner *Runner) stop(ctx context.Context) (*OutputProperty, error) { w := wool.Get(ctx).In("service.RunnerDoStop", wool.ThisField(runner.instance)) // Info-level so a user watching `codefly run service` shutdown @@ -1406,11 +1428,18 @@ func (runner *Runner) stop(ctx context.Context) (*OutputProperty, error) { // the runtime's own bounded cleanup and abandons its final response. stoppingContext, cancel := context.WithTimeout(ctx, defaultStopPhaseBudget) defer cancel() - _, err := runner.instance.Runtime.Stop(stoppingContext, &runtimev0.StopRequest{}) - if err != nil { - return nil, w.Wrapf(err, "cannot stop service instance: %s", runner.Unique()) + _, stopErr := runner.instance.Runtime.Stop(stoppingContext, &runtimev0.StopRequest{}) + var cleanupErr error + if runner.world != nil && runner.world.RuntimeStopped != nil { + cleanupErr = runner.world.RuntimeStopped(runner.instance.Service) + } + if stopErr != nil { + return nil, w.Wrapf(errors.Join(stopErr, cleanupErr), "cannot stop service instance: %s", runner.Unique()) } w.Info(fmt.Sprintf("stopped %s in %s", runner.Unique(), time.Since(start).Round(time.Millisecond))) + if cleanupErr != nil { + return &OutputProperty{}, &runtimeStopCleanupError{w.Wrapf(cleanupErr, "runtime stopped but cleanup failed for %s", runner.Unique())} + } return &OutputProperty{}, nil } @@ -1629,7 +1658,7 @@ func (runner *Runner) WithOverrides(overrides map[string]string) { // isolation available to already-published service agents; newer agents also // derive the same carrier from Environment, so both paths converge on the // identical value. -func (runner *Runner) runtimeOverrides() map[string]string { +func (runner *Runner) runtimeOverrides() (map[string]string, error) { return runner.runtimeOverridesFor(runner.networkMappings) } @@ -1656,7 +1685,7 @@ func (runner *Runner) selfNetworkAccess() *basev0.NetworkAccess { // override seam so it reaches agents built before core emits it themselves // (RuntimeWrapper.AddSelfEndpoints); the derivation is core's, so both paths // converge on the same value. -func (runner *Runner) runtimeOverridesFor(own []*basev0.NetworkMapping) map[string]string { +func (runner *Runner) runtimeOverridesFor(own []*basev0.NetworkMapping) (map[string]string, error) { self := SelfEndpointEnvironmentVariables(context.Background(), own, runner.selfNetworkAccess()) overrides := make(map[string]string, len(self)+len(runner.overrides)+1) for key, value := range self { @@ -1679,14 +1708,33 @@ func (runner *Runner) runtimeOverridesFor(own []*basev0.NetworkMapping) map[stri overrides[key] = value } if runner.world != nil && runner.world.RuntimeOverrides != nil { - for key, value := range runner.world.RuntimeOverrides(runner.instance.Service) { + owned, err := runner.world.RuntimeOverrides(runner.instance.Service) + if err != nil { + return nil, err + } + if runner.world.Env != nil && runner.world.Env.NativeIdentity != nil { + if len(owned) > 0 { + if err := runner.world.Env.ValidateNativeExecution(runner.runtimeContext); err != nil { + return nil, err + } + } + for key := range runner.overrides { + if environments.NativeIdentityCarrier(key) { + return nil, fmt.Errorf("runtime override %s conflicts with native-identity launcher ownership", key) + } + } + } + for key, value := range owned { + if authored, exists := runner.overrides[key]; exists && authored != value { + return nil, fmt.Errorf("runtime override %s conflicts with the native launcher projection", key) + } overrides[key] = value } } if runner.world != nil && runner.world.Env != nil && runner.world.Env.NamingScope != "" { overrides[resources.NamingScopePrefix] = runner.world.Env.NamingScope } - return overrides + return overrides, nil } // hostRoutes is the host gateway's route carriers for this run, derived once @@ -2024,3 +2072,12 @@ func (runner *Runner) initReadFailure(cfgCtx context.Context, w *wool.Wool, err } return w.Wrapf(err, "cannot get %s", what) } + +// Remote services cannot receive this run's local credential files or anchor. +// Refuse explicit recipients before remote initialization or network exposure. +func (runner *Runner) validateRemoteIdentity() error { + if runner.world != nil && runner.world.Env != nil && runner.world.Env.NativeIdentityRecipient(runner.Unique()) { + return fmt.Errorf("native-identity recipient %s cannot run with --remote", runner.Unique()) + } + return nil +} diff --git a/pkg/orchestration/runner_init_inputs_test.go b/pkg/orchestration/runner_init_inputs_test.go index bd26d118..95c88b75 100644 --- a/pkg/orchestration/runner_init_inputs_test.go +++ b/pkg/orchestration/runner_init_inputs_test.go @@ -2,8 +2,10 @@ package orchestration import ( "context" + "fmt" "testing" + "github.com/codefly-dev/cli/pkg/environments" "github.com/codefly-dev/core/agents/contract" agentv0 "github.com/codefly-dev/core/generated/go/codefly/services/agent/v0" runtimev0 "github.com/codefly-dev/core/generated/go/codefly/services/runtime/v0" @@ -107,3 +109,49 @@ func TestInitCarriesNoFixtureOrOverridesWhenTheInvocationSelectsNone(t *testing. require.Equal(t, "", observed["fixture"]) require.NotContains(t, observed, "CODEFLY__API_CONSUMES") } + +func TestNativeProjectionReachesInitAndRefusesFailure(t *testing.T) { + runner, world := gatewayRunner(t, startRealAgent(t, realAgentInputs)) + world.RuntimeOverrides = func(*resources.Service) (map[string]string, error) { + return nil, fmt.Errorf("credential projection failed") + } + _, err := runner.Init(t.Context()) + require.ErrorContains(t, err, "credential projection failed") + mapping := `{"second/audience":"/run/generated/credential"}` + world.RuntimeOverrides = func(*resources.Service) (map[string]string, error) { + return map[string]string{environments.NativeIdentityCredentials: mapping}, nil + } + _, err = runner.Init(t.Context()) + require.NoError(t, err) + require.Equal(t, mapping, exposedValues(t.Context(), t, world)[environments.NativeIdentityCredentials]) + world.RuntimeOverrides = func(*resources.Service) (map[string]string, error) { + return nil, fmt.Errorf("credential projection failed before Start") + } + _, err = runner.Start(t.Context()) + require.ErrorContains(t, err, "credential projection failed before Start") +} + +// A remote recipient must fail before either remote arm. The existing peer +// reports Init inputs, so an unrelated remote remains a positive control. +func TestNativeIdentityRemoteRecipientsRefuseBeforeRemoteLifecycle(t *testing.T) { + for _, verifier := range []bool{false, true} { + t.Run(fmt.Sprint("verifier=", verifier), func(t *testing.T) { + runner, world := gatewayRunner(t, startRealAgent(t, realAgentInputs)) + declaration := &environments.EnvironmentNativeIdentity{} + if verifier { + declaration.Verifiers = []string{"web/gateway"} + } else { + declaration.Projections = []environments.NativeIdentityProjection{{Service: "web/gateway", Audiences: []string{"second"}}} + } + world.Env.NativeIdentity = declaration + runner.WithRemote(&environments.Environment{Name: "remote"}) + _, err := runner.Init(t.Context()) + require.ErrorContains(t, err, "native-identity recipient web/gateway cannot run with --remote") + _, err = runner.Start(t.Context()) + require.ErrorContains(t, err, "native-identity recipient web/gateway cannot run with --remote") + world.Env.NativeIdentity = &environments.EnvironmentNativeIdentity{Verifiers: []string{"web/frontend"}} + _, err = runner.Init(t.Context()) + require.NoError(t, err, "an unrelated remote still uses the existing remote lifecycle") + }) + } +} diff --git a/pkg/orchestration/runner_runtime_identity_test.go b/pkg/orchestration/runner_runtime_identity_test.go index 0a9fe766..5f6643ba 100644 --- a/pkg/orchestration/runner_runtime_identity_test.go +++ b/pkg/orchestration/runner_runtime_identity_test.go @@ -1,10 +1,13 @@ package orchestration import ( + "fmt" "testing" "github.com/codefly-dev/cli/pkg/environments" + basev0 "github.com/codefly-dev/core/generated/go/codefly/base/v0" "github.com/codefly-dev/core/resources" + "github.com/codefly-dev/core/services" "github.com/stretchr/testify/require" ) @@ -18,7 +21,7 @@ func TestRunnerRuntimeOverridesCarryAuthoritativeNamingScope(t *testing.T) { overrides: input, } - got := runner.runtimeOverrides() + got := testRuntimeOverrides(t, runner) require.Equal(t, "dogfood", got["APPLICATION_MODE"]) require.Equal(t, "stable", got[resources.NamingScopePrefix]) @@ -32,5 +35,48 @@ func TestRunnerRuntimeOverridesCarryAuthoritativeNamingScope(t *testing.T) { func TestRunnerRuntimeOverridesOmitUnselectedNamingScope(t *testing.T) { runner := &Runner{world: &World{Env: &environments.Environment{}}} - require.Empty(t, runner.runtimeOverrides()) + require.Empty(t, testRuntimeOverrides(t, runner)) +} + +func testRuntimeOverrides(t *testing.T, runner *Runner) map[string]string { + t.Helper() + got, err := runner.runtimeOverrides() + require.NoError(t, err) + return got +} +func testRuntimeOverridesFor(t *testing.T, runner *Runner, own []*basev0.NetworkMapping) map[string]string { + t.Helper() + got, err := runner.runtimeOverridesFor(own) + require.NoError(t, err) + return got +} + +func TestRuntimeProjectionRefusalAndConflictingOverride(t *testing.T) { + service := &resources.Service{Name: "api"} + runner := &Runner{instance: &services.Instance{Service: service}, world: &World{ + RuntimeOverrides: func(*resources.Service) (map[string]string, error) { return nil, fmt.Errorf("projection unavailable") }, + }} + _, err := runner.runtimeOverrides() + require.ErrorContains(t, err, "projection unavailable") + runner.world.RuntimeOverrides = func(*resources.Service) (map[string]string, error) { + return map[string]string{environments.NativeIdentityCredentials: `{"second":"/generated"}`}, nil + } + runner.overrides = map[string]string{environments.NativeIdentityCredentials: `{"second":"/authored"}`} + _, err = runner.runtimeOverrides() + require.ErrorContains(t, err, "conflicts") +} + +func TestNativeIdentityRefusesPerServiceContainerSelection(t *testing.T) { + runner := &Runner{instance: &services.Instance{Service: &resources.Service{Name: "api"}}, runtimeContext: resources.RuntimeContextContainer, + world: &World{Env: &environments.Environment{Host: &environments.EnvironmentHost{}, NativeIdentity: &environments.EnvironmentNativeIdentity{Verifiers: []string{"example/api"}}}, + RuntimeOverrides: func(*resources.Service) (map[string]string, error) { + return map[string]string{"CODEFLY_NATIVE_IDENTITY_ANCHOR": "/run/anchor"}, nil + }, + }, + } + _, err := runner.runtimeOverrides() + require.ErrorContains(t, err, "native-identity requires") + runner.world.RuntimeOverrides = func(*resources.Service) (map[string]string, error) { return nil, nil } + _, err = runner.runtimeOverrides() + require.NoError(t, err, "an unrelated container service receives no native identity") } diff --git a/pkg/orchestration/runner_test.go b/pkg/orchestration/runner_test.go index 87d64f53..309fed8f 100644 --- a/pkg/orchestration/runner_test.go +++ b/pkg/orchestration/runner_test.go @@ -4,8 +4,10 @@ import ( "context" "errors" "fmt" + "github.com/codefly-dev/cli/pkg/run/identity" "net" "os" + "path/filepath" "testing" "time" @@ -294,3 +296,47 @@ func TestFollowKeepsMonitoringAfterFastRestart(t *testing.T) { t.Fatal("runner death was not reported after restart") } } + +func TestRunnerStopRetiresNativeCredentialsAndRestartProjectsFresh(t *testing.T) { + client := &restartRuntimeClient{} + runner := runnerWithRuntimeClient(client) + runner.instance.Service = &resources.Service{Name: "auth-sidecar"} + issuer, err := identity.Start("example.test", "host", t.TempDir()) + require.NoError(t, err) + workload := issuer.Workload("backend", "auth-sidecar") + issuer.Declare(workload) + runner.world = &World{ + RuntimeOverrides: func(*resources.Service) (map[string]string, error) { + credential, err := issuer.ProjectForAudience(runner.Unique(), workload, "second") + return map[string]string{"credential": credential.File}, err + }, + RuntimeStopped: func(*resources.Service) error { return issuer.Forget(runner.Unique()) }, + } + projected, err := runner.runtimeOverrides() + require.NoError(t, err) + before, err := os.ReadFile(projected["credential"]) + require.NoError(t, err) + _, err = runner.Stop(t.Context()) + require.NoError(t, err) + require.Equal(t, 1, client.stopCalls) + require.NoError(t, issuer.Refresh()) + require.NoFileExists(t, projected["credential"]) + // The same Init/Start projection seam is used after an individual stop; + // the issuer and declaration remain available throughout the run. + projected, err = runner.runtimeOverrides() + require.NoError(t, err) + after, err := os.ReadFile(projected["credential"]) + require.NoError(t, err) + require.NotEqual(t, before, after) + require.NoError(t, os.Remove(projected["credential"])) + require.NoError(t, os.Mkdir(projected["credential"], 0o700)) + blocker := filepath.Join(projected["credential"], "busy") + require.NoError(t, os.WriteFile(blocker, []byte("fixture"), 0o600)) + client.stopErr = errors.New("agent stop failed") + _, err = runner.Stop(t.Context()) + require.ErrorContains(t, err, "agent stop failed") + require.ErrorContains(t, err, "remove native credential") + require.NoError(t, issuer.Refresh(), "cleanup failure cannot keep refreshing") + require.NoError(t, os.Remove(blocker)) + require.NoError(t, issuer.Close()) +} diff --git a/pkg/orchestration/self_endpoint_test.go b/pkg/orchestration/self_endpoint_test.go index b8791de9..390bce9e 100644 --- a/pkg/orchestration/self_endpoint_test.go +++ b/pkg/orchestration/self_endpoint_test.go @@ -63,15 +63,15 @@ func TestRunnerOverridesCarryTheSelfEndpointAtInitAndStart(t *testing.T) { runtimeContext: resources.RuntimeContextNative, } - init := runner.runtimeOverridesFor(proposed) + init := testRuntimeOverridesFor(t, runner, proposed) require.Equal(t, "localhost:8080", init[wikiSelfKey]) require.Equal(t, "dogfood", init["APPLICATION_MODE"]) runner.networkMappings = accepted - require.Equal(t, "localhost:8081", runner.runtimeOverrides()[wikiSelfKey]) + require.Equal(t, "localhost:8081", testRuntimeOverrides(t, runner)[wikiSelfKey]) runner.runtimeContext = resources.RuntimeContextContainer - require.Equal(t, "host.docker.internal:8081", runner.runtimeOverrides()[wikiSelfKey]) + require.Equal(t, "host.docker.internal:8081", testRuntimeOverrides(t, runner)[wikiSelfKey]) } func TestRunnerOverridesLetAnExplicitSetWinOverTheSelfEndpoint(t *testing.T) { @@ -81,7 +81,7 @@ func TestRunnerOverridesLetAnExplicitSetWinOverTheSelfEndpoint(t *testing.T) { networkMappings: []*basev0.NetworkMapping{ownMapping("localhost:8080", "host.docker.internal:8080")}, runtimeContext: resources.RuntimeContextContainer, } - require.Equal(t, "https://wiki.example.com", runner.runtimeOverrides()[wikiSelfKey]) + require.Equal(t, "https://wiki.example.com", testRuntimeOverrides(t, runner)[wikiSelfKey]) } // TestNativeRunDeliversTheEnvironmentsTelemetryCarriers pins that a native run @@ -96,11 +96,11 @@ func TestNativeRunDeliversTheEnvironmentsTelemetryCarriers(t *testing.T) { overrides: map[string]string{"OTEL_SERVICE_NAME": "operator-named"}, runtimeContext: resources.RuntimeContextNative, } - got := runner.runtimeOverridesFor(nil) + got := testRuntimeOverridesFor(t, runner, nil) require.Equal(t, "http://localhost:4317", got["OTEL_EXPORTER_OTLP_ENDPOINT"]) require.Equal(t, "service.namespace=saas,deployment.environment.name=local", got["OTEL_RESOURCE_ATTRIBUTES"]) require.Equal(t, "operator-named", got["OTEL_SERVICE_NAME"], "an explicit --set of the same key is the operator's to make") silent := &Runner{instance: instance, world: &World{Env: &environments.Environment{Name: "local"}}, runtimeContext: resources.RuntimeContextNative} - require.NotContains(t, silent.runtimeOverridesFor(nil), "OTEL_EXPORTER_OTLP_ENDPOINT") + require.NotContains(t, testRuntimeOverridesFor(t, silent, nil), "OTEL_EXPORTER_OTLP_ENDPOINT") } diff --git a/pkg/run/identity/identity.go b/pkg/run/identity/identity.go index dfa83fba..43d1cb46 100644 --- a/pkg/run/identity/identity.go +++ b/pkg/run/identity/identity.go @@ -11,10 +11,13 @@ package identity import ( "crypto/rand" + "crypto/sha256" "encoding/hex" + "errors" "fmt" "os" "path/filepath" + "strings" "sync" "github.com/codefly-dev/core/workloadidentity" @@ -38,7 +41,9 @@ type Launcher struct { runID string mu sync.Mutex - credentials map[string]Credential // unique -> projected credential + credentials map[projection]Credential // active refresh only + cleanup map[string]bool // retired paths retained until removal succeeds + closed bool } // Start creates the run's issuer. trustDomain is the environment's, audience @@ -51,7 +56,7 @@ func Start(trustDomain, audience, stateDir string) (*Launcher, error) { if err != nil { return nil, err } - return &Launcher{issuer: issuer, trustDomain: trustDomain, audience: audience, stateDir: stateDir, runID: randomID(), credentials: map[string]Credential{}}, nil + return &Launcher{issuer: issuer, trustDomain: trustDomain, audience: audience, stateDir: stateDir, runID: randomID(), credentials: map[projection]Credential{}}, nil } // Workload is the declared workload of module/service under this run's trust @@ -93,53 +98,139 @@ func (l *Launcher) WriteAnchor(path string) error { return l.issuer.Anchor().WriteFile(path) } -// Project issues the credential of unique's workload once, to a file the -// process reads, and returns it. A second Project of the same unique returns -// the same credential. +type projection struct { + Unique string + Audience string +} + +// Project preserves the implicit host-audience projection. func (l *Launcher) Project(unique string, workload workloadidentity.Workload) (Credential, error) { + return l.ProjectForAudience(unique, workload, l.audience) +} + +// ProjectForAudience projects one declared workload for one audience. Every +// audience shares the workload instance, but reads a separate private file. +func (l *Launcher) ProjectForAudience(unique string, workload workloadidentity.Workload, audience string) (Credential, error) { l.mu.Lock() defer l.mu.Unlock() - if credential, done := l.credentials[unique]; done { + if l.closed { + return Credential{}, fmt.Errorf("native identity launcher is closed") + } + if unique != workload.Unique() || workload.TrustDomain != l.trustDomain || workload.Instance != "" { + return Credential{}, fmt.Errorf("projection %q must name its declared workload under this run's trust domain", unique) + } + if audience == "" { + return Credential{}, fmt.Errorf("native projection requires an audience") + } + for _, part := range []string{workload.Module, workload.Service} { + if part == "." || part == ".." || part == "" || strings.ContainsAny(part, "/\\\x00") { + return Credential{}, fmt.Errorf("native workload name %q is not a safe path component", part) + } + } + key := projection{unique, audience} + if credential, done := l.credentials[key]; done { return credential, nil } workload.Instance = workload.Service + "-" + l.runID - credential := Credential{ - Workload: workload, Container: workload.Service, - File: filepath.Join(l.stateDir, "identities", workload.Module, workload.Service, "credential"), + // Preserve the host file path. Additional audiences are opaque, so only + // their full digest enters the generated path, under the same workload. + path := filepath.Join(l.stateDir, "identities", workload.Module, workload.Service, "credential") + if audience != l.audience { + audienceKey := sha256.Sum256([]byte(audience)) + path = filepath.Join(filepath.Dir(path), "audiences", hex.EncodeToString(audienceKey[:]), "credential") + } + if l.cleanup[path] { + if err := l.removeCredential(path); err != nil { + return Credential{}, err + } } - if err := l.issuer.WriteCredential(credential.File, l.subject(&credential), l.audience); err != nil { + credential := Credential{Workload: workload, Container: workload.Service, File: path} + if err := l.issuer.WriteCredential(credential.File, l.subject(&credential), audience); err != nil { return Credential{}, err } - l.credentials[unique] = credential + l.credentials[key] = credential return credential, nil } -// Refresh re-issues every projected credential, as a cell rotates a projected -// token: a client that re-reads the file before every mint always presents -// one the host has not recorded yet. -func (l *Launcher) Refresh() { +// Refresh re-issues each projection under its original audience. +func (l *Launcher) Refresh() error { l.mu.Lock() defer l.mu.Unlock() - for _, credential := range l.credentials { - _ = l.issuer.WriteCredential(credential.File, l.subject(&credential), l.audience) + var failures []error + for key, credential := range l.credentials { + if err := l.issuer.WriteCredential(credential.File, l.subject(&credential), key.Audience); err != nil { + failures = append(failures, fmt.Errorf("refresh %s for %q: %w", key.Unique, key.Audience, err)) + } } + return errors.Join(failures...) } // RefreshInterval is how often Refresh runs. const RefreshInterval = native.RefreshInterval -// Forget drops a stopped workload's projection. -func (l *Launcher) Forget(unique string) { +// Forget retires every audience before attempting removal. Failed paths stay +// pending, never active: Refresh cannot recreate a stopped workload's files. +func (l *Launcher) Forget(unique string) error { + l.mu.Lock() + defer l.mu.Unlock() + l.retire(unique) + return l.cleanupFiles() +} + +func (l *Launcher) retire(unique string) { + if l.cleanup == nil { + l.cleanup = map[string]bool{} + } + for key, credential := range l.credentials { + if unique != "" && key.Unique != unique { + continue + } + delete(l.credentials, key) + l.cleanup[credential.File] = true + } +} + +// RetryCleanup retries retired files without reactivating their credentials. +func (l *Launcher) RetryCleanup() error { + l.mu.Lock() + defer l.mu.Unlock() + return l.cleanupFiles() +} + +func (l *Launcher) cleanupFiles() error { + var failures []error + for path := range l.cleanup { + if err := l.removeCredential(path); err != nil { + failures = append(failures, err) + } + } + return errors.Join(failures...) +} + +func (l *Launcher) removeCredential(path string) error { + if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("remove native credential %s: %w", path, err) + } + delete(l.cleanup, path) + return nil +} + +// Close prevents further projections, retires all credentials and retries any +// pending removals. It leaves unrelated run state intact. Copied tokens expire; +// deleting a projected file does not revoke bytes already read by a process. +func (l *Launcher) Close() error { l.mu.Lock() defer l.mu.Unlock() - delete(l.credentials, unique) + l.closed = true + l.retire("") + return l.cleanupFiles() } // Projected reports whether unique has a credential. func (l *Launcher) Projected(unique string) (Credential, bool) { l.mu.Lock() defer l.mu.Unlock() - credential, ok := l.credentials[unique] + credential, ok := l.credentials[projection{unique, l.audience}] return credential, ok } diff --git a/pkg/run/identity/identity_test.go b/pkg/run/identity/identity_test.go new file mode 100644 index 00000000..ef15eaff --- /dev/null +++ b/pkg/run/identity/identity_test.go @@ -0,0 +1,140 @@ +package identity + +import ( + "os" + "path/filepath" + "testing" + + "github.com/codefly-dev/core/workloadidentity/native" + "github.com/stretchr/testify/require" +) + +func TestHostProjectionCannotBePresentedToSecondAudience(t *testing.T) { + launcher, err := Start("example.test", "host", t.TempDir()) + require.NoError(t, err) + workload := launcher.Workload("example", "api") + launcher.Declare(workload) + credential, err := launcher.Project("example/api", workload) + require.NoError(t, err) + token, err := os.ReadFile(credential.File) + require.NoError(t, err) + attester, err := native.NewAttester(launcher.Anchor()) + require.NoError(t, err) + _, err = attester.Attest(t.Context(), string(token), "host") + require.NoError(t, err) + _, err = attester.Attest(t.Context(), string(token), "second/audience") + require.Error(t, err, "the existing projection issues only the host audience") +} + +func TestAudienceProjectionsRefreshAndForget(t *testing.T) { + launcher, err := Start("example.test", "host", t.TempDir()) + require.NoError(t, err) + workload := launcher.Workload("example", "api") + launcher.Declare(workload) + host, err := launcher.Project("example/api", workload) + require.NoError(t, err) + second, err := launcher.ProjectForAudience("example/api", workload, "https://verifier.example/audience") + require.NoError(t, err) + require.NotEqual(t, host.File, second.File) + require.Equal(t, filepath.Join(launcher.stateDir, "identities", "example", "api", "credential"), host.File) + require.Equal(t, host.Workload, second.Workload) + same, err := launcher.ProjectForAudience("example/api", workload, "host") + require.NoError(t, err) + require.Equal(t, host, same) + anchor := filepath.Join(t.TempDir(), "anchor.json") + require.NoError(t, launcher.WriteAnchor(anchor)) + t.Setenv(native.AnchorEnvironment, anchor) + attester, err := native.LoadAttester() + require.NoError(t, err) + previous := map[string]string{} + verify := func(refreshed bool) { + for audience, credential := range map[string]Credential{"host": host, "https://verifier.example/audience": second} { + token, err := os.ReadFile(credential.File) + require.NoError(t, err) + info, err := os.Stat(credential.File) + require.NoError(t, err) + require.Equal(t, os.FileMode(0o600), info.Mode().Perm()) + attested, err := attester.Attest(t.Context(), string(token), audience) + require.NoError(t, err) + require.Equal(t, credential.Workload, attested.Workload) + other := "host" + if audience == "host" { + other = "https://verifier.example/audience" + } + _, err = attester.Attest(t.Context(), string(token), other) + require.Error(t, err) + if refreshed { + require.NotEqual(t, previous[audience], string(token)) + } + previous[audience] = string(token) + } + } + verify(false) + require.NoError(t, launcher.Refresh()) + verify(true) + require.NoError(t, launcher.Forget("example/api")) + require.NoError(t, launcher.Refresh()) + for _, credential := range []Credential{host, second} { + require.NoFileExists(t, credential.File) + } + _, exists := launcher.Projected("example/api") + require.False(t, exists) + // Redeclaration/reprojection cannot retain the previous credential bytes. + launcher.Declare(workload) + fresh, err := launcher.Project("example/api", workload) + require.NoError(t, err) + token, err := os.ReadFile(fresh.File) + require.NoError(t, err) + require.NotEqual(t, previous["host"], string(token)) + require.NoFileExists(t, second.File) +} + +func TestProjectionRefusesUndeclaredAndConflictingWorkloads(t *testing.T) { + launcher, err := Start("example.test", "host", t.TempDir()) + require.NoError(t, err) + workload := launcher.Workload("example", "api") + _, err = launcher.ProjectForAudience("example/api", workload, "second") + require.ErrorContains(t, err, "did not declare") + launcher.Declare(workload) + _, err = launcher.ProjectForAudience("../escape", workload, "second") + require.Error(t, err) + _, err = launcher.ProjectForAudience("example/api", workload, "") + require.Error(t, err) + _, err = launcher.Project("example/api", workload) + require.NoError(t, err) + workload.TrustDomain = "other.test" + _, err = launcher.Project("example/api", workload) + require.Error(t, err) +} + +func TestFailedCleanupRetiresRefreshAndRetriesEveryAudience(t *testing.T) { + launcher, err := Start("example.test", "host", t.TempDir()) + require.NoError(t, err) + workload := launcher.Workload("example", "api") + launcher.Declare(workload) + first, err := launcher.Project("example/api", workload) + require.NoError(t, err) + second, err := launcher.ProjectForAudience("example/api", workload, "second") + require.NoError(t, err) + require.NoError(t, os.Remove(first.File)) + require.NoError(t, os.Mkdir(first.File, 0o700)) + blocker := filepath.Join(first.File, "busy") + require.NoError(t, os.WriteFile(blocker, []byte("fixture"), 0o600)) + require.Error(t, launcher.Forget("example/api")) + require.NoFileExists(t, second.File) + require.NoError(t, launcher.Refresh(), "retired paths are not refreshed even when removal fails") + require.NoFileExists(t, second.File) + require.NoError(t, os.Remove(blocker)) + require.NoError(t, launcher.RetryCleanup()) + _, err = os.Stat(first.File) + require.ErrorIs(t, err, os.ErrNotExist) + require.NoError(t, launcher.RetryCleanup()) + fresh, err := launcher.Project("example/api", workload) + require.NoError(t, err) + require.FileExists(t, fresh.File) + require.NoError(t, launcher.Close()) + require.NoError(t, launcher.Refresh()) + require.NoFileExists(t, fresh.File) + _, err = launcher.Project("example/api", workload) + require.ErrorContains(t, err, "closed") +}