diff --git a/cmd/deploy/dev.go b/cmd/deploy/dev.go index 3a09b9fc..f763730f 100644 --- a/cmd/deploy/dev.go +++ b/cmd/deploy/dev.go @@ -20,6 +20,7 @@ var ( devCommit bool devPush bool devRebuild bool + devPublished bool ) // DevCmd is the dev escape hatch: ship one service's current code into an @@ -78,7 +79,7 @@ the next full ` + "`codefly deploy gitops render --env `" + ` clea cli.Warning("DEV DEPLOYMENT: %s/%s from %s (%s) into %s — this environment will run code no release describes until the next full render", module.Name, target.Name, source.Dir, source.Origin, env.Name) result, err := gitops.DeployDev(ctx, &gitops.DevRequest{ Workspace: workspace, Module: module, Service: target.Name, Environment: env, - AppProject: devAppProject, Source: source, Rebuild: devRebuild, Sink: cli.NewOutputSink(), + AppProject: devAppProject, Source: source, Rebuild: devRebuild, Published: devPublished, Sink: cli.NewOutputSink(), }) if err != nil { return err @@ -108,6 +109,7 @@ the next full ` + "`codefly deploy gitops render --env `" + ` clea func init() { DevCmd.Flags().StringVar(&devEnv, "env", "", "Environment whose rendered tree to patch (required)") + DevCmd.Flags().BoolVar(&devPublished, "published", false, "Patch the existing publication under the environment's GitOps path instead of the render scratch tree") DevCmd.Flags().StringVar(&devPath, "path", "", "Service source directory to deploy (default: the machine-local service override)") DevCmd.Flags().StringVar(&devAppProject, "app-project", "", "AppProject the environment was rendered for (checked against the render)") DevCmd.Flags().BoolVar(&devCommit, "commit", false, "Commit the change as `dev: / from @[-dirty]`") diff --git a/pkg/environments/environment_test.go b/pkg/environments/environment_test.go index b641f10c..4a504f54 100644 --- a/pkg/environments/environment_test.go +++ b/pkg/environments/environment_test.go @@ -90,10 +90,11 @@ func serviceIdentityDeclaration(serviceName string) string { // service passes the graph check. func TestValidateEnvironmentsAllowsKnownService(t *testing.T) { for name, declaration := range map[string]string{ - "service-secrets": serviceSecretsDeclaration("accounts"), - "service-config": serviceConfigDeclaration("accounts"), - "qualified service-config": serviceConfigDeclaration("saas/accounts"), - "service-identity": serviceIdentityDeclaration("accounts"), + "service-secrets": serviceSecretsDeclaration("accounts"), + "qualified service-secrets": serviceSecretsDeclaration("saas/accounts"), + "service-config": serviceConfigDeclaration("accounts"), + "qualified service-config": serviceConfigDeclaration("saas/accounts"), + "service-identity": serviceIdentityDeclaration("accounts"), } { t.Run(name, func(t *testing.T) { ctx := context.Background() @@ -113,10 +114,11 @@ func TestValidateEnvironmentsAllowsKnownService(t *testing.T) { // projection time; the graph check must reject it. func TestValidateEnvironmentsRejectsUnknownService(t *testing.T) { for name, declaration := range map[string]string{ - "service-secrets": serviceSecretsDeclaration("accunts"), - "service-config": serviceConfigDeclaration("accunts"), - "qualified service-config": serviceConfigDeclaration("wrong/accounts"), - "service-identity": serviceIdentityDeclaration("accunts"), + "service-secrets": serviceSecretsDeclaration("accunts"), + "qualified service-secrets": serviceSecretsDeclaration("wrong/accounts"), + "service-config": serviceConfigDeclaration("accunts"), + "qualified service-config": serviceConfigDeclaration("wrong/accounts"), + "service-identity": serviceIdentityDeclaration("accunts"), } { t.Run(name, func(t *testing.T) { ctx := context.Background() diff --git a/pkg/environments/workspace.go b/pkg/environments/workspace.go index 8e5807bb..9b9cae41 100644 --- a/pkg/environments/workspace.go +++ b/pkg/environments/workspace.go @@ -235,7 +235,7 @@ func ValidateWorkspace(ctx context.Context, workspace *resources.Workspace) erro // A module-qualified service-identity key names exactly one // service of that module, or nothing (a typo that would leave // the workload on the default identity with no report). - if _, service, qualified := strings.Cut(name, "/"); qualified && (block == "service-identity" || block == "service-mesh" || block == "service-config") { + if _, service, qualified := strings.Cut(name, "/"); qualified { if !slices.Contains(byName[service], name) { return w.Wrap(fmt.Errorf("environment %q %s references unknown service %q", env.Name, block, name)) } diff --git a/pkg/gitops/cell.go b/pkg/gitops/cell.go index 3e6b09a0..04476756 100644 --- a/pkg/gitops/cell.go +++ b/pkg/gitops/cell.go @@ -103,13 +103,19 @@ func RenderCell(ctx context.Context, workspace *resources.Workspace, env *enviro // deriveCell is RenderCell without the write: the cell file the rendered // trees and the composition describe, and the path it belongs at. func deriveCell(ctx context.Context, workspace *resources.Workspace, env *environments.Environment) (cellfile.File, CellResult, error) { + return deriveCellFrom(ctx, workspace, env, "") +} + +func deriveCellFrom(ctx context.Context, workspace *resources.Workspace, env *environments.Environment, modulesDir string) (cellfile.File, CellResult, error) { if workspace == nil || env == nil { return cellfile.File{}, CellResult{}, errors.New("a cell is rendered for a workspace and an environment") } if env.Host == nil { return cellfile.File{}, CellResult{}, fmt.Errorf("environment %s declares no host, so it has no cell to describe", env.Name) } - modulesDir := filepath.Dir(moduleRenderDestination(workspace, "module")) + if modulesDir == "" { + modulesDir = filepath.Dir(moduleRenderDestination(workspace, "module")) + } entries, err := os.ReadDir(modulesDir) if errors.Is(err, fs.ErrNotExist) { return cellfile.File{}, CellResult{}, fmt.Errorf("no module has been rendered under %s", modulesDir) @@ -164,6 +170,22 @@ func deriveCell(ctx context.Context, workspace *resources.Workspace, env *enviro if err = refuseUnrenderedModules(ctx, workspace, env, result.Modules); err != nil { return cellfile.File{}, CellResult{}, err } + // Undelivered modules have no workload identity the cell can admit. + delivered := map[string]bool{} + for _, namespace := range file.Namespaces { + for _, workload := range namespace.Workloads { + delivered[workload.Service] = true + } + } + for ni := range file.Namespaces { + for wi := range file.Namespaces[ni].Workloads { + workload := &file.Namespaces[ni].Workloads[wi] + for ei := range workload.Endpoints { + endpoint := &workload.Endpoints[ei] + endpoint.Consumers = slices.DeleteFunc(endpoint.Consumers, func(service string) bool { return !delivered[service] }) + } + } + } sort.Slice(file.Namespaces, func(i, j int) bool { return file.Namespaces[i].Name < file.Namespaces[j].Name }) sort.Strings(result.Modules) sort.Strings(result.Skipped) @@ -204,6 +226,34 @@ func refuseUnrenderedModules(ctx context.Context, workspace *resources.Workspace if len(missing) == 0 { return nil } + previousPath, err := CellPath(workspace, env) + if err != nil { + return err + } + data, err := os.ReadFile(previousPath) + if err != nil && !errors.Is(err, fs.ErrNotExist) { + return fmt.Errorf("read previous cell before checking missing renders: %w", err) + } + if err == nil { + previous, parseErr := cellfile.Parse(data) + if parseErr != nil { + return fmt.Errorf("read previous cell before checking missing renders: %w", parseErr) + } + if previous.Environment != env.Name || previous.Coordinate != env.Host.Coordinate || previous.Component != env.Host.Component || previous.TrustDomain != env.Host.TrustDomain || previous.Domain != env.Host.Domain { + return errors.New("previous cell identity differs from this environment; cannot establish which modules were delivered") + } + delivered := make(map[string]bool, len(previous.Namespaces)) + for _, namespace := range previous.Namespaces { + if namespace.Module == "" { + return errors.New("previous cell namespace names no module; cannot establish which modules were delivered") + } + delivered[namespace.Module] = true + } + missing = slices.DeleteFunc(missing, func(module string) bool { return !delivered[module] }) + if len(missing) == 0 { + return nil + } + } sort.Strings(missing) commands := make([]string, 0, len(missing)) for _, module := range missing { diff --git a/pkg/gitops/cell_test.go b/pkg/gitops/cell_test.go index 1a171ad9..845144c4 100644 --- a/pkg/gitops/cell_test.go +++ b/pkg/gitops/cell_test.go @@ -658,3 +658,47 @@ func TestThePreparedRunnableReachIsAConsumerEdge(t *testing.T) { require.EqualError(t, err, "module shop derived operation shop-workload-sync: service api publishes it on endpoint workload, which that service does not declare") }) } + +func TestExistingCellPermitsNewUnrenderedModulesWithoutLosingDeliveredModules(t *testing.T) { + workspace := writeCellWorkspace(t) + env := selectedEnvironment(t, workspace, "staging") + renderCellTree(t, workspace, "shop", "api", "acme-shop", "staging") + renderCellTree(t, workspace, "billing", "ledger", "acme-billing", "staging") + result, err := RenderCell(context.Background(), workspace, env) + require.NoError(t, err) + before, err := os.ReadFile(result.Path) + require.NoError(t, err) + workspace.Modules = append(workspace.Modules, &resources.ModuleReference{Name: "future"}) + require.NoError(t, refuseUnrenderedModules(context.Background(), workspace, env, []string{"shop", "billing"})) + require.ErrorContains(t, refuseUnrenderedModules(context.Background(), workspace, env, []string{"shop"}), "billing") + after, err := os.ReadFile(result.Path) + require.NoError(t, err) + require.Equal(t, before, after) + modulePath := filepath.Join(workspace.Dir(), "modules", "future") + servicePath := filepath.Join(modulePath, "services", "consumer") + require.NoError(t, os.MkdirAll(servicePath, 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(modulePath, resources.ModuleConfigurationName), []byte("kind: module\nname: future\nservices:\n - name: consumer\n"), 0o644)) + service := cellServiceYAML("consumer") + "service-dependencies:\n - name: api\n module: shop\n endpoints:\n - name: grpc\n" + require.NoError(t, os.WriteFile(filepath.Join(servicePath, resources.ServiceConfigurationName), []byte(service), 0o644)) + _, err = RenderCell(context.Background(), workspace, env) + require.NoError(t, err) + current, err := os.ReadFile(result.Path) + require.NoError(t, err) + file, err := cellfile.Parse(current) + require.NoError(t, err) + for _, namespace := range file.Namespaces { + for _, workload := range namespace.Workloads { + for _, endpoint := range workload.Endpoints { + require.NotContains(t, endpoint.Consumers, "future/consumer") + } + } + } + + previous, err := cellfile.Parse(before) + require.NoError(t, err) + previous.Coordinate = "another-cell" + require.NoError(t, writeCell(previous, result.Path)) + require.ErrorContains(t, refuseUnrenderedModules(context.Background(), workspace, env, []string{"shop", "billing"}), "identity differs") + require.NoError(t, os.Remove(result.Path)) + require.ErrorContains(t, refuseUnrenderedModules(context.Background(), workspace, env, []string{"shop", "billing"}), "future") +} diff --git a/pkg/gitops/dev.go b/pkg/gitops/dev.go index 1d0416a0..5d9294c6 100644 --- a/pkg/gitops/dev.go +++ b/pkg/gitops/dev.go @@ -87,6 +87,7 @@ func ResolveDevSource(module, service, path string, override *resources.ServiceR // DevRequest is one dev deployment of a single service. type DevRequest struct { + Published bool Workspace *resources.Workspace Module *resources.Module Service string @@ -186,6 +187,16 @@ func digestImages(root string) ([]string, error) { // already-rendered tree of the requested environment. func DeployDev(ctx context.Context, request *DevRequest) (DevResult, error) { root := moduleRenderDestination(request.Workspace, request.Module.Name) + if request.Published { + owned, err := moduleOwnedPath(request.Workspace, request.Environment, request.Module.Name) + if err != nil { + return DevResult{}, err + } + root, err = confinedJoin(request.Workspace.Dir(), owned) + if err != nil { + return DevResult{}, err + } + } inventory, unit, err := loadDevTarget(root, request.Module.Name, request.Service, request.Environment.Name, request.AppProject) if err != nil { return DevResult{}, err @@ -211,11 +222,18 @@ func DeployDev(ctx context.Context, request *DevRequest) (DevResult, error) { if startCell, err = loadDevCell(ctx, request.Workspace, request.Environment); err != nil { return DevResult{}, err } + if request.Published { + startCell.modulesDir = filepath.Dir(root) + } if _, _, err = deriveDevCell(ctx, request.Workspace, request.Environment, startCell, cellService, nil); err != nil { return DevResult{}, err } } - images, err := buildServiceImage(ctx, request.Workspace, request.Module, service, request.Environment, request.Rebuild, request.Sink) + buildEnvironment, err := devBuildEnvironment(root, unit, request.Environment, cellService) + if err != nil { + return DevResult{}, err + } + images, err := buildServiceImage(ctx, request.Workspace, request.Module, service, buildEnvironment, request.Rebuild, request.Sink) if err != nil { return DevResult{}, err } @@ -265,6 +283,43 @@ func DeployDev(ctx context.Context, request *DevRequest) (DevResult, error) { return DevResult{Root: root, Changed: relative, Entry: entry, Cell: cell}, nil } +// A coordinate import can shorten a registry URL to its host while an existing +// publication still uses a product repository prefix. A one-service deployment +// preserves that prefix inside the configured registry; it does not migrate +// repositories or reinterpret mirrors. +func devBuildEnvironment(root string, unit InventoryUnit, env *environments.Environment, service string) (*environments.Environment, error) { + if env.Registry == nil || env.Registry.URL == "" || env.Registry.PullRepository() != strings.TrimSuffix(env.Registry.URL, "/") { + return env, nil + } + images, err := digestImages(filepath.Join(root, filepath.FromSlash(unit.Path))) + if err != nil { + return nil, err + } + base := strings.TrimSuffix(env.Registry.URL, "/") + var selected string + for _, image := range images { + repository, _, _ := strings.Cut(image, "@") + if tag := strings.LastIndex(repository, ":"); tag > strings.LastIndex(repository, "/") { + repository = repository[:tag] + } + prefix, matches := strings.CutSuffix(repository, "/"+service) + if !matches || (prefix != base && !strings.HasPrefix(prefix, base+"/")) { + continue + } + if selected != "" && selected != prefix { + return nil, fmt.Errorf("rendered service %s names multiple registry prefixes", service) + } + selected = prefix + } + if selected == "" || selected == base { + return env, nil + } + copyEnvironment, copyRegistry := *env, *env.Registry + copyRegistry.URL = selected + copyEnvironment.Registry = ©Registry + return ©Environment, nil +} + // moduleRenderDestination is where a full module render writes its owned tree. func moduleRenderDestination(workspace *resources.Workspace, module string) string { return filepath.Join(workspace.Dir(), "deployments", "modules", module) diff --git a/pkg/gitops/dev_test.go b/pkg/gitops/dev_test.go index 5f89397b..e9a61485 100644 --- a/pkg/gitops/dev_test.go +++ b/pkg/gitops/dev_test.go @@ -465,6 +465,33 @@ func TestDeployDevIntoAHostedEnvironmentRederivesTheCellFile(t *testing.T) { require.ElementsMatch(t, result.Changed, strings.Split(committed, "\n"), "the commit carries the cell file with the re-pin") } +func TestDeployDevUsesPublishedEnvironmentWithoutTouchingOtherRender(t *testing.T) { + isolateGit(t) + workspace, env, request, _ := hostedDevFixture(t) + env.Gitops = &environments.EnvironmentGitops{Path: "deployments/environments/staging"} + published := filepath.Join(workspace.Dir(), env.Gitops.Path, "modules") + require.NoError(t, os.MkdirAll(filepath.Dir(published), 0o755)) + require.NoError(t, os.Rename(filepath.Join(workspace.Dir(), "deployments/modules"), published)) + file, cell, err := deriveCellFrom(t.Context(), workspace, env, published) + require.NoError(t, err) + require.NoError(t, writeCell(&file, cell.Path)) + // A scratch render for another environment must neither block nor be edited. + renderCellTree(t, workspace, "shop", "api", "acme-shop", "production") + scratch := moduleRenderDestination(workspace, "shop") + before := snapshotTree(t, scratch) + runGit(t, workspace.Dir(), "init", "-q") + runGit(t, workspace.Dir(), "add", ".") + runGit(t, workspace.Dir(), "commit", "-q", "-m", "publication") + request.Published = true + stubBuild(t, []string{"registry.example.test/acme/api:1.2.0@" + devNewDigest}, nil) + result, err := DeployDev(t.Context(), request) + require.NoError(t, err) + require.Equal(t, before, snapshotTree(t, scratch)) + require.Equal(t, filepath.Join(published, "shop"), result.Root) + require.Equal(t, devNewDigest, cellWorkloadImage(t, cell.Path, "api").Digest) + require.Equal(t, "sha256:"+strings.Repeat("a", 64), cellWorkloadImage(t, cell.Path, "ledger").Digest) +} + // The cell file is re-derived from the whole workspace, so a dev deployment // whose re-derived file would approve anything the branch's file does not — // beyond the one service's new build — is refused before anything is built, @@ -581,3 +608,26 @@ func TestDeployDevCarriesThePublishedDeliveryOverlayAsFound(t *testing.T) { require.True(t, listed, "the re-derived inventory lists the delivery overlay as found: %+v", after.Files) require.Equal(t, moduleManifestDir, after.ModuleManifestPath) } + +func TestDevBuildPreservesPublishedRepositoryPrefix(t *testing.T) { + for _, registry := range []string{"registry.example.com", "localhost:5000"} { + t.Run(registry, func(t *testing.T) { + root := t.TempDir() + unit := InventoryUnit{Path: "services/api"} + unitRoot := filepath.Join(root, unit.Path) + require.NoError(t, os.MkdirAll(unitRoot, 0755)) + require.NoError(t, os.WriteFile(filepath.Join(unitRoot, "deployment.yaml"), []byte(devDeployment("api", registry+"/product/shop/api:old@"+devOldDigest)), 0644)) + env := environmentNamed("staging") + env.Registry = &environments.EnvironmentRegistry{URL: registry, Auth: "docker-creds"} + selected, err := devBuildEnvironment(root, unit, env, "shop/api") + require.NoError(t, err) + require.Equal(t, registry+"/product", selected.Registry.URL) + require.Equal(t, env.Registry.Auth, selected.Registry.Auth) + require.Equal(t, registry, env.Registry.URL, "leave other services' environment unchanged") + env.Registry.URL = "another.example.com" + selected, err = devBuildEnvironment(root, unit, env, "shop/api") + require.NoError(t, err) + require.Same(t, env, selected, "never redirect to a registry outside the declaration") + }) + } +} diff --git a/pkg/gitops/devcell.go b/pkg/gitops/devcell.go index 3a6975d7..7d3e53dd 100644 --- a/pkg/gitops/devcell.go +++ b/pkg/gitops/devcell.go @@ -27,9 +27,10 @@ import ( // approves (the workspace's HEAD), or, in a workspace that is not a git // checkout, the one on disk. type devCell struct { - path string - origin string - file *cellfile.File + modulesDir string + path string + origin string + file *cellfile.File } // loadDevCell reads the cell file a dev deployment into a hosted environment @@ -77,7 +78,7 @@ func loadDevCell(ctx context.Context, workspace *resources.Workspace, env *envir // may move; before the build, when they are not known yet, any of the // service's may. func deriveDevCell(ctx context.Context, workspace *resources.Workspace, env *environments.Environment, start devCell, service string, images []string) (cellfile.File, CellResult, error) { - derived, result, err := deriveCell(ctx, workspace, env) + derived, result, err := deriveCellFrom(ctx, workspace, env, start.modulesDir) if err != nil { return cellfile.File{}, CellResult{}, fmt.Errorf("environment %s's cell file cannot be re-derived, and its cell admits only the builds that file names: %w", env.Name, err) }