fix: deploy services from scoped published environments

cmd/deploy/dev.go

@@ -20,6 +20,7 @@ var (
 	devCommit     bool
 	devPush       bool
 	devRebuild    bool
+	devPublished  bool
 )
 
 // DevCmd is the dev escape hatch: ship one service's current code into an
@@ -78,7 +79,7 @@ the next full ` + "`codefly deploy gitops render <module> --env <env>`" + ` clea
 		cli.Warning("DEV DEPLOYMENT: %s/%s from %s (%s) into %s — this environment will run code no release describes until the next full render", module.Name, target.Name, source.Dir, source.Origin, env.Name)
 		result, err := gitops.DeployDev(ctx, &gitops.DevRequest{
 			Workspace: workspace, Module: module, Service: target.Name, Environment: env,
-			AppProject: devAppProject, Source: source, Rebuild: devRebuild, Sink: cli.NewOutputSink(),
+			AppProject: devAppProject, Source: source, Rebuild: devRebuild, Published: devPublished, Sink: cli.NewOutputSink(),
 		})
 		if err != nil {
 			return err
@@ -108,6 +109,7 @@ the next full ` + "`codefly deploy gitops render <module> --env <env>`" + ` clea
 
 func init() {
 	DevCmd.Flags().StringVar(&devEnv, "env", "", "Environment whose rendered tree to patch (required)")
+	DevCmd.Flags().BoolVar(&devPublished, "published", false, "Patch the existing publication under the environment's GitOps path instead of the render scratch tree")
 	DevCmd.Flags().StringVar(&devPath, "path", "", "Service source directory to deploy (default: the machine-local service override)")
 	DevCmd.Flags().StringVar(&devAppProject, "app-project", "", "AppProject the environment was rendered for (checked against the render)")
 	DevCmd.Flags().BoolVar(&devCommit, "commit", false, "Commit the change as `dev: <module>/<service> from <path>@<sha>[-dirty]`")

pkg/environments/environment_test.go

@@ -90,10 +90,11 @@ func serviceIdentityDeclaration(serviceName string) string {
 // service passes the graph check.
 func TestValidateEnvironmentsAllowsKnownService(t *testing.T) {
 	for name, declaration := range map[string]string{
-		"service-secrets":          serviceSecretsDeclaration("accounts"),
-		"service-config":           serviceConfigDeclaration("accounts"),
-		"qualified service-config": serviceConfigDeclaration("saas/accounts"),
-		"service-identity":         serviceIdentityDeclaration("accounts"),
+		"service-secrets":           serviceSecretsDeclaration("accounts"),
+		"qualified service-secrets": serviceSecretsDeclaration("saas/accounts"),
+		"service-config":            serviceConfigDeclaration("accounts"),
+		"qualified service-config":  serviceConfigDeclaration("saas/accounts"),
+		"service-identity":          serviceIdentityDeclaration("accounts"),
 	} {
 		t.Run(name, func(t *testing.T) {
 			ctx := context.Background()
@@ -113,10 +114,11 @@ func TestValidateEnvironmentsAllowsKnownService(t *testing.T) {
 // projection time; the graph check must reject it.
 func TestValidateEnvironmentsRejectsUnknownService(t *testing.T) {
 	for name, declaration := range map[string]string{
-		"service-secrets":          serviceSecretsDeclaration("accunts"),
-		"service-config":           serviceConfigDeclaration("accunts"),
-		"qualified service-config": serviceConfigDeclaration("wrong/accounts"),
-		"service-identity":         serviceIdentityDeclaration("accunts"),
+		"service-secrets":           serviceSecretsDeclaration("accunts"),
+		"qualified service-secrets": serviceSecretsDeclaration("wrong/accounts"),
+		"service-config":            serviceConfigDeclaration("accunts"),
+		"qualified service-config":  serviceConfigDeclaration("wrong/accounts"),
+		"service-identity":          serviceIdentityDeclaration("accunts"),
 	} {
 		t.Run(name, func(t *testing.T) {
 			ctx := context.Background()

pkg/environments/workspace.go

@@ -235,7 +235,7 @@ func ValidateWorkspace(ctx context.Context, workspace *resources.Workspace) erro
 				// A module-qualified service-identity key names exactly one
 				// service of that module, or nothing (a typo that would leave
 				// the workload on the default identity with no report).
-				if _, service, qualified := strings.Cut(name, "/"); qualified && (block == "service-identity" || block == "service-mesh" || block == "service-config") {
+				if _, service, qualified := strings.Cut(name, "/"); qualified {
 					if !slices.Contains(byName[service], name) {
 						return w.Wrap(fmt.Errorf("environment %q %s references unknown service %q", env.Name, block, name))
 					}

pkg/gitops/cell.go

@@ -103,13 +103,19 @@
 // deriveCell is RenderCell without the write: the cell file the rendered
 // trees and the composition describe, and the path it belongs at.
 func deriveCell(ctx context.Context, workspace *resources.Workspace, env *environments.Environment) (cellfile.File, CellResult, error) {
+	return deriveCellFrom(ctx, workspace, env, "")
+}
+
+func deriveCellFrom(ctx context.Context, workspace *resources.Workspace, env *environments.Environment, modulesDir string) (cellfile.File, CellResult, error) {
 	if workspace == nil || env == nil {
 		return cellfile.File{}, CellResult{}, errors.New("a cell is rendered for a workspace and an environment")
 	}
 	if env.Host == nil {
 		return cellfile.File{}, CellResult{}, fmt.Errorf("environment %s declares no host, so it has no cell to describe", env.Name)
 	}
-	modulesDir := filepath.Dir(moduleRenderDestination(workspace, "module"))
+	if modulesDir == "" {
+		modulesDir = filepath.Dir(moduleRenderDestination(workspace, "module"))
+	}
 	entries, err := os.ReadDir(modulesDir)
 	if errors.Is(err, fs.ErrNotExist) {
 		return cellfile.File{}, CellResult{}, fmt.Errorf("no module has been rendered under %s", modulesDir)
@@ -164,6 +170,22 @@
 	if err = refuseUnrenderedModules(ctx, workspace, env, result.Modules); err != nil {
 		return cellfile.File{}, CellResult{}, err
 	}
+	// Undelivered modules have no workload identity the cell can admit.
+	delivered := map[string]bool{}
+	for _, namespace := range file.Namespaces {
+		for _, workload := range namespace.Workloads {
+			delivered[workload.Service] = true
+		}
+	}
+	for ni := range file.Namespaces {
+		for wi := range file.Namespaces[ni].Workloads {
+			workload := &file.Namespaces[ni].Workloads[wi]
+			for ei := range workload.Endpoints {
+				endpoint := &workload.Endpoints[ei]
+				endpoint.Consumers = slices.DeleteFunc(endpoint.Consumers, func(service string) bool { return !delivered[service] })
+			}
+		}
+	}
 	sort.Slice(file.Namespaces, func(i, j int) bool { return file.Namespaces[i].Name < file.Namespaces[j].Name })
 	sort.Strings(result.Modules)
 	sort.Strings(result.Skipped)
@@ -204,6 +226,34 @@
 	if len(missing) == 0 {
 		return nil
 	}
+	previousPath, err := CellPath(workspace, env)
+	if err != nil {
+		return err
+	}
+	data, err := os.ReadFile(previousPath)
+	if err != nil && !errors.Is(err, fs.ErrNotExist) {
+		return fmt.Errorf("read previous cell before checking missing renders: %w", err)
+	}
+	if err == nil {
+		previous, parseErr := cellfile.Parse(data)
+		if parseErr != nil {
+			return fmt.Errorf("read previous cell before checking missing renders: %w", parseErr)
+		}
+		if previous.Environment != env.Name || previous.Coordinate != env.Host.Coordinate || previous.Component != env.Host.Component || previous.TrustDomain != env.Host.TrustDomain || previous.Domain != env.Host.Domain {
+			return errors.New("previous cell identity differs from this environment; cannot establish which modules were delivered")
+		}
+		delivered := make(map[string]bool, len(previous.Namespaces))
+		for _, namespace := range previous.Namespaces {
+			if namespace.Module == "" {
+				return errors.New("previous cell namespace names no module; cannot establish which modules were delivered")
+			}
+			delivered[namespace.Module] = true
+		}
+		missing = slices.DeleteFunc(missing, func(module string) bool { return !delivered[module] })
+		if len(missing) == 0 {
+			return nil
+		}
+	}
 	sort.Strings(missing)
 	commands := make([]string, 0, len(missing))
 	for _, module := range missing {
@@ -222,7 +272,7 @@
 	if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
 		return fmt.Errorf("create the cell directory: %w", err)
 	}
 	if err := os.WriteFile(path, body, 0o600); err != nil {
 		return fmt.Errorf("write the cell file: %w", err)
 	}
 	return nil

pkg/gitops/cell_test.go

@@ -658,3 +658,47 @@ func TestThePreparedRunnableReachIsAConsumerEdge(t *testing.T) {
 		require.EqualError(t, err, "module shop derived operation shop-workload-sync: service api publishes it on endpoint workload, which that service does not declare")
 	})
 }
+
+func TestExistingCellPermitsNewUnrenderedModulesWithoutLosingDeliveredModules(t *testing.T) {
+	workspace := writeCellWorkspace(t)
+	env := selectedEnvironment(t, workspace, "staging")
+	renderCellTree(t, workspace, "shop", "api", "acme-shop", "staging")
+	renderCellTree(t, workspace, "billing", "ledger", "acme-billing", "staging")
+	result, err := RenderCell(context.Background(), workspace, env)
+	require.NoError(t, err)
+	before, err := os.ReadFile(result.Path)
+	require.NoError(t, err)
+	workspace.Modules = append(workspace.Modules, &resources.ModuleReference{Name: "future"})
+	require.NoError(t, refuseUnrenderedModules(context.Background(), workspace, env, []string{"shop", "billing"}))
+	require.ErrorContains(t, refuseUnrenderedModules(context.Background(), workspace, env, []string{"shop"}), "billing")
+	after, err := os.ReadFile(result.Path)
+	require.NoError(t, err)
+	require.Equal(t, before, after)
+	modulePath := filepath.Join(workspace.Dir(), "modules", "future")
+	servicePath := filepath.Join(modulePath, "services", "consumer")
+	require.NoError(t, os.MkdirAll(servicePath, 0o755))
+	require.NoError(t, os.WriteFile(filepath.Join(modulePath, resources.ModuleConfigurationName), []byte("kind: module\nname: future\nservices:\n  - name: consumer\n"), 0o644))
+	service := cellServiceYAML("consumer") + "service-dependencies:\n  - name: api\n    module: shop\n    endpoints:\n      - name: grpc\n"
+	require.NoError(t, os.WriteFile(filepath.Join(servicePath, resources.ServiceConfigurationName), []byte(service), 0o644))
+	_, err = RenderCell(context.Background(), workspace, env)
+	require.NoError(t, err)
+	current, err := os.ReadFile(result.Path)
+	require.NoError(t, err)
+	file, err := cellfile.Parse(current)
+	require.NoError(t, err)
+	for _, namespace := range file.Namespaces {
+		for _, workload := range namespace.Workloads {
+			for _, endpoint := range workload.Endpoints {
+				require.NotContains(t, endpoint.Consumers, "future/consumer")
+			}
+		}
+	}
+
+	previous, err := cellfile.Parse(before)
+	require.NoError(t, err)
+	previous.Coordinate = "another-cell"
+	require.NoError(t, writeCell(previous, result.Path))
+	require.ErrorContains(t, refuseUnrenderedModules(context.Background(), workspace, env, []string{"shop", "billing"}), "identity differs")
+	require.NoError(t, os.Remove(result.Path))
+	require.ErrorContains(t, refuseUnrenderedModules(context.Background(), workspace, env, []string{"shop", "billing"}), "future")
+}

pkg/gitops/dev.go

@@ -87,6 +87,7 @@
 
 // DevRequest is one dev deployment of a single service.
 type DevRequest struct {
+	Published   bool
 	Workspace   *resources.Workspace
 	Module      *resources.Module
 	Service     string
@@ -186,6 +187,16 @@
 // already-rendered tree of the requested environment.
 func DeployDev(ctx context.Context, request *DevRequest) (DevResult, error) {
 	root := moduleRenderDestination(request.Workspace, request.Module.Name)
+	if request.Published {
+		owned, err := moduleOwnedPath(request.Workspace, request.Environment, request.Module.Name)
+		if err != nil {
+			return DevResult{}, err
+		}
+		root, err = confinedJoin(request.Workspace.Dir(), owned)
+		if err != nil {
+			return DevResult{}, err
+		}
+	}
 	inventory, unit, err := loadDevTarget(root, request.Module.Name, request.Service, request.Environment.Name, request.AppProject)
 	if err != nil {
 		return DevResult{}, err
@@ -211,11 +222,18 @@
 		if startCell, err = loadDevCell(ctx, request.Workspace, request.Environment); err != nil {
 			return DevResult{}, err
 		}
+		if request.Published {
+			startCell.modulesDir = filepath.Dir(root)
+		}
 		if _, _, err = deriveDevCell(ctx, request.Workspace, request.Environment, startCell, cellService, nil); err != nil {
 			return DevResult{}, err
 		}
 	}
-	images, err := buildServiceImage(ctx, request.Workspace, request.Module, service, request.Environment, request.Rebuild, request.Sink)
+	buildEnvironment, err := devBuildEnvironment(root, unit, request.Environment, cellService)
+	if err != nil {
+		return DevResult{}, err
+	}
+	images, err := buildServiceImage(ctx, request.Workspace, request.Module, service, buildEnvironment, request.Rebuild, request.Sink)
 	if err != nil {
 		return DevResult{}, err
 	}
@@ -265,6 +283,43 @@
 	return DevResult{Root: root, Changed: relative, Entry: entry, Cell: cell}, nil
 }
 
+// A coordinate import can shorten a registry URL to its host while an existing
+// publication still uses a product repository prefix. A one-service deployment
+// preserves that prefix inside the configured registry; it does not migrate
+// repositories or reinterpret mirrors.
+func devBuildEnvironment(root string, unit InventoryUnit, env *environments.Environment, service string) (*environments.Environment, error) {
+	if env.Registry == nil || env.Registry.URL == "" || env.Registry.PullRepository() != strings.TrimSuffix(env.Registry.URL, "/") {
+		return env, nil
+	}
+	images, err := digestImages(filepath.Join(root, filepath.FromSlash(unit.Path)))
+	if err != nil {
+		return nil, err
+	}
+	base := strings.TrimSuffix(env.Registry.URL, "/")
+	var selected string
+	for _, image := range images {
+		repository, _, _ := strings.Cut(image, "@")
+		if tag := strings.LastIndex(repository, ":"); tag > strings.LastIndex(repository, "/") {
+			repository = repository[:tag]
+		}
+		prefix, matches := strings.CutSuffix(repository, "/"+service)
+		if !matches || (prefix != base && !strings.HasPrefix(prefix, base+"/")) {
+			continue
+		}
+		if selected != "" && selected != prefix {
+			return nil, fmt.Errorf("rendered service %s names multiple registry prefixes", service)
+		}
+		selected = prefix
+	}
+	if selected == "" || selected == base {
+		return env, nil
+	}
+	copyEnvironment, copyRegistry := *env, *env.Registry
+	copyRegistry.URL = selected
+	copyEnvironment.Registry = &copyRegistry
+	return &copyEnvironment, nil
+}
+
 // moduleRenderDestination is where a full module render writes its owned tree.
 func moduleRenderDestination(workspace *resources.Workspace, module string) string {
 	return filepath.Join(workspace.Dir(), "deployments", "modules", module)

pkg/gitops/dev_test.go

@@ -465,6 +465,33 @@ func TestDeployDevIntoAHostedEnvironmentRederivesTheCellFile(t *testing.T) {
 	require.ElementsMatch(t, result.Changed, strings.Split(committed, "\n"), "the commit carries the cell file with the re-pin")
 }
 
+func TestDeployDevUsesPublishedEnvironmentWithoutTouchingOtherRender(t *testing.T) {
+	isolateGit(t)
+	workspace, env, request, _ := hostedDevFixture(t)
+	env.Gitops = &environments.EnvironmentGitops{Path: "deployments/environments/staging"}
+	published := filepath.Join(workspace.Dir(), env.Gitops.Path, "modules")
+	require.NoError(t, os.MkdirAll(filepath.Dir(published), 0o755))
+	require.NoError(t, os.Rename(filepath.Join(workspace.Dir(), "deployments/modules"), published))
+	file, cell, err := deriveCellFrom(t.Context(), workspace, env, published)
+	require.NoError(t, err)
+	require.NoError(t, writeCell(&file, cell.Path))
+	// A scratch render for another environment must neither block nor be edited.
+	renderCellTree(t, workspace, "shop", "api", "acme-shop", "production")
+	scratch := moduleRenderDestination(workspace, "shop")
+	before := snapshotTree(t, scratch)
+	runGit(t, workspace.Dir(), "init", "-q")
+	runGit(t, workspace.Dir(), "add", ".")
+	runGit(t, workspace.Dir(), "commit", "-q", "-m", "publication")
+	request.Published = true
+	stubBuild(t, []string{"registry.example.test/acme/api:1.2.0@" + devNewDigest}, nil)
+	result, err := DeployDev(t.Context(), request)
+	require.NoError(t, err)
+	require.Equal(t, before, snapshotTree(t, scratch))
+	require.Equal(t, filepath.Join(published, "shop"), result.Root)
+	require.Equal(t, devNewDigest, cellWorkloadImage(t, cell.Path, "api").Digest)
+	require.Equal(t, "sha256:"+strings.Repeat("a", 64), cellWorkloadImage(t, cell.Path, "ledger").Digest)
+}
+
 // The cell file is re-derived from the whole workspace, so a dev deployment
 // whose re-derived file would approve anything the branch's file does not —
 // beyond the one service's new build — is refused before anything is built,
@@ -581,3 +608,26 @@ func TestDeployDevCarriesThePublishedDeliveryOverlayAsFound(t *testing.T) {
 	require.True(t, listed, "the re-derived inventory lists the delivery overlay as found: %+v", after.Files)
 	require.Equal(t, moduleManifestDir, after.ModuleManifestPath)
 }
+
+func TestDevBuildPreservesPublishedRepositoryPrefix(t *testing.T) {
+	for _, registry := range []string{"registry.example.com", "localhost:5000"} {
+		t.Run(registry, func(t *testing.T) {
+			root := t.TempDir()
+			unit := InventoryUnit{Path: "services/api"}
+			unitRoot := filepath.Join(root, unit.Path)
+			require.NoError(t, os.MkdirAll(unitRoot, 0755))
+			require.NoError(t, os.WriteFile(filepath.Join(unitRoot, "deployment.yaml"), []byte(devDeployment("api", registry+"/product/shop/api:old@"+devOldDigest)), 0644))
+			env := environmentNamed("staging")
+			env.Registry = &environments.EnvironmentRegistry{URL: registry, Auth: "docker-creds"}
+			selected, err := devBuildEnvironment(root, unit, env, "shop/api")
+			require.NoError(t, err)
+			require.Equal(t, registry+"/product", selected.Registry.URL)
+			require.Equal(t, env.Registry.Auth, selected.Registry.Auth)
+			require.Equal(t, registry, env.Registry.URL, "leave other services' environment unchanged")
+			env.Registry.URL = "another.example.com"
+			selected, err = devBuildEnvironment(root, unit, env, "shop/api")
+			require.NoError(t, err)
+			require.Same(t, env, selected, "never redirect to a registry outside the declaration")
+		})
+	}
+}

pkg/gitops/devcell.go

@@ -27,9 +27,10 @@ import (
 // approves (the workspace's HEAD), or, in a workspace that is not a git
 // checkout, the one on disk.
 type devCell struct {
-	path   string
-	origin string
-	file   *cellfile.File
+	modulesDir string
+	path       string
+	origin     string
+	file       *cellfile.File
 }
 
 // loadDevCell reads the cell file a dev deployment into a hosted environment
@@ -77,7 +78,7 @@ func loadDevCell(ctx context.Context, workspace *resources.Workspace, env *envir
 // may move; before the build, when they are not known yet, any of the
 // service's may.
 func deriveDevCell(ctx context.Context, workspace *resources.Workspace, env *environments.Environment, start devCell, service string, images []string) (cellfile.File, CellResult, error) {
-	derived, result, err := deriveCell(ctx, workspace, env)
+	derived, result, err := deriveCellFrom(ctx, workspace, env, start.modulesDir)
 	if err != nil {
 		return cellfile.File{}, CellResult{}, fmt.Errorf("environment %s's cell file cannot be re-derived, and its cell admits only the builds that file names: %w", env.Name, err)
 	}