fix(gitops): the cell file never silently drops a composed module's workloads

cmd/deploy/gitops.go

@@ -486,6 +486,7 @@ func publishRequest(module string) gitops.PublishRequest {
 		Title: gitOpsTitle, Body: gitOpsBody, Local: gitOpsLocal,
 		AllowUnresolvedContracts: gitOpsAllowUnresolvedContracts,
 		Resign:                   gitOpsResign,
+		AllowCellRemovals:        gitOpsAllowCellRemovals,
 	}
 }
 
@@ -580,6 +581,9 @@ func printPublishPlan(plan *gitops.PublishPlan) {
 		cli.Info("Contract checks:")
 		cli.Info("%s", formatContractChecks(plan.ContractChecks))
 	}
+	for _, removal := range plan.CellRemovals {
+		cli.Warning("The cell file removes %s, as --allow-cell-removal names", removal)
+	}
 	for _, stale := range plan.StaleConsumers {
 		cli.Warning("Delivered consumer %s bakes in another value of a workspace configuration group this publication carries; publish it next, so %s does not keep delivering two values of one group", stale, plan.Environment)
 	}
@@ -603,6 +607,7 @@ var (
 	gitOpsLocal                    bool
 	gitOpsAllowUnresolvedContracts bool
 	gitOpsResign                   bool
+	gitOpsAllowCellRemovals        []string
 	gitOpsValidateCluster          bool
 	gitOpsRebuild                  bool
 )
@@ -640,6 +645,10 @@ func init() {
 		command.Flags().BoolVar(&gitOpsResign, "resign", false,
 			"Sign again a delivered document whose carrier the release policy no longer admits (a rotated signing identity); deliberate, never implied")
 	}
+	for _, command := range []*cobra.Command{gitOpsPlanCmd, gitOpsPublishCmd} {
+		command.Flags().StringSliceVar(&gitOpsAllowCellRemovals, "allow-cell-removal", nil,
+			"Module whose removal from the environment's cell file, relative to the base branch's, is intended (repeatable); any other removal refuses")
+	}
 	for _, command := range []*cobra.Command{gitOpsPublishCmd, gitOpsRollbackCmd} {
 		command.Flags().StringVar(&gitOpsMessage, "message", "", "Signed commit message")
 		command.Flags().StringVar(&gitOpsTitle, "title", "", "Promotion pull request title")

docs/cli-reference.md

@@ -2090,6 +2090,7 @@ codefly deploy gitops plan [module]
 Flags:
 
 ```
+      --allow-cell-removal strings                                                        Module whose removal from the environment's cell file, relative to the base branch's, is intended (repeatable); any other removal refuses
       --allow-unresolved-contracts                                                        Downgrade a consumed contract whose exposing module is not yet deployed to a warning, for bootstrap ordering
       --env string                                                                        Environment to promote (default "local")
       --local                                                                             Use a disposable local file Git remote for k3d qualification
@@ -2109,6 +2110,7 @@ codefly deploy gitops publish [module]
 Flags:
 
 ```
+      --allow-cell-removal strings                                                        Module whose removal from the environment's cell file, relative to the base branch's, is intended (repeatable); any other removal refuses
       --allow-unresolved-contracts                                                        Downgrade a consumed contract whose exposing module is not yet deployed to a warning, for bootstrap ordering
       --body string                                                                       Promotion pull request body
       --env string                                                                        Environment to promote (default "local")

docs/commands.md

@@ -888,6 +888,34 @@ and verifier allows from, instead of transcribing them. The consumer validates
 it as another repository's output; this side writes it deterministically and
 reads it back through its own reader.
 
+Because absence is removal, **the cell file is never written without a module
+the workspace composes.** The per-module render records
+(`deployments/modules/<module>/.codefly-render.json`) are the file's only
+source, and they are uncommitted render outputs: a `git clean` of
+`deployments/` removes them. A render that finds a composed module with no
+record for the environment — none at all, or one rendered for another
+environment — writes no cell file, leaves the one on disk as it was, and names
+each such module with the command that renders it (`codefly deploy gitops
+render <module> --env <env>`, or `render --all`). It does not keep that
+module's entries from the file already on disk: an entry is derived from the
+whole composition (its consumers are other modules' dependencies, its bindings
+and egress the environment's), so an entry carried over is a claim about a
+composition that may no longer exist, and no digest is invented for a module
+that was not rendered.
+
+**A publish holds the cell file to the base branch's.** The cell file is
+committed with the rendered trees, and a cell admits only what it names, so
+`plan` and `publish` read the workspace's cell file for a hosted environment
+and the one the delivery base branch carries at
+`<gitops path>/cells/<environment>/cell.yaml`, and refuse when the workspace's
+removes a namespace, a workload, an endpoint or a consumer of an endpoint the
+base admits — naming each removal and the modules it concerns. A removal that
+is intended (a module the workspace no longer composes, a dependency dropped)
+is named with `--allow-cell-removal <module>` (repeatable); a consumer edge is
+covered by naming either the endpoint's module or the consumer's, and the plan
+lists every removal it lets through. A base with no cell file has nothing to
+hold the workspace's to.
+
 For a complete module promotion, use the GitOps lifecycle:
 
 ```bash
@@ -912,7 +940,9 @@ one command renders one composition and a stale sibling tree is never a reason
 to run the render again. A single-module `render` re-derives the cell file
 from its own tree and the sibling trees already rendered; a sibling the cell
 file cannot read, or one left at another inventory schema, is reported as
-stale by name rather than failing the render. An image whose inputs did not
+stale by name rather than failing the render, and so is a composed module with
+no render at all: the module's tree is written, the cell file is not, and the
+warning names the renders still owed. An image whose inputs did not
 change is kept rather than rebuilt, and that decision is taken before any Go
 module graph is fetched, so a render of an up-to-date composition fetches
 nothing; `--rebuild` builds every image regardless. A service that declares

pkg/gitops/cell.go

@@ -34,7 +34,10 @@ import (
 // environment, in the shape the consumer reads (pkg/gitops/cellfile).
 //
 // It is regenerated whole after every module render, so absence is correct: a
-// workload not in the file is not delivered. It cannot live in a module's owned
+// workload not in the file is not delivered. That is why it is never written
+// while a module the workspace composes has no render for the environment
+// (refuseUnrenderedModules), and why a publication holds it to the delivery
+// base branch's file (holdCellToBase). It cannot live in a module's owned
 // tree, where the promotable ruleset admits only Kubernetes manifests and
 // Kustomizations; it lives beside the module trees, under
 // <gitops path>/cells/<environment>/cell.yaml, and is committed with them.
@@ -159,6 +162,9 @@ func deriveCell(ctx context.Context, workspace *resources.Workspace, env *enviro
 	if len(file.Namespaces) == 0 {
 		return cellfile.File{}, CellResult{}, fmt.Errorf("no module is rendered for environment %s under %s", env.Name, modulesDir)
 	}
+	if err = refuseUnrenderedModules(workspace, env, result.Modules); err != nil {
+		return cellfile.File{}, CellResult{}, err
+	}
 	sort.Slice(file.Namespaces, func(i, j int) bool { return file.Namespaces[i].Name < file.Namespaces[j].Name })
 	sort.Strings(result.Modules)
 	sort.Strings(result.Skipped)
@@ -168,6 +174,40 @@ func deriveCell(ctx context.Context, workspace *resources.Workspace, env *enviro
 	return file, result, nil
 }
 
+// refuseUnrenderedModules refuses a cell file that would leave out a module
+// the workspace composes because no render of it for this environment is in
+// the checkout: the record is missing (never rendered, or cleaned away with
+// the rest of the uncommitted render outputs) or describes another
+// environment. The file is regenerated whole and absence is removal, so such
+// a file would withdraw every workload that module delivers from the cell's
+// admission set, and nothing would say so.
+//
+// It refuses rather than keeping the module's entries from the cell file
+// already on disk: an entry is derived from the composition as a whole — its
+// consumers are other modules' declared dependencies, its bindings and egress
+// the environment's — so an entry carried over from an earlier render is a
+// claim about a composition that may no longer exist, and the file it was
+// read from is as likely to have been cleaned as the records were. No digest
+// is invented either: the render of the module is what knows it.
+func refuseUnrenderedModules(workspace *resources.Workspace, env *environments.Environment, rendered []string) error {
+	var missing []string
+	for _, reference := range workspace.Modules {
+		if reference != nil && !slices.Contains(rendered, reference.Name) {
+			missing = append(missing, reference.Name)
+		}
+	}
+	if len(missing) == 0 {
+		return nil
+	}
+	sort.Strings(missing)
+	commands := make([]string, 0, len(missing))
+	for _, module := range missing {
+		commands = append(commands, fmt.Sprintf("codefly deploy gitops render %s --env %s", module, env.Name))
+	}
+	return fmt.Errorf("environment %s's cell file is not written: this checkout holds no render for %s of the composed module(s) %s, and a cell file without them would withdraw every workload they deliver from the cell; render them (%s), or every module at once with `codefly deploy gitops render --all --env %s`",
+		env.Name, env.Name, strings.Join(missing, ", "), strings.Join(commands, "; "), env.Name)
+}
+
 // writeCell encodes the cell file and writes it at path.
 func writeCell(file *cellfile.File, path string) error {
 	body, err := file.Encode()

pkg/gitops/cell_removals.go

@@ -0,0 +1,150 @@
+package gitops
+
+// A render is a claim about a cell: the cell file is regenerated whole and a
+// cell admits only what it names, so a workload, a namespace or a consumer
+// absent from it is withdrawn. A publication promotes a render, so before it
+// does, the workspace's cell file is held to the one the delivery base branch
+// carries, and a removal it does not name is refused: a cell file derived
+// from a checkout missing some modules' renders would otherwise remove those
+// modules from admission without anyone having said so.
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"slices"
+	"strings"
+
+	"github.com/codefly-dev/cli/pkg/environments"
+	"github.com/codefly-dev/cli/pkg/gitops/cellfile"
+	"github.com/codefly-dev/core/resources"
+)
+
+// cellRemoval is one thing the base cell file admits that the next one does
+// not, and the modules it concerns: the namespace's module, and for a
+// consumer, the consumer's module too.
+type cellRemoval struct {
+	modules []string
+	what    string
+}
+
+// cellRemovals names every namespace, workload, endpoint and endpoint
+// consumer base carries that next does not, in base's order.
+func cellRemovals(base, next *cellfile.File) []cellRemoval {
+	var removals []cellRemoval
+	for n := range base.Namespaces {
+		was := &base.Namespaces[n]
+		index := slices.IndexFunc(next.Namespaces, func(candidate cellfile.Namespace) bool { return candidate.Name == was.Name })
+		if index < 0 {
+			removals = append(removals, cellRemoval{modules: []string{was.Module}, what: fmt.Sprintf("namespace %s (module %s, %d workload(s))", was.Name, was.Module, len(was.Workloads))})
+			continue
+		}
+		is := &next.Namespaces[index]
+		for w := range was.Workloads {
+			workload := &was.Workloads[w]
+			found := slices.IndexFunc(is.Workloads, func(candidate cellfile.Workload) bool {
+				return candidate.Name == workload.Name && candidate.Kind == workload.Kind
+			})
+			if found < 0 {
+				removals = append(removals, cellRemoval{modules: []string{was.Module}, what: fmt.Sprintf("workload %s/%s %s", was.Name, workload.Kind, workload.Name)})
+				continue
+			}
+			removals = append(removals, endpointRemovals(was, workload, &is.Workloads[found])...)
+		}
+	}
+	return removals
+}
+
+// endpointRemovals names the endpoints of one workload, and the consumers of
+// each, that the next file no longer carries.
+func endpointRemovals(namespace *cellfile.Namespace, was, is *cellfile.Workload) []cellRemoval {
+	var removals []cellRemoval
+	for e := range was.Endpoints {
+		endpoint := &was.Endpoints[e]
+		where := fmt.Sprintf("%s/%s/%s", namespace.Name, was.Name, endpoint.Name)
+		found := slices.IndexFunc(is.Endpoints, func(candidate cellfile.Endpoint) bool { return candidate.Name == endpoint.Name })
+		if found < 0 {
+			removals = append(removals, cellRemoval{modules: []string{namespace.Module}, what: "endpoint " + where})
+			continue
+		}
+		for _, consumer := range endpoint.Consumers {
+			if slices.Contains(is.Endpoints[found].Consumers, consumer) {
+				continue
+			}
+			consumerModule, _, _ := strings.Cut(consumer, "/")
+			removals = append(removals, cellRemoval{modules: []string{namespace.Module, consumerModule}, what: fmt.Sprintf("consumer %s of %s", consumer, where)})
+		}
+	}
+	return removals
+}
+
+// deliveryCellPath is the environment's cell file in the delivery
+// repository: beside the modules tree, as the render lays it out in the
+// workspace (CellPath).
+func deliveryCellPath(gitopsPath, environment string) string {
+	if gitopsPath == "" {
+		gitopsPath = DefaultGitopsPath
+	}
+	return filepath.ToSlash(filepath.Join(gitopsPath, cellsDir, environment, cellfile.FileName))
+}
+
+// holdCellToBase refuses the publication when the workspace's cell file for a
+// hosted environment removes anything the delivery base revision's file
+// carries, unless every removal concerns a module allowed names. It returns
+// the removals allowed, for the plan to show. A workspace with no cell file,
+// or a base with none, has nothing to hold.
+func holdCellToBase(ctx context.Context, workspace *resources.Workspace, env *environments.Environment, repo, baseBranch, baseRevision, gitopsPath string, allowed []string) ([]string, error) {
+	if env.Host == nil {
+		return nil, nil
+	}
+	path, err := CellPath(workspace, env)
+	if err != nil {
+		return nil, err
+	}
+	data, err := os.ReadFile(path)
+	if errors.Is(err, fs.ErrNotExist) {
+		return nil, nil
+	}
+	if err != nil {
+		return nil, fmt.Errorf("read environment %s's cell file: %w", env.Name, err)
+	}
+	next, err := cellfile.Parse(data)
+	if err != nil {
+		return nil, fmt.Errorf("environment %s's cell file %s cannot be read, so it cannot be held to %s's: %w", env.Name, path, baseBranch, err)
+	}
+	basePath := deliveryCellPath(gitopsPath, env.Name)
+	baseData, err := gitCommandBytes(ctx, repo, "show", baseRevision+":"+basePath)
+	if err != nil {
+		if gitSaysAbsent(err) {
+			return nil, nil
+		}
+		return nil, fmt.Errorf("read %s's cell file %s: %w", baseBranch, basePath, err)
+	}
+	base, err := cellfile.Parse(baseData)
+	if err != nil {
+		return nil, fmt.Errorf("%s's cell file %s cannot be read: %w", baseBranch, basePath, err)
+	}
+	var refused, admitted []string
+	unnamed := map[string]bool{}
+	for _, removal := range cellRemovals(base, next) {
+		if slices.ContainsFunc(removal.modules, func(module string) bool { return slices.Contains(allowed, module) }) {
+			admitted = append(admitted, removal.what)
+			continue
+		}
+		refused = append(refused, removal.what)
+		unnamed[removal.modules[len(removal.modules)-1]] = true
+	}
+	if len(refused) > 0 {
+		modules := make([]string, 0, len(unnamed))
+		for module := range unnamed {
+			modules = append(modules, module)
+		}
+		slices.Sort(modules)
+		return nil, fmt.Errorf("environment %s's cell file %s removes from the cell what %s's admits, and says nothing of it: %s. Publishing it would withdraw those from the cell's admission set. If a module's render is missing from this checkout, render it (`codefly deploy gitops render --all --env %s`); if the removal is intended — a module the workspace no longer composes, a dependency dropped — name each module it concerns with --allow-cell-removal (%s)",
+			env.Name, path, baseBranch, strings.Join(refused, "; "), env.Name, strings.Join(modules, ", "))
+	}
+	return admitted, nil
+}