fix: recover disposable container scopes with durable ownership

agents/agents.go

@@ -442,6 +442,7 @@ func Serve(reg PluginRegistration) {
 			// insecure default.
 			authUnaryInterceptor(expectedToken),
 			principalUnaryInterceptor(),
+			containerRecoveryInterceptor(),
 			runtimeLoadTracker(&runtimeLoaded),
 			agentRPCInterceptor(),
 		),

agents/container_recovery.go

@@ -0,0 +1,23 @@
+package agents
+
+import (
+	"context"
+
+	agentv0 "github.com/codefly-dev/core/generated/go/codefly/services/agent/v0"
+	"github.com/codefly-dev/core/runners/dockerrun"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/metadata"
+)
+
+func containerRecoveryInterceptor() grpc.UnaryServerInterceptor {
+	return func(ctx context.Context, req any, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (any, error) {
+		if info.FullMethod == agentv0.Agent_GetAgentInformation_FullMethodName {
+			if scope := dockerrun.InheritedContainerRecoveryScope(); scope != "" {
+				if err := grpc.SetHeader(ctx, metadata.Pairs(dockerrun.ContainerRecoveryScopeHeader, scope)); err != nil {
+					return nil, err
+				}
+			}
+		}
+		return handler(ctx, req)
+	}
+}

agents/container_recovery_test.go

@@ -0,0 +1,60 @@
+package agents
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"testing"
+	"time"
+
+	agentv0 "github.com/codefly-dev/core/generated/go/codefly/services/agent/v0"
+	"github.com/codefly-dev/core/runners/dockerrun"
+	"github.com/stretchr/testify/require"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
+	"google.golang.org/grpc/metadata"
+)
+
+func TestAgentAcknowledgesInheritedContainerRecoveryOverGRPC(t *testing.T) {
+	binary := filepath.Join(t.TempDir(), "recoveryagent")
+	output, err := exec.Command("go", "build", "-o", binary, "./testdata/recoveryagent").CombinedOutput()
+	require.NoError(t, err, "%s", output)
+	t.Setenv(dockerrun.ContainerRecoveryScopeEnvironment, "")
+	scope, err := dockerrun.NewContainerRecoveryScope(t.TempDir(), t.TempDir(), "test")
+	require.NoError(t, err)
+	require.NoError(t, dockerrun.SetContainerRecoveryScope(scope))
+	for _, valid := range []bool{true, false} {
+		t.Run(fmt.Sprint(valid), func(t *testing.T) {
+			marker := os.Getenv(dockerrun.ContainerRecoveryScopeEnvironment)
+			if !valid {
+				marker = "0:invalid"
+			}
+			command := exec.Command(binary)
+			command.Env = append(os.Environ(), "CODEFLY_AGENT_TOKEN=recovery-test", "CODEFLY_AGENT_UDS_PATH=", dockerrun.ContainerRecoveryScopeEnvironment+"="+marker)
+			stdout, err := command.StdoutPipe()
+			require.NoError(t, err)
+			var stderr bytes.Buffer
+			command.Stderr = &stderr
+			require.NoError(t, command.Start())
+			t.Cleanup(func() { _ = command.Process.Kill(); _ = command.Wait() })
+			endpoint := readHandshakeEndpoint(t, stdout, &stderr)
+			conn, err := grpc.NewClient(endpoint, grpc.WithTransportCredentials(insecure.NewCredentials()))
+			require.NoError(t, err)
+			t.Cleanup(func() { _ = conn.Close() })
+			ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second)
+			defer cancel()
+			ctx = metadata.AppendToOutgoingContext(ctx, AuthMetadataKey, "recovery-test")
+			var headers metadata.MD
+			_, err = agentv0.NewAgentClient(conn).GetAgentInformation(ctx, &agentv0.AgentInformationRequest{}, grpc.Header(&headers))
+			require.NoError(t, err)
+			if valid {
+				require.Equal(t, []string{dockerrun.InheritedContainerRecoveryScope()}, headers.Get(dockerrun.ContainerRecoveryScopeHeader))
+			} else {
+				require.Empty(t, headers.Get(dockerrun.ContainerRecoveryScopeHeader))
+			}
+		})
+	}
+}

agents/testdata/recoveryagent/main.go

@@ -0,0 +1,18 @@
+package main
+
+import (
+	"context"
+
+	"github.com/codefly-dev/core/agents"
+	agentv0 "github.com/codefly-dev/core/generated/go/codefly/services/agent/v0"
+)
+
+type server struct {
+	agentv0.UnimplementedAgentServer
+}
+
+func (server) GetAgentInformation(context.Context, *agentv0.AgentInformationRequest) (*agentv0.AgentInformation, error) {
+	return &agentv0.AgentInformation{}, nil
+}
+
+func main() { agents.Serve(agents.PluginRegistration{Agent: server{}}) }

docs/session-ledger.md

@@ -300,14 +300,6 @@ cannot claim a container, even when its creator PID is gone. Containers without
 this label, or with missing/malformed owner PIDs, require explicit owner recovery;
 startup does not infer ownership from container names.
 
-Disposable SDK invocations additionally delegate orphan cleanup through a durable
-`codefly.recovery-group` label. Its identity includes the same canonical home and
-workspace and the caller's naming scope, excluding only the current SDK session's
-verified invocation suffix. This delegation is issued only in ephemeral mode with
-matching session metadata. A later disposable invocation in that group can remove
-dead-owner ephemeral siblings; live owners, stateful siblings and ledgered
-containers remain protected. Ordinary naming scopes never gain this delegation.
-
 The CLI sets the scope after resolving its run environment and before spawning
 agents. `SetContainerRecoveryScope` projects a PID-bound process marker to direct
 children; Docker environments built against this Core version emit the label.
@@ -322,8 +314,68 @@ replacement or shutdown. A scoped caller cannot claim an unscoped container, and
 an unscoped caller cannot claim a scoped one. A mismatch is an error requiring
 explicit owner recovery, even when runtime configurations match. Each Docker
 environment retains its scope from first use so another flow changing the process
-marker cannot redirect its cleanup. Recovery groups authorize orphan sweeping
-only; they never authorize adopting another invocation's container.
+marker cannot redirect its cleanup. Cross-scope recovery authorizes orphan
+sweeping only; it never authorizes adopting another invocation's container.
+
+The agent's `GetAgentInformation` response includes the
+`codefly-container-recovery-scope` gRPC header with the validated inherited scope.
+The CLI must require this acknowledgement before Docker provisioning. The
+startup parent identity is retained after reparenting so a CLI crash cannot make
+an already-spawned agent create unlabeled containers or lose its ephemeral intent.
+
+Containers also carry `codefly.recovery-namespace`, a hash of the stable caller
+host identity and canonical home/workspace, without the invocation's naming
+scope. The host identity prevents local PID checks from claiming another host's
+containers on a shared Docker daemon. On Linux it requires a persistent machine
+ID, never a boot ID that would strand containers after reboot, and it includes
+the caller's PID namespace so a PID number from one namespace is never read as
+liveness evidence in another.
+
+That PID namespace is a same-namespace proof, not a durable name: the kernel
+recycles nsfs inodes, so sequential containers can reuse one and concurrent
+containers never share one. Both directions err toward retention, so a mismatch
+only ever declines to reap — but cross-scope recovery between concurrent
+containerized runs on a shared daemon is consequently unavailable.
+
+A host that cannot prove a durable identity resolves a scope with no namespace.
+Cross-scope recovery is then unavailable and `ReapDisposableContainers` says so,
+while the exact-scope sweep continues unchanged; resolution never fails for this
+reason, because the common Linux container base images provide no machine ID at
+all. A namespace that is present and different is a foreign host and is never
+eligible for any sweep or for reuse.
+
+The namespace lets `ReapDisposableContainers` recover explicitly ephemeral
+containers after an SDK/test invocation dies, even when the next invocation has a
+fresh scope. It needs no SDK session metadata and survives a successor that
+picked an entirely unrelated scope. It checks the original agent PID and
+preserves live owners, stateful containers, and session-ledger containers. No
+on-disk registry or inference from names is needed; failed removals leave the
+labels available for retry.
+
+Containers created before the namespace label existed carry only the exact
+scope. Docker cannot add a label to a container that already exists, so the
+exact-scope sweep and reuse both continue to accept a missing namespace label —
+refusing it would strand every pre-upgrade container permanently and refuse to
+reuse retained databases. Cross-scope authority is withheld from them, since
+outside the exact scope the hash proves nothing about home, workspace or host.
+
+The process marker is tagged with its field layout (`pid:v2:scope:namespace`). A
+marker written before the tag existed is honored only for the exact scope, and
+one carrying untagged trailing fields is refused: two revisions gave the field
+after the scope different meanings, and a reader that guesses wrong stamps a
+container with ownership no sweep can match.
+
+The exact-scope sweep no longer delegates across naming scopes. That authority
+belongs solely to `ReapDisposableContainers` and the durable namespace, which
+subsumes what a naming-scope prefix could ever authorize: the CLI runs both
+sweeps, so a prefix-scoped delegation could only ever remove containers the
+namespace sweep had already collected.
+
+Provisioning checks ownership before reusing, replacing, starting or following
+logs from an existing container. It refuses foreign/unverified ownership and
+cross-agent adoption of ephemeral containers: Docker labels are immutable, so
+adoption would leave the old creator recorded and let recovery delete a live
+adopter's container. Stateful reuse within the same verified scope remains valid.
 
 Within the same scope, live owners and running stateful containers are retained.
 Stopped containers and running ephemeral containers with dead owners can be

runners/dockerrun/container_ownership_test.go

@@ -81,7 +81,11 @@ func TestContainerOwnershipGuardsReuseReplacementAndShutdown(t *testing.T) {
 			require.Nil(t, second.instance)
 			// Failed initialization must not let teardown rediscover and delete
 			// the very foreign container that initialization refused to claim.
-			require.ErrorContains(t, second.Shutdown(ctx), "refusing adoption or replacement")
+			// Teardown is fenced to the generation Init acquired (#462), so a
+			// refused Init leaves nothing to tear down. What matters is that the
+			// foreign container survives, which the inspection below asserts;
+			// whether Shutdown reports anything is not part of that contract.
+			_ = second.Shutdown(ctx)
 			inspected, err := first.client.ContainerInspect(ctx, originalID)
 			require.NoError(t, err)
 			require.True(t, inspected.State.Running)

runners/dockerrun/container_recovery_host.go

@@ -0,0 +1,51 @@
+package dockerrun
+
+import (
+	"context"
+	"encoding/hex"
+	"fmt"
+	"os"
+	"runtime"
+	"strings"
+	"time"
+
+	"github.com/shirou/gopsutil/v3/host"
+)
+
+func containerRecoveryHostID() (string, error) {
+	const linuxOS = "linux"
+	if runtime.GOOS == linuxOS {
+		// The PID namespace proves "same namespace as the container's recorded
+		// owner", which is what makes a local liveness check meaningful. It is
+		// not a durable name: the kernel recycles nsfs inodes, so sequential
+		// containers can reuse one and concurrent containers never share one.
+		// Both directions only ever decline to reap, so a stale or distinct
+		// inode costs recovery, never a wrongly deleted container.
+		pidNamespace, err := os.Readlink("/proc/self/ns/pid")
+		if err != nil {
+			return "", fmt.Errorf("identify local PID namespace: %w", err)
+		}
+		// gopsutil's Linux fallback is a boot ID. It changes on reboot and
+		// would silently strand containers, so require a persistent identity.
+		// Absence is reported as no identity rather than an error: alpine,
+		// debian-slim and ubuntu images ship no usable machine ID, and failing
+		// here would stop every containerized run instead of narrowing recovery
+		// to the exact scope. ReapDisposableContainers traces the consequence.
+		for _, path := range []string{"/etc/machine-id", "/var/lib/dbus/machine-id", "/sys/class/dmi/id/product_uuid"} {
+			data, err := os.ReadFile(path) // #nosec G304 -- fixed operating-system identity paths.
+			id := strings.ToLower(strings.ReplaceAll(strings.TrimSpace(string(data)), "-", ""))
+			decoded, decodeErr := hex.DecodeString(id)
+			if err == nil && decodeErr == nil && len(decoded) == 16 && strings.Trim(id, "0") != "" && strings.Trim(id, "f") != "" {
+				return id + ":" + pidNamespace, nil
+			}
+		}
+		return "", nil
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
+	defer cancel()
+	id, err := host.HostIDWithContext(ctx)
+	if err != nil {
+		return "", nil // Same degrade: no durable identity, not a failed run.
+	}
+	return id, nil
+}

runners/dockerrun/container_recovery_reuse.go

@@ -0,0 +1,35 @@
+package dockerrun
+
+import (
+	"fmt"
+
+	"github.com/docker/docker/api/types/container"
+)
+
+// validateContainerRecoveryReuse judges the reuse and fingerprint-replacement
+// paths that name-based lookup cannot. IsContainerPresent has already proven
+// Codefly ownership and an identical exact scope before this runs, so repeating
+// those comparisons here only produced unreachable branches and a second error
+// message for a refusal it had already made. What genuinely remains is the
+// host-binding namespace and the immutability of an ephemeral owner: labels
+// cannot be rewritten, so adopting an ephemeral container would leave its old
+// creator recorded and let recovery remove it while its adopter lives.
+func validateContainerRecoveryReuse(existing, desired *container.Config) error {
+	if existing == nil {
+		return fmt.Errorf("existing container has no verifiable configuration")
+	}
+	// A container created before the namespace label existed carries none, and
+	// Docker cannot add one after creation. Treating that absence as foreign
+	// would refuse to reuse every pre-upgrade container — including the stateful
+	// databases the reuse path exists to preserve across CLI restarts. A label
+	// that is present and different is a genuinely foreign host or PID namespace.
+	if existingNamespace := existing.Labels[LabelCodeflyRecoveryNamespace]; existingNamespace != "" &&
+		existingNamespace != desired.Labels[LabelCodeflyRecoveryNamespace] {
+		return fmt.Errorf("existing container belongs to another host or PID namespace")
+	}
+	if (existing.Labels[LabelCodeflyEphemeral] == labelTrue || desired.Labels[LabelCodeflyEphemeral] == labelTrue) &&
+		existing.Labels[LabelCodeflySession] != desired.Labels[LabelCodeflySession] {
+		return fmt.Errorf("ephemeral container belongs to another agent; its ownership cannot be transferred")
+	}
+	return nil
+}

runners/dockerrun/container_recovery_reuse_test.go

@@ -0,0 +1,100 @@
+//go:build !skip_infra
+
+package dockerrun
+
+import (
+	"context"
+	"os"
+	"os/exec"
+	"strconv"
+	"testing"
+
+	"github.com/codefly-dev/core/resources"
+	"github.com/docker/docker/api/types/container"
+	"github.com/stretchr/testify/require"
+)
+
+func TestContainerRecoveryRejectsAdoptionBeforeReuseOrReplacement(t *testing.T) {
+	backend := newBackend(t)
+	t.Setenv(ContainerRecoveryScopeEnvironment, "")
+	scope, err := NewContainerRecoveryScope(t.TempDir(), t.TempDir(), "reuse")
+	require.NoError(t, err)
+	require.NoError(t, SetContainerRecoveryScope(scope))
+	owner := exec.Command("true")
+	require.NoError(t, owner.Run())
+	for _, tc := range []struct {
+		name                                                     string
+		ephemeral, sameOwner, foreign, legacy, changed, unscoped bool
+		// reject names the refusal expected, or is empty when reuse is allowed.
+		// An unscoped caller is turned away by name-based lookup before reuse
+		// validation is ever reached, so it reports that earlier refusal.
+		reject string
+	}{
+		{name: "ephemeral cannot transfer", ephemeral: true, reject: "recover its owner explicitly"},
+		{name: "own ephemeral can restart", ephemeral: true, sameOwner: true},
+		{name: "stateful can survive agent restart"},
+		{name: "foreign cannot reuse", foreign: true, reject: "recover its owner explicitly"},
+		{name: "foreign cannot replace", foreign: true, changed: true, reject: "recover its owner explicitly"},
+		// Docker cannot add a namespace label to a container that already exists,
+		// so a pre-upgrade container must stay reusable — refusing it would strand
+		// the retained databases this path exists to preserve across restarts.
+		{name: "pre-upgrade container still reusable", legacy: true},
+		{name: "unscoped cannot adopt", unscoped: true, reject: "refusing adoption or replacement"},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			name := uniqueName(t)
+			newEnvironment := func() *DockerEnvironment {
+				t.Helper()
+				env, err := NewDockerHeadlessEnvironment(t.Context(), resources.NewDockerImage("alpine:latest"), name)
+				require.NoError(t, err)
+				t.Cleanup(func() { _ = env.client.Close() })
+				env.WithPause()
+				if tc.ephemeral {
+					env.WithEphemeral()
+				}
+				return env
+			}
+			env := newEnvironment()
+			config, hostConfig, err := env.desiredContainerConfigs(t.Context())
+			require.NoError(t, err)
+			config.Labels[LabelCodeflySession] = strconv.Itoa(owner.Process.Pid)
+			if tc.sameOwner {
+				config.Labels[LabelCodeflySession] = strconv.Itoa(os.Getpid())
+			}
+			if tc.foreign {
+				config.Labels[LabelCodeflyRecoveryNamespace] = "foreign"
+			}
+			if tc.legacy {
+				delete(config.Labels, LabelCodeflyRecoveryNamespace)
+			}
+			if tc.changed {
+				config.Labels[LabelCodeflyConfig] = "old-configuration"
+			}
+			created, err := backend.client.ContainerCreate(t.Context(), config, hostConfig, nil, nil, env.name)
+			require.NoError(t, err)
+			t.Cleanup(func() {
+				_ = backend.client.ContainerRemove(context.Background(), created.ID, container.RemoveOptions{Force: true})
+			})
+			reuser := env
+			if tc.unscoped {
+				// Recovery ownership is fixed on an environment's first use, so
+				// an unscoped caller is a fresh environment that never inherited
+				// a marker — not this one with the ambient marker pulled out
+				// from under its already-resolved identity.
+				t.Setenv(ContainerRecoveryScopeEnvironment, "")
+				reuser = newEnvironment()
+			}
+			err = reuser.GetContainer(t.Context())
+			if tc.reject != "" {
+				require.ErrorContains(t, err, tc.reject)
+			} else {
+				require.NoError(t, err)
+			}
+			inspected, err := backend.client.ContainerInspect(t.Context(), created.ID)
+			require.NoError(t, err, "ownership rejection must not delete or replace the old container")
+			if tc.reject != "" {
+				require.False(t, inspected.State.Running, "ownership rejection must precede starting the old container")
+			}
+		})
+	}
+}