Merge pull request #98 from codefly-dev/issue-97-report-the-artifact-a-native-start-ran-runtime

feat(runtime): report the artifact a native start ran

Author: antoinetoussaint-byte

AGENTS.md

@@ -127,6 +127,7 @@ anything from it.
 | `builder.go` | `Create` scaffolding, OpenAPI endpoint loading, Docker + kustomize recipes |
 | `uvsources.go` | the assembled build context: `[tool.uv.sources]` paths leaving the project |
 | `uvgitsources.go` | the git commits `uv.lock` pins, declared for the caller to mirror into the build |
+| `startedartifact.go` | what a native start ran, reported before *every* `StartResponse` (`runtime-started-artifact/v1`) |
 | `probes.go`, `probe.py` | endpoint health translated to Kubernetes probes / a local probe |
 | `templates/` | **what ships** — embedded via `go:embed`, rendered on `Create` |
 | `base/` | a checked-in sample service, embedded nowhere |
@@ -195,6 +196,5 @@ carried here:
   `AGENTS.md` beside what it describes, or into `.claude/skills/`. The cap and
   each skill's frontmatter are asserted in `conformance_test.go`, so `go test
   ./...` enforces them with no workflow change.
-- There is no `CLAUDE.md`. If one is added, it is a one-line `@AGENTS.md`
-  pointer — one canonical source, never a second copy to drift.
+- There is no `CLAUDE.md`; if one is added, it is a one-line `@AGENTS.md` pointer.
 - Treat this file as code: the PR that changes a process updates it.

conformance_test.go

@@ -420,3 +420,68 @@ func TestNixBackendIsAssertedInCI(t *testing.T) {
 			"so a runner that lost nix skips instead of failing")
 	}
 }
+
+// TestEveryStartResponseNamesWhatItRan protects the trap in
+// runtime-started-artifact/v1: core CONSUMES the report at each
+// `StartResponse`, so a Start path that answers STARTED without going through
+// `started()` stamps no artifact, and a native run then refuses a service that
+// is up (cli#993). The failure is silent here — the agent builds, the suite
+// passes, the service serves — and only shows up as a refused delivery, so the
+// invariant is asserted on the source rather than left to review.
+//
+// `started()` is the one legal caller. A new path that needs its own response
+// calls it too; a path that must NOT report is a failure, and failures go
+// through StartError, which discards the report by contract.
+func TestEveryStartResponseNamesWhatItRan(t *testing.T) {
+	files, err := runtimeGoFiles(".")
+	if err != nil {
+		t.Fatalf("enumerate runtime source: %v", err)
+	}
+	sites := 0
+	reported := false
+	for _, name := range files {
+		src, err := os.ReadFile(name)
+		if err != nil {
+			t.Fatalf("read %s: %v", name, err)
+		}
+		content := string(src)
+		sites += strings.Count(content, "Runtime.StartResponse()")
+
+		// The one legal call site, and the report that must precede it in the
+		// same body.
+		body := startedBody(content)
+		if body == "" {
+			continue
+		}
+		if !strings.Contains(body, "Runtime.StartResponse()") {
+			t.Errorf("%s: started() does not answer STARTED", name)
+		}
+		if index := strings.Index(body, "ReportStartedArtifact("); index >= 0 && index < strings.Index(body, "Runtime.StartResponse()") {
+			reported = true
+		} else {
+			t.Errorf("%s: started() answers STARTED without naming what it ran first; StartResponse consumes the report, so the order is the behaviour", name)
+		}
+	}
+	if sites != 1 {
+		t.Errorf("found %d StartResponse call sites, want exactly 1 (started()); a second one is a start that names no build", sites)
+	}
+	if !reported {
+		t.Error("no started() reporting the artifact was found in the runtime source")
+	}
+}
+
+// startedBody is the source of `func (s *Runtime) started()`, from its opening
+// brace to the first line that closes it at column zero. Crude, and enough: it
+// is one short function whose body is what the assertion above is about.
+func startedBody(content string) string {
+	const signature = "func (s *Runtime) started() "
+	start := strings.Index(content, signature)
+	if start < 0 {
+		return ""
+	}
+	rest := content[start:]
+	if end := strings.Index(rest, "\n}\n"); end >= 0 {
+		return rest[:end]
+	}
+	return rest
+}

main.go

@@ -259,6 +259,12 @@ func (s *Service) GetAgentInformation(ctx context.Context, _ *agentv0.AgentInfor
 	// requires the live advertisement, not the linked core version.
 	advertisement.Contract = contract.Current()
 	advertisement.Contract.Capabilities = append(advertisement.Contract.Capabilities, contract.DeploymentCompositionProvenance)
+	// The Runtime names what a native start ran on its StartStatus
+	// (startedartifact.go), so a consumer pinning the build a process runs
+	// reads it here instead of deriving a path from the python layout. The
+	// capability is advertised because this agent calls
+	// ReportStartedArtifact; an agent that does not must not advertise it.
+	advertisement.Contract.Capabilities = append(advertisement.Contract.Capabilities, contract.RuntimeStartedArtifact)
 	return advertisement, nil
 }
 

nix_backend_test.go

@@ -5,6 +5,8 @@ package main
 import (
 	"bytes"
 	"context"
+	"os"
+	"path/filepath"
 	"strings"
 	"testing"
 
@@ -39,3 +41,48 @@ func TestNixBackendToolchain(t *testing.T) {
 		testmatrix.Only("nix"),
 	)
 }
+
+// TestNixBackendNamesTheInterpreterItWouldRun closes the backend half of
+// runtime-started-artifact/v1.
+//
+// The CLI's native delivery no-ops only for a container runtime, so a nix run
+// goes through it and needs a report — and under nix both `uv` and the python
+// it resolves come from the devShell, not from the host. The claim that asking
+// uv "answers for every backend" is therefore asserted here rather than
+// reasoned about: the probe runs through the materialized devShell and must
+// name an absolute, regular file, which is what the consumer digests.
+func TestNixBackendNamesTheInterpreterItWouldRun(t *testing.T) {
+	dir := t.TempDir()
+	require.NoError(t, os.MkdirAll(filepath.Join(dir, "src"), 0o755))
+	// Dependency-free: the interpreter is what is under test, and resolving a
+	// wheel would only make the nix job slower.
+	require.NoError(t, os.WriteFile(filepath.Join(dir, "pyproject.toml"),
+		[]byte("[project]\nname = \"nix-probe\"\nversion = \"0.0.0\"\nrequires-python = \">=3.10\"\n"), 0o644))
+	require.NoError(t, os.WriteFile(filepath.Join(dir, "src", "main.py"), []byte("app = None\n"), 0o644))
+	require.NoError(t, ensureNixFlake(dir))
+
+	runtime := NewRuntime(NewService())
+	runtime.Location = dir
+	runtime.Logger = runtime.Wool
+	runtime.Service.SourceLocation = dir
+
+	testmatrix.ForEachEnvironment(t, dir,
+		func(t *testing.T, env runners.RunnerEnvironment) {
+			// The environment the probe observes, as Init leaves it.
+			sync, err := env.NewProcess("uv", "sync")
+			require.NoError(t, err)
+			sync.WithDir(dir)
+			var synced bytes.Buffer
+			sync.WithOutput(&synced)
+			require.NoError(t, sync.Run(context.Background()), "uv sync in the devShell: %s", synced.String())
+
+			interpreter, err := runtime.probeInterpreter(context.Background(), env, nil)
+			require.NoError(t, err)
+			require.True(t, filepath.IsAbs(interpreter), "%s is not absolute", interpreter)
+			info, err := os.Stat(interpreter)
+			require.NoError(t, err, "the consumer could not read %s", interpreter)
+			require.True(t, info.Mode().IsRegular(), "%s is not a regular file", interpreter)
+		},
+		testmatrix.Only("nix"),
+	)
+}

runtime.go

@@ -66,6 +66,15 @@ type Runtime struct {
 	runnerEnvironment runners.RunnerEnvironment
 	runner            *runnerHandle
 
+	// interpreter is the absolute path of the python `uv run` executes the app
+	// with, as uv resolved it for the launch that produced s.runner (see
+	// startedartifact.go). The launch that publishes the handle publishes it,
+	// a Start that keeps that handle re-reports it unchanged, and the teardown
+	// that drops the environment drops it too — a stale one would name a
+	// program nothing is running. Guarded by runnerMu, like the handle it
+	// describes.
+	interpreter string
+
 	// runnerMu serializes the runner lifecycle. Start, Stop and the supervisor
 	// goroutines are concurrent and all replace s.runner; without it two
 	// overlapping Starts can each stop the same process and leave one of their
@@ -659,7 +668,12 @@ func (s *Runtime) Start(ctx context.Context, req *runtimev0.StartRequest) (*runt
 			// an Init between two identical Starts can republish the ports,
 			// and the process bound to the old one is not what this request
 			// describes however unchanged the request is.
-			return s.Base.Runtime.StartResponse()
+			//
+			// It still names what it runs: StartResponse consumes the report,
+			// so keeping a process without describing it again would stamp no
+			// artifact on this start and have a native run refuse a service
+			// that is up and unchanged.
+			return s.started()
 		default:
 			// Different inputs only reach the service through a new process:
 			// --reload re-executes the app inside the process it was launched
@@ -722,6 +736,13 @@ func (s *Runtime) Start(ctx context.Context, req *runtimev0.StartRequest) (*runt
 		proc.WithEnvironmentVariables(ctx, resources.Env("CODEFLY_CONNECT_PORT", s.connectPort))
 	}
 
+	// What this launch runs, asked of uv in the environment this process is
+	// being given — the overrides above included, since one of them can move
+	// the project environment uv resolves (startedartifact.go). It is a
+	// property of the launch, so it is resolved here rather than in Init and
+	// re-reported unchanged by a later Start that keeps this process.
+	s.interpreter = s.launchedInterpreter(ctx, startEnvs)
+
 	// The process must outlive this RPC, so it runs under a background context
 	// — cancellable, so Stop, Destroy and a replacement can tell the supervisor
 	// below that the exit it is about to see was asked for.
@@ -767,13 +788,27 @@ func (s *Runtime) Start(ctx context.Context, req *runtimev0.StartRequest) (*runt
 	// STARTED is recorded before the supervisor is armed: revoking a status
 	// that was never committed would leave the orchestrator reading STARTED for
 	// a process that is already gone.
-	resp, err := s.Base.Runtime.StartResponse()
+	resp, err := s.started()
 	go s.superviseRunner(runningContext, handle)
 
 	s.Wool.Debug("start done")
 	return resp, err
 }
 
+// started names what this start ran and answers STARTED.
+//
+// Every successful Start returns through here, because core consumes the
+// report at each StartResponse: an agent that names its build at the first
+// start and not at the restart reports nothing for the restart, which a
+// consumer refuses rather than carrying the previous process's answer over to
+// the one now running. The failure paths deliberately do not go through it —
+// StartError discards any report, so a build named for a process that never
+// served is never stamped on the attempt that does.
+func (s *Runtime) started() (*runtimev0.StartResponse, error) {
+	s.Base.Runtime.ReportStartedArtifact(s.startedArtifact())
+	return s.Base.Runtime.StartResponse()
+}
+
 // defaultServeTimeout bounds how long Start waits for the service to answer.
 // It covers `uv run` resolving the environment, the app import and its
 // lifespan startup; a service that has not answered by then is not coming up.
@@ -1279,6 +1314,7 @@ func (s *Runtime) endExecution(ctx context.Context) (bool, error) {
 			// a fresh one rather than hand Code and the REPL a dead handle.
 			released = true
 			s.runnerEnvironment = nil
+			s.interpreter = ""
 			s.FastAPI.Service.SetActiveEnvironment(nil)
 		}
 	}

serve_readiness_test.go

@@ -58,8 +58,8 @@ func TestStartedMeansTheEndpointAnswers(t *testing.T) {
 		t.Fatal("STARTED was returned before anything listened on the endpoint")
 	}
 	require.GreaterOrEqual(t, time.Since(began), 600*time.Millisecond)
-	require.Len(t, env.started(), 1)
-	require.False(t, env.started()[0].isStopped())
+	require.Len(t, uvicornProcs(env), 1)
+	require.False(t, uvicornProcs(env)[0].isStopped())
 }
 
 // TestStartAcceptsAnyHTTPAnswer: a redirect or an error status is still an
@@ -87,10 +87,10 @@ func TestStartFailsWhenTheAppExitsBeforeServing(t *testing.T) {
 	runtime.serveAddress = "http://" + closedAddress(t)
 
 	go func() {
-		for len(env.started()) == 0 || !env.started()[0].isStarted() {
+		for len(uvicornProcs(env)) == 0 || !uvicornProcs(env)[0].isStarted() {
 			time.Sleep(10 * time.Millisecond)
 		}
-		proc := env.started()[0]
+		proc := uvicornProcs(env)[0]
 		_, _ = proc.output.Write([]byte("ModuleNotFoundError: No module named 'src.main'\n"))
 		proc.die(errors.New("exit status 1"))
 	}()
@@ -114,16 +114,16 @@ func TestStartFailsWhenTheEndpointNeverAnswers(t *testing.T) {
 	require.NoError(t, err)
 	require.Equal(t, runtimev0.StartStatus_ERROR, resp.GetStatus().GetState())
 	require.Contains(t, resp.GetStatus().GetMessage(), "did not answer within")
-	require.Len(t, env.started(), 1)
-	require.True(t, env.started()[0].isStopped(), "the process that never served must be stopped")
+	require.Len(t, uvicornProcs(env), 1)
+	require.True(t, uvicornProcs(env)[0].isStopped(), "the process that never served must be stopped")
 	require.Nil(t, currentRunner(runtime))
 
 	// The next Start is a fresh attempt, not an answer from the failed one.
 	runtime.serveAddress = ""
 	resp, err = runtime.Start(context.Background(), &runtimev0.StartRequest{})
 	require.NoError(t, err)
 	require.Equal(t, runtimev0.StartStatus_STARTED, resp.GetStatus().GetState(), resp.GetStatus().GetMessage())
-	require.Len(t, env.started(), 2)
+	require.Len(t, uvicornProcs(env), 2)
 }
 
 // TestStartRefusesAConnectFaceNobodyServes: with connect-endpoint on, an app