From d1681ff75f5c3532ac1d2af0d8274ef4a5d951a0 Mon Sep 17 00:00:00 2001 From: Antoine Toussaint <312536060+antoinetoussaint-byte@users.noreply.github.com> Date: Wed, 23 Sep 2026 08:26:53 -0400 Subject: [PATCH] fix: assemble a build context for uv path sources and pin the uv that resolves it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two independent defects made this recipe emit an image build that resolved on the developer's machine and failed inside the builder stage. A `[tool.uv.sources]` path is resolved against the directory of the pyproject.toml that declares it, and uv.lock records the same relative path. The recipe's context was the service directory and nothing beside it, so a path naming a sibling service was absent from the builder stage and `uv sync --frozen` failed there with error: Could not compute absolute path from workspace root and lockfile path Two services sharing one Python project in a repository is a layout this recipe has to build. It now assembles a context when, and only when, a path leaves the project: the service's own tree and each escaping target are placed at their repository-relative paths under one root, and the builder stage's working directory is the project's own path under it, so every relative path resolves there to what it resolves to on the host. Nothing in pyproject.toml or uv.lock is rewritten. Containment is the rule: a path leaving the repository, or naming a directory that is not there, is reported with the declaration rather than skipped, because a silently dropped source would build the image against a different dependency than the developer builds against. That build claims the whole tree it assembled (RECIPE_INVENTORY_SCOPE_TREE), which is what roots the docker context at the assembled output; a build that only renders a Dockerfile and its ignore still claims exactly those and still builds the service directory. Second, the builder stage ran whichever uv the base image had been built with, a version stated nowhere a project or this agent can read. A project written against a newer `[tool.uv]` key was diagnosed there as an unknown field — the base image carries uv 0.5.29, measured, and that uv rejects build-constraint-dependencies. A recipe meant to be durable and reproducible cannot leave its resolver implicit, and the Go recipe already states its Go and Alpine versions outright. The builder stage now copies a pinned uv in before anything resolves. Co-Authored-By: Claude Opus 5 (1M context) --- AGENTS.md | 10 + builder.go | 72 +++++- go.mod | 2 +- main.go | 20 ++ templates/builder/Dockerfile.tmpl | 33 ++- uv_sources_recipe_test.go | 299 +++++++++++++++++++++++ uv_toolchain_recipe_test.go | 109 +++++++++ uvsources.go | 387 ++++++++++++++++++++++++++++++ 8 files changed, 918 insertions(+), 14 deletions(-) create mode 100644 uv_sources_recipe_test.go create mode 100644 uv_toolchain_recipe_test.go create mode 100644 uvsources.go diff --git a/AGENTS.md b/AGENTS.md index 6bc61b1..de3b4b0 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -112,6 +112,7 @@ anything from it. | `main.go` | settings, agent advertisement, composition of the generic Python layer | | `runtime.go` | Init/Start/Stop/Destroy, uvicorn supervision, hot reload, the lock order | | `builder.go` | `Create` scaffolding, OpenAPI endpoint loading, Docker + kustomize recipes | +| `uvsources.go` | the assembled build context: `[tool.uv.sources]` paths leaving the project | | `probes.go`, `probe.py` | endpoint health translated to Kubernetes probes / a local probe | | `templates/` | **what ships** — embedded via `go:embed`, rendered on `Create` | | `base/` | a checked-in sample service, embedded nowhere | @@ -135,6 +136,15 @@ into `templates/factory/code/pyproject.toml.tmpl` yourself. - **Readiness is the endpoint's declared health check, never an open port.** Probe intents (readiness / startup / liveness) stay separate through `resources.PlanEndpointProbes`; collapsing them silently drops a predicate. +- **The build recipe owns its uv, and its build context.** The emitted + Dockerfile copies a pinned uv (`uvVersion` in `main.go`) rather than running + whichever one the base image was built with — a project written against a + newer `[tool.uv]` key otherwise resolves locally and fails in the builder + stage. And a `[tool.uv.sources]` path leaving the project makes the recipe + assemble its own context, mirroring the repository-relative layout so the + same relative path resolves there; that plan claims the whole tree + (`RECIPE_INVENTORY_SCOPE_TREE`), the Dockerfile-only one claims just what it + emitted. Never rewrite `pyproject.toml` or `uv.lock` to make a path resolve. - **The runtime image is pinned.** `docker-image` rejects a bare name and `:latest`. The default `codeflydev/python` companion is built by core and repinned on release — prefer it. diff --git a/builder.go b/builder.go index 5eb541c..f497566 100644 --- a/builder.go +++ b/builder.go @@ -167,6 +167,18 @@ type DockerTemplating struct { Components []string RuntimePackages []string Envs []Env + + // UVImage is the pinned uv distribution the builder stage copies its uv + // from, so the tool that reads pyproject.toml and uv.lock is a declared + // input of the recipe rather than whatever the base image was built with. + UVImage string + + // ServicePrefix, ProjectWorkdir and Carried describe the build context. + // They are empty / defaulted when the recipe builds the service directory + // itself, which renders the Dockerfile this agent has always rendered. + ServicePrefix string + ProjectWorkdir string + Carried []CarriedSource } func (s *Builder) BuildCapabilities(context.Context, *builderv0.BuildCapabilitiesRequest) (*builderv0.BuildCapabilitiesResponse, error) { @@ -183,19 +195,69 @@ func (s *Builder) Build(ctx context.Context, req *builderv0.BuildRequest) (*buil return s.Base.Builder.BuildError(err) } ctx = s.Wool.Inject(ctx) - docker := DockerTemplating{ - Builder: runtimeImage.FullName(), - Components: requirements.All(), - } outputDir := req.GetOutputDirectory() emitted, err := services.PrepareRecipeDestination(builderFS, outputDir) if err != nil { return s.Base.Builder.BuildError(err) } + // A `[tool.uv.sources]` path resolves against the project directory, and the + // default context is the service directory and nothing beside it: carry what + // leaves the project into an assembled context so the builder stage resolves + // what the host resolves. A service with no such path assembles nothing. + assembled, err := assembleUVContext(s.Location, s.SourceLocation, outputDir) + if err != nil { + return s.Base.Builder.BuildError(err) + } + docker := DockerTemplating{ + Builder: runtimeImage.FullName(), + Components: requirements.All(), + UVImage: uvImage, + ProjectWorkdir: defaultProjectWorkdir, + } + if assembled != nil { + docker.ServicePrefix = assembled.ServicePrefix + docker.ProjectWorkdir = assembled.ProjectWorkdir + docker.Carried = assembled.Carried + } if err := s.Base.Templates(ctx, docker, services.WithBuilder(builderFS).WithDestination("%s", outputDir)); err != nil { return s.Base.Builder.BuildError(err) } - return s.Base.Builder.SingleImageBuildResponse(req, s.DockerImage(dockerRequest).FullName(), emitted) + image := s.DockerImage(dockerRequest).FullName() + if assembled == nil { + // Only the Dockerfile and its ignore were written, and the application's + // inputs are the service's own tree: the plan claims what it emitted and + // the caller builds the service directory. + return s.Base.Builder.SingleImageBuildResponse(req, image, emitted) + } + // The whole destination was assembled by this build, so the plan claims all + // of it and the caller builds the assembled tree rather than the service + // directory — which is what RECIPE_INVENTORY_SCOPE_TREE declares. + plan, err := services.BuildDockerBuildPlan(outputDir, []*builderv0.DockerBuildRecipe{{ + Name: "app", + Dockerfile: "Dockerfile", + Context: ".", + Dockerignore: dockerignoreOf(emitted), + Image: image, + Platforms: services.RecipeBuildPlatforms(), + }}) + if err != nil { + return s.Base.Builder.BuildError(err) + } + s.Base.Builder.WithBuildPlan(plan) + return s.Base.Builder.BuildResponse() +} + +// dockerignoreOf names the ignore file only when this build rendered one. +// Probing the destination instead would adopt a stale file: the executor copies +// the referenced ignore to the path buildx discovers, so one left behind by an +// older template set would silently keep files out of the image. +func dockerignoreOf(emitted []string) string { + for _, name := range emitted { + if name == "dockerignore" { + return name + } + } + return "" } // SBOM serves both evidence scopes. Source inventory stays with the generic diff --git a/go.mod b/go.mod index d5e7046..aeeac3f 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.27.0 require ( github.com/codefly-dev/core v0.5.1 github.com/codefly-dev/service-python v0.0.66 + github.com/pelletier/go-toml/v2 v2.4.3 github.com/stretchr/testify v1.12.1 google.golang.org/grpc v1.83.2 gopkg.in/yaml.v3 v3.0.1 @@ -92,7 +93,6 @@ require ( github.com/oklog/ulid/v2 v2.1.2 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.1.1 // indirect - github.com/pelletier/go-toml/v2 v2.4.3 // indirect github.com/pjbgf/sha1cd v0.6.0 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/power-devops/perfstat v0.0.0-20260805114148-88456608a4f6 // indirect diff --git a/main.go b/main.go index a885442..bcf76e0 100644 --- a/main.go +++ b/main.go @@ -94,6 +94,26 @@ type Settings struct { // other codefly-built image. var runtimeImage = &resources.DockerImage{Name: "codeflydev/python", Tag: "0.0.1"} +// uvVersion is the uv the emitted build recipe resolves its lockfile with. +// +// The base image carries a uv of its own, installed when that image was built +// and stated nowhere a project or this agent can read. A build recipe is meant +// to be durable and reproducible, and a recipe whose resolver version is a +// property of its base image is neither: a project written against a newer +// `[tool.uv]` key resolves on the developer's machine and fails inside the +// builder stage with a TOML parse error naming a field the older uv does not +// know. Pinning it here is the same contract the Go recipe already keeps for +// its own toolchain, which states its Go and Alpine versions outright. +// +// Bump it deliberately, together with uvImage. +const uvVersion = "0.12.16" + +// uvImage is the distribution the builder stage copies that uv from. astral +// publishes each release as a multi-architecture image holding the static +// binary, so the copy needs no package manager and no network beyond the +// registry the build already talks to. +const uvImage = "ghcr.io/astral-sh/uv:" + uvVersion + // Service is the FastAPI specialization. It embeds the generic Python // Service so methods defined on *pythonservice.Service (and transitively // *services.Base: Wool, Logger, Location, Identity, …) are promoted. diff --git a/templates/builder/Dockerfile.tmpl b/templates/builder/Dockerfile.tmpl index c087a80..9c5ffb1 100644 --- a/templates/builder/Dockerfile.tmpl +++ b/templates/builder/Dockerfile.tmpl @@ -1,12 +1,29 @@ # The builder image — installs the venv via uv, then we copy it into the runtime. FROM {{.Builder}} as builder -WORKDIR /app - -COPY code/pyproject.toml code/uv.lock ./ - -# Install deps to a project-local .venv. --frozen requires uv.lock to match -# pyproject.toml; this catches drift at build time rather than at runtime. +# uv is part of what the recipe has to state. A base image carries whichever uv +# it happened to be built with, so a project written against a newer `[tool.uv]` +# key resolves on the developer's machine and fails inside the builder stage +# with a TOML parse error naming a field that uv there does not know. Copying a +# pinned uv in makes the tool that reads pyproject.toml and uv.lock a declared +# input of the recipe instead of a property of the base image. +COPY --from={{.UVImage}} /uv /usr/local/bin/uv + +WORKDIR {{.ProjectWorkdir}} + +COPY {{.ServicePrefix}}code/pyproject.toml {{.ServicePrefix}}code/uv.lock ./ +{{- if .Carried}} + +# A `[tool.uv.sources]` path leaving the project, placed at the same +# repository-relative path the developer resolves it from, so the directive and +# the lockfile entry that record it resolve here to what they resolve to there. +{{- range .Carried}} +COPY {{.Context}} {{.Image}} +{{- end}} +{{- end}} + +# Install deps to a project-local .venv. --frozen installs the locked resolution +# as it stands instead of re-resolving it at build time. ENV UV_PROJECT_ENVIRONMENT=/app/.venv \ UV_LINK_MODE=copy RUN uv sync --frozen --no-dev --no-install-project @@ -27,10 +44,10 @@ ENV VIRTUAL_ENV=/app/.venv \ COPY --from=builder ${VIRTUAL_ENV} ${VIRTUAL_ENV} -COPY --chown=appuser service.codefly.yaml . +COPY --chown=appuser {{.ServicePrefix}}service.codefly.yaml . {{ range .Components}} -COPY --chown=appuser {{.}} {{.}} +COPY --chown=appuser {{$.ServicePrefix}}{{.}} {{.}} {{end}} {{ range .Envs}} diff --git a/uv_sources_recipe_test.go b/uv_sources_recipe_test.go new file mode 100644 index 0000000..f279fbf --- /dev/null +++ b/uv_sources_recipe_test.go @@ -0,0 +1,299 @@ +package main + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/codefly-dev/core/agents/services" + builderv0 "github.com/codefly-dev/core/generated/go/codefly/services/builder/v0" + "github.com/stretchr/testify/require" +) + +// writeFile writes content at path, creating parents. +func writeFile(t *testing.T, path, content string) { + t.Helper() + require.NoError(t, os.MkdirAll(filepath.Dir(path), 0o755)) + require.NoError(t, os.WriteFile(path, []byte(content), 0o644)) +} + +// siblingLibrary writes a minimal installable project at directory — the shape +// a second codefly service's Python project has when a first one depends on it +// by path. +func siblingLibrary(t *testing.T, directory string) { + t.Helper() + writeFile(t, filepath.Join(directory, "pyproject.toml"), + "[project]\nname = \"example-lib\"\nversion = \"0.1.0\"\nrequires-python = \">=3.10\"\ndependencies = []\n") + writeFile(t, filepath.Join(directory, "src", "__init__.py"), "NAME = \"example-lib\"\n") +} + +// pathSourceProject replaces the scaffold's project with one whose only +// dependency is resolved from declared, so the recipe's behaviour is what the +// test exercises and not a dependency download. +func pathSourceProject(t *testing.T, builder *Builder, declared string) { + t.Helper() + writeFile(t, filepath.Join(builder.Location, "code", "pyproject.toml"), fmt.Sprintf(`[project] +name = "example-worker" +version = "0.1.0" +requires-python = ">=3.10" +dependencies = ["example-lib"] + +[tool.uv] +package = false + +[tool.uv.sources] +example-lib = { path = %q } +`, declared)) +} + +// escapingSourceBuilder is a created service whose project depends on a library +// in a sibling service of the same repository — the layout the default context, +// which is the service directory and nothing beside it, cannot build. It +// reports whether the project could be locked, which needs uv on the host. +func escapingSourceBuilder(t *testing.T) (*Builder, bool) { + t.Helper() + builder, _ := createdBuilder(t) + siblingLibrary(t, filepath.Join(filepath.Dir(builder.Location), "lib", "code")) + pathSourceProject(t, builder, "../../lib/code") + if _, err := exec.LookPath("uv"); err != nil { + return builder, false + } + lock := exec.Command("uv", "lock") + lock.Dir = filepath.Join(builder.Location, "code") + output, err := lock.CombinedOutput() + // uv is installed, so a failure here is a broken fixture, not an + // unequipped host. + require.NoError(t, err, "uv lock: %s", output) + return builder, true +} + +// emitRecipe runs Build into a fresh directory and returns the plan and that +// directory. +func emitRecipe(t *testing.T, builder *Builder, outputDir string) *builderv0.DockerBuildPlan { + t.Helper() + resp, err := builder.Build(t.Context(), recipeBuildRequest(outputDir)) + require.NoError(t, err) + require.Equal(t, builderv0.BuildStatus_SUCCESS, resp.GetState().GetState(), resp.GetState().GetMessage()) + plan := resp.GetResult().GetDockerBuildPlan() + require.NotNil(t, plan) + return plan +} + +// dockerfileOf reads the Dockerfile a plan's single recipe names. +func dockerfileOf(t *testing.T, outputDir string, plan *builderv0.DockerBuildPlan) string { + t.Helper() + require.Len(t, plan.GetRecipes(), 1) + content, err := os.ReadFile(filepath.Join(outputDir, plan.GetRecipes()[0].GetDockerfile())) + require.NoError(t, err) + return string(content) +} + +// TestBuildRecipeCarriesEscapingUVPathSource is the defect this recipe had: a +// project whose `[tool.uv.sources]` path leaves its own directory emitted a +// context without the directory it names, and the builder stage failed on +// `uv sync --frozen`, while the identical resolution succeeded on the +// developer's machine. +// +// It proves both halves against the real tool: the context as the old recipe +// built it — the service directory alone — does not resolve the path source, +// and the context this recipe assembles does, in a location where the original +// relative path leads nowhere. +func TestBuildRecipeCarriesEscapingUVPathSource(t *testing.T) { + builder, locked := escapingSourceBuilder(t) + outputDir := t.TempDir() + plan := emitRecipe(t, builder, outputDir) + + // The whole destination is this build's output, so the plan claims all of + // it and the CLI roots the docker context there instead of at the service. + require.Equal(t, builderv0.RecipeInventoryScope_RECIPE_INVENTORY_SCOPE_TREE, plan.GetScope()) + require.NoError(t, services.VerifyDockerBuildPlan(outputDir, plan)) + + service, err := filepath.Rel(repositoryOf(t, builder.Location), evalSymlinks(t, builder.Location)) + require.NoError(t, err) + library := filepath.Join(filepath.Dir(service), "lib", "code") + for _, rel := range []string{ + filepath.Join("context", service, "code", "pyproject.toml"), + filepath.Join("context", service, "service.codefly.yaml"), + filepath.Join("context", library, "pyproject.toml"), + filepath.Join("context", library, "src", "__init__.py"), + } { + require.FileExists(t, filepath.Join(outputDir, rel)) + } + + dockerfile := dockerfileOf(t, outputDir, plan) + slashedService := filepath.ToSlash(service) + slashedLibrary := filepath.ToSlash(library) + require.Contains(t, dockerfile, "WORKDIR /src/"+slashedService+"/code") + require.Contains(t, dockerfile, + fmt.Sprintf("COPY context/%s/code/pyproject.toml context/%s/code/uv.lock ./", slashedService, slashedService)) + require.Contains(t, dockerfile, + fmt.Sprintf("COPY context/%s /src/%s", slashedLibrary, slashedLibrary)) + require.Contains(t, dockerfile, + fmt.Sprintf("COPY --chown=appuser context/%s/service.codefly.yaml .", slashedService)) + require.Contains(t, dockerfile, + fmt.Sprintf("COPY --chown=appuser context/%s/code/src code/src", slashedService)) + + if !locked { + t.Skip("uv is not installed: the emitted context cannot be resolved against the real tool") + } + require.FileExists(t, filepath.Join(outputDir, "context", service, "code", "uv.lock")) + + // The old recipe, reconstructed: the service directory alone, in a place + // where the path source's original relative path leads nowhere. + old := filepath.Join(t.TempDir(), "app") + copyTree(t, filepath.Join(outputDir, "context", service, "code"), old) + output, err := runUVSync(t, old) + require.Error(t, err, "the old context must not resolve its path source, got:\n%s", output) + require.Contains(t, strings.ToLower(output), "lib/code", + "expected the path-resolution failure this fix is for, got:\n%s", output) + + // The emitted context, placed the way the builder stage places it. + source := filepath.Join(t.TempDir(), "src") + copyTree(t, filepath.Join(outputDir, "context"), source) + output, err = runUVSync(t, filepath.Join(source, service, "code")) + require.NoError(t, err, "uv sync in the emitted context:\n%s", output) +} + +// TestBuildRecipeWithoutEscapingPathSourceIsUnchanged holds the no-op case: a +// service whose project stays inside its own directory emits exactly the recipe +// it emitted before — only the Dockerfile and its ignore, claimed as such, with +// the context left to the caller's service directory. +func TestBuildRecipeWithoutEscapingPathSourceIsUnchanged(t *testing.T) { + builder, _ := createdBuilder(t) + outputDir := t.TempDir() + plan := emitRecipe(t, builder, outputDir) + + require.Equal(t, builderv0.RecipeInventoryScope_RECIPE_INVENTORY_SCOPE_EMITTED, plan.GetScope()) + require.NoDirExists(t, filepath.Join(outputDir, "context")) + + dockerfile := dockerfileOf(t, outputDir, plan) + require.Contains(t, dockerfile, "WORKDIR /app") + require.Contains(t, dockerfile, "COPY code/pyproject.toml code/uv.lock ./") + require.Contains(t, dockerfile, "COPY --chown=appuser service.codefly.yaml .") + require.Contains(t, dockerfile, "COPY --chown=appuser code/src code/src") + require.NotContains(t, dockerfile, "context/") +} + +// TestBuildRecipeCarryIsIdempotentInAReusedDirectory holds the layout the CLI +// hands the agent: one recipe directory per service, reused from build to +// build. A tree an earlier emission carried must not outlive the declaration +// that asked for it, and the second emission must not trip over its own output. +func TestBuildRecipeCarryIsIdempotentInAReusedDirectory(t *testing.T) { + builder, _ := escapingSourceBuilder(t) + outputDir := t.TempDir() + stale := filepath.Join(outputDir, "context", "gone", "leftover.py") + + for attempt := 1; attempt <= 2; attempt++ { + plan := emitRecipe(t, builder, outputDir) + require.Equal(t, builderv0.RecipeInventoryScope_RECIPE_INVENTORY_SCOPE_TREE, plan.GetScope(), + "emission %d", attempt) + require.NoError(t, services.VerifyDockerBuildPlan(outputDir, plan), "emission %d", attempt) + if attempt == 1 { + writeFile(t, stale, "# carried for a declaration that no longer exists\n") + } + } + require.NoFileExists(t, stale, "a tree no declaration asks for survived re-emission") +} + +// TestBuildRecipeRefusesPathSourceOutsideRepository proves a path source that +// leaves the repository is reported, naming the declaration. Copying it would +// put one machine's filesystem into an image; skipping it silently would build +// the image against a different dependency than the developer builds against. +func TestBuildRecipeRefusesPathSourceOutsideRepository(t *testing.T) { + outside := filepath.Join(t.TempDir(), "lib", "code") + siblingLibrary(t, outside) + builder, _ := createdBuilder(t) + pathSourceProject(t, builder, outside) + + resp, err := builder.Build(t.Context(), recipeBuildRequest(t.TempDir())) + require.NoError(t, err) + require.Equal(t, builderv0.BuildStatus_ERROR, resp.GetState().GetState()) + require.Contains(t, resp.GetState().GetMessage(), "[tool.uv.sources] example-lib") + require.Contains(t, resp.GetState().GetMessage(), "outside the repository") +} + +// TestBuildRecipeReportsAMissingPathSource proves a declaration naming a +// directory that is not there is reported as such, rather than emitting a +// recipe that fails inside the builder stage. +func TestBuildRecipeReportsAMissingPathSource(t *testing.T) { + builder, _ := createdBuilder(t) + pathSourceProject(t, builder, "../../absent/code") + + resp, err := builder.Build(t.Context(), recipeBuildRequest(t.TempDir())) + require.NoError(t, err) + require.Equal(t, builderv0.BuildStatus_ERROR, resp.GetState().GetState()) + require.Contains(t, resp.GetState().GetMessage(), "does not exist") +} + +// runUVSync runs the builder stage's install command in directory, with its own +// environment so the test never writes into a project's own venv. +func runUVSync(t *testing.T, directory string) (string, error) { + t.Helper() + command := exec.Command("uv", "sync", "--frozen", "--no-dev", "--no-install-project") + command.Dir = directory + command.Env = append(os.Environ(), "UV_PROJECT_ENVIRONMENT="+t.TempDir()) + output, err := command.CombinedOutput() + return string(output), err +} + +// copyTree copies src to dst the way the Dockerfile's COPY does. +func copyTree(t *testing.T, src, dst string) { + t.Helper() + require.NoError(t, os.MkdirAll(filepath.Dir(dst), 0o755)) + output, err := exec.Command("cp", "-R", src, dst).CombinedOutput() + require.NoError(t, err, "copy %s to %s: %s", src, dst, output) +} + +func evalSymlinks(t *testing.T, path string) string { + t.Helper() + resolved, err := filepath.EvalSymlinks(path) + require.NoError(t, err) + return resolved +} + +func repositoryOf(t *testing.T, path string) string { + t.Helper() + root, err := repositoryRoot(evalSymlinks(t, path)) + require.NoError(t, err) + return root +} + +// TestAssembledContextBuildsTheImage is the end-to-end half: the context this +// recipe assembles is built by docker from the root the plan's TREE scope tells +// the caller to build, and the image that comes out carries the dependency the +// escaping path source names. A Dockerfile whose COPY paths were right on paper +// and wrong against the assembled tree fails here and nowhere else. +func TestAssembledContextBuildsTheImage(t *testing.T) { + for _, tool := range []string{"docker", "uv"} { + if _, err := exec.LookPath(tool); err != nil { + t.Skipf("%s is not installed: the assembled context cannot be built", tool) + } + } + builder, locked := escapingSourceBuilder(t) + require.True(t, locked) + outputDir := t.TempDir() + plan := emitRecipe(t, builder, outputDir) + require.Equal(t, builderv0.RecipeInventoryScope_RECIPE_INVENTORY_SCOPE_TREE, plan.GetScope()) + + tag := fmt.Sprintf("codefly-fastapi-uv-context-test:%d", time.Now().UnixMilli()) + // The context root a TREE plan declares is the assembled output directory, + // not the service directory — which is what the caller resolves from the + // scope, and what this build has to be exercised against. + build := exec.CommandContext(t.Context(), "docker", "build", + "-f", filepath.Join(outputDir, plan.GetRecipes()[0].GetDockerfile()), + "-t", tag, outputDir) + if out, err := build.CombinedOutput(); err != nil { + t.Fatalf("the assembled context did not build: %v\n%s", err, out) + } + t.Cleanup(func() { _ = exec.Command("docker", "rmi", "-f", tag).Run() }) + + installed, err := exec.CommandContext(t.Context(), "docker", "run", "--rm", tag, + "python", "-c", "import importlib.metadata as m; print(m.version('example-lib'))").CombinedOutput() + require.NoError(t, err, "%s", installed) + require.Contains(t, string(installed), "0.1.0", + "the image does not carry the dependency the path source names") +} diff --git a/uv_toolchain_recipe_test.go b/uv_toolchain_recipe_test.go new file mode 100644 index 0000000..d5b531a --- /dev/null +++ b/uv_toolchain_recipe_test.go @@ -0,0 +1,109 @@ +package main + +import ( + "context" + "fmt" + "os/exec" + "path/filepath" + "regexp" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +// buildConstraintProject is a project declaring a `[tool.uv]` key that only a +// current uv knows. It is the shape that resolved on the developer's machine +// and failed inside the builder stage, reported there as +// +// TOML parse error at line N, column 1 +// unknown field `build-constraint-dependencies` +// +// because the uv the builder stage ran was the one the base image was built +// with, not the one the project was written against. +const buildConstraintProject = `[project] +name = "example-worker" +version = "0.1.0" +requires-python = ">=3.10" +dependencies = [] + +[tool.uv] +package = false +build-constraint-dependencies = ["setuptools==80.9.0"] +` + +// TestRecipePinsTheUVItResolvesWith holds the toolchain half of the recipe +// contract: the version of the tool that reads pyproject.toml and uv.lock is +// stated by the recipe, at an exact version, and copied in before anything +// resolves — not inherited from whichever uv the base image happens to carry. +func TestRecipePinsTheUVItResolvesWith(t *testing.T) { + require.Regexp(t, regexp.MustCompile(`^\d+\.\d+\.\d+$`), uvVersion, + "uv must be pinned to an exact version; a range or a moving tag is not a reproducible recipe") + require.Equal(t, "ghcr.io/astral-sh/uv:"+uvVersion, uvImage) + + builder, _ := createdBuilder(t) + outputDir := t.TempDir() + plan := emitRecipe(t, builder, outputDir) + dockerfile := dockerfileOf(t, outputDir, plan) + + copied := fmt.Sprintf("COPY --from=%s /uv /usr/local/bin/uv", uvImage) + require.Contains(t, dockerfile, copied) + require.Less(t, strings.Index(dockerfile, copied), strings.Index(dockerfile, "uv sync --frozen"), + "the pinned uv must be in place before the recipe resolves anything with it") +} + +// TestRecipeBuilderStageResolvesCurrentUVKeys is the behavioural half, and the +// regression for the failure this pin exists for: a project declaring a +// `[tool.uv]` key introduced after the base image was built resolves in the +// builder stage the recipe emits, and the uv that resolved it is the pinned +// one. +// +// It also records what the base image's own uv makes of the same project, so +// the evidence for the pin is in the run rather than in a claim about it. +func TestRecipeBuilderStageResolvesCurrentUVKeys(t *testing.T) { + for _, tool := range []string{"docker", "uv"} { + if _, err := exec.LookPath(tool); err != nil { + t.Skipf("%s is not installed: the builder stage cannot be built", tool) + } + } + + builder, _ := createdBuilder(t) + code := filepath.Join(builder.Location, "code") + writeFile(t, filepath.Join(code, "pyproject.toml"), buildConstraintProject) + + ctx, cancel := context.WithTimeout(t.Context(), 20*time.Minute) + defer cancel() + + lock := exec.CommandContext(ctx, "uv", "lock") + lock.Dir = code + output, err := lock.CombinedOutput() + require.NoError(t, err, "uv lock: %s", output) + + // What the base image's uv makes of the same project, named rather than + // assumed. This is evidence, not a gate: a newer base image knowing the key + // is an improvement, never a regression of the pin. + base := exec.CommandContext(ctx, "docker", "run", "--rm", "-v", code+":/probe", "-w", "/probe", + runtimeImage.FullName(), "sh", "-c", "uv --version; uv sync --frozen --no-dev --no-install-project 2>&1 | head -20") + probe, _ := base.CombinedOutput() + t.Logf("base image %s resolving the same project:\n%s", runtimeImage.FullName(), probe) + + outputDir := t.TempDir() + plan := emitRecipe(t, builder, outputDir) + + tag := fmt.Sprintf("codefly-fastapi-uv-pin-test:%d", time.Now().UnixMilli()) + build := exec.CommandContext(ctx, "docker", "build", "--target", "builder", + "-f", filepath.Join(outputDir, plan.GetRecipes()[0].GetDockerfile()), + "-t", tag, builder.Location) + if out, buildErr := build.CombinedOutput(); buildErr != nil { + // docker is installed and the project resolves on this host, so this is + // the builder stage refusing the project — the failure the pin is for. + t.Fatalf("the builder stage did not resolve a project using current uv keys: %v\n%s", buildErr, out) + } + t.Cleanup(func() { _ = exec.Command("docker", "rmi", "-f", tag).Run() }) + + version, err := exec.CommandContext(ctx, "docker", "run", "--rm", tag, "uv", "--version").CombinedOutput() + require.NoError(t, err, "%s", version) + require.Contains(t, string(version), uvVersion, + "the builder stage resolved with a uv the recipe does not name") +} diff --git a/uvsources.go b/uvsources.go new file mode 100644 index 0000000..74b180d --- /dev/null +++ b/uvsources.go @@ -0,0 +1,387 @@ +package main + +import ( + "errors" + "fmt" + "io" + "os" + "path/filepath" + "sort" + + "github.com/pelletier/go-toml/v2" +) + +// The recipe emitted for a service whose pyproject declares a +// `[tool.uv.sources]` path outside its own directory has to assemble its own +// build context. uv resolves such a path against the directory of the +// pyproject.toml that declares it, and it records the same relative path in +// uv.lock. The default context is the service directory and nothing beside it, +// so a path leaving the service is absent from the builder stage and +// `uv sync --frozen` fails there with +// +// error: Could not compute absolute path from workspace root and lockfile path +// +// while the identical resolution succeeds on the developer's machine. Two +// services sharing one Python project in a repository is a layout this recipe +// has to build; it is not something a consumer should restructure around. +// +// The context is assembled so that every relative path resolves in the builder +// stage exactly as it resolves on the host: the service's own tree and each +// escaping target are placed at their repository-relative paths under one +// root, and the builder stage's working directory is the project's own +// repository-relative path under that root. Nothing in pyproject.toml or +// uv.lock is rewritten — a rewrite would have to keep two files agreeing about +// a path uv reads from both, and a lockfile is a resolver's output, not an +// input a recipe gets to edit. +// +// Containment is the rule: an escaping path must resolve inside the repository +// that owns the service. A path pointing outside it names a directory true on +// one machine only, and is reported as an error naming the declaration rather +// than skipped, because a silently dropped source would build the image against +// a different dependency than the developer builds against. + +const ( + // assembledContextDirectory holds every tree the recipe assembles, under + // the caller-owned output directory. It is output, not source: each + // emission replaces it wholesale, so a tree carried for a declaration that + // has since been removed cannot outlive it. + assembledContextDirectory = "context" + + // builderSourceRoot is where the assembled context is placed in the builder + // stage. Mirroring the repository-relative layout under it is what makes a + // relative path resolve to the same directory it resolves to on the host, + // with no depth left to chance — resolving through the filesystem root + // would clamp two different paths onto one directory. + builderSourceRoot = "/src" + + // defaultProjectWorkdir is the builder-stage working directory when no + // context is assembled: the recipe's context is the service directory and + // the project is copied straight to it, exactly as before. + defaultProjectWorkdir = "/app" +) + +// repositoryMarkers name the root of the repository that owns a service. A +// codefly module ships as one package from one repository, so its manifest +// marks the same boundary a checkout does — and it still marks it after the CLI +// has resolved the module into its cache, where there is no checkout metadata. +var repositoryMarkers = []string{".git", "module.codefly.yaml", "workspace.codefly.yaml"} + +// ephemeralSourceDirectories are Python build outputs and tool caches. They are +// never inputs to an image build, and copying them would make the recipe's +// digest a function of whatever the developer last ran locally — a virtual +// environment also carries absolute paths and symlinks the recipe inventory +// rejects outright. +var ephemeralSourceDirectories = map[string]bool{ + ".venv": true, + "venv": true, + "__pycache__": true, + ".pytest_cache": true, + ".mypy_cache": true, + ".ruff_cache": true, + ".tox": true, + ".git": true, + ".codefly": true, + "node_modules": true, +} + +// CarriedSource is one COPY the builder stage performs for a tree the recipe +// carried: Context is the path inside the emitted build context, Image the +// absolute path the builder stage places it at. +type CarriedSource struct { + Context string + Image string +} + +// assembledContext describes a build context this recipe assembled, and is nil +// when the service declares no path source that leaves its own directory. +type assembledContext struct { + // ServicePrefix is the context-relative directory holding the service's own + // tree, with a trailing separator. It is empty when no context is + // assembled, which renders the recipe's COPY paths exactly as before. + ServicePrefix string + // ProjectWorkdir is the builder stage's working directory for the project. + ProjectWorkdir string + // Carried are the trees the builder stage must place beside the project. + Carried []CarriedSource +} + +// pathSource is one `[tool.uv.sources]` entry that names a directory. +type pathSource struct { + name string + declared string + resolved string +} + +// declaration renders a path source the way pyproject.toml spells it, so an +// error names the line the reader has to change. +func (p pathSource) declaration() string { + return fmt.Sprintf("pyproject.toml `[tool.uv.sources] %s = { path = %q }`", p.name, p.declared) +} + +// assembleUVContext makes the recipe's build context self-contained for a +// service whose project declares a `[tool.uv.sources]` path outside its own +// directory. +// +// serviceDir is the service root (the directory holding service.codefly.yaml), +// sourceDir the Python project inside it, and outputDir the caller-owned recipe +// directory the assembled tree is written to. It returns nil when nothing +// escapes: the recipe then emits only its Dockerfile and dockerignore, claims +// only those, and builds the service directory exactly as before. +func assembleUVContext(serviceDir, sourceDir, outputDir string) (*assembledContext, error) { + escaping, err := escapingPathSources(sourceDir) + if err != nil { + return nil, err + } + if len(escaping) == 0 { + return nil, nil + } + + serviceRoot, err := filepath.EvalSymlinks(serviceDir) + if err != nil { + return nil, err + } + repository, err := repositoryRoot(serviceRoot) + if err != nil { + return nil, fmt.Errorf("%w: %s", err, escaping[0].declaration()) + } + serviceRelative, err := filepath.Rel(repository, serviceRoot) + if err != nil { + return nil, err + } + + // The recipe directory is reused from build to build and the copy only ever + // adds to it, so a tree an earlier emission left behind would outlive the + // declaration that asked for it. + root := filepath.Join(outputDir, assembledContextDirectory) + if err = os.RemoveAll(root); err != nil { + return nil, err + } + serviceContext := filepath.Join(root, serviceRelative) + if err = copySourceTree(serviceRoot, serviceContext, outputDir); err != nil { + return nil, err + } + + context := &assembledContext{ + ServicePrefix: assembledContextDirectory + "/" + filepath.ToSlash(serviceRelative) + "/", + ProjectWorkdir: builderSourceRoot + "/" + filepath.ToSlash(filepath.Join(serviceRelative, filepath.Base(sourceDir))), + } + + placed := map[string]bool{} + for _, source := range escaping { + if within(source.resolved, serviceRoot) { + return nil, fmt.Errorf("%s resolves to %s, which contains the service itself: the image build context cannot include it", + source.declaration(), source.resolved) + } + if !within(repository, source.resolved) { + return nil, fmt.Errorf("%s resolves to %s, outside the repository %s that owns the service: the image build context cannot include it", + source.declaration(), source.resolved, repository) + } + relative, relErr := filepath.Rel(repository, source.resolved) + if relErr != nil { + return nil, relErr + } + slashed := filepath.ToSlash(relative) + if placed[slashed] { + // Two declarations may name one directory; carrying it once is enough. + continue + } + placed[slashed] = true + destination := filepath.Join(root, relative) + // A target inside the service was copied with it; one outside still has + // to be carried. Either way the builder stage places it explicitly, + // because it copies the project's manifests rather than the whole tree. + if !within(serviceRoot, source.resolved) { + if err = copySourceTree(source.resolved, destination, outputDir); err != nil { + return nil, fmt.Errorf("carry %s into the build context: %w", source.declaration(), err) + } + } + context.Carried = append(context.Carried, CarriedSource{ + Context: assembledContextDirectory + "/" + slashed, + Image: builderSourceRoot + "/" + slashed, + }) + } + return context, nil +} + +// escapingPathSources reads the project's pyproject.toml and returns every +// `[tool.uv.sources]` entry naming a directory outside the project, sorted by +// dependency name so an emission is deterministic. +// +// Only `path` entries are resolved. A `[tool.uv.workspace]` member is relative +// to the project too, and one inside the service resolves in the assembled +// context because the whole service tree is placed there; a member naming a +// directory outside the service is not handled here. +func escapingPathSources(sourceDir string) ([]pathSource, error) { + manifest := filepath.Join(sourceDir, "pyproject.toml") + content, err := os.ReadFile(manifest) + if errors.Is(err, os.ErrNotExist) { + return nil, nil + } + if err != nil { + return nil, err + } + var project struct { + Tool struct { + UV struct { + Sources map[string]any `toml:"sources"` + } `toml:"uv"` + } `toml:"tool"` + } + if err = toml.Unmarshal(content, &project); err != nil { + return nil, fmt.Errorf("parse %s: %w", manifest, err) + } + + projectRoot, err := filepath.EvalSymlinks(sourceDir) + if err != nil { + return nil, err + } + var escaping []pathSource + for name, declared := range project.Tool.UV.Sources { + for _, entry := range sourceEntries(declared) { + declaredPath, ok := entry["path"].(string) + if !ok || declaredPath == "" { + // A source resolved from an index, a version control system or + // the workspace carries no directory of its own. + continue + } + target := declaredPath + if !filepath.IsAbs(target) { + target = filepath.Join(projectRoot, filepath.FromSlash(target)) + } + resolved, resolveErr := filepath.EvalSymlinks(target) + if resolveErr != nil { + return nil, fmt.Errorf("pyproject.toml `[tool.uv.sources] %s = { path = %q }` resolves to %s, which does not exist", + name, declaredPath, target) + } + if within(projectRoot, resolved) { + // Inside the project: the default context already carries it. + continue + } + escaping = append(escaping, pathSource{name: name, declared: declaredPath, resolved: resolved}) + } + } + sort.Slice(escaping, func(i, j int) bool { + if escaping[i].name != escaping[j].name { + return escaping[i].name < escaping[j].name + } + return escaping[i].declared < escaping[j].declared + }) + return escaping, nil +} + +// sourceEntries normalizes a `[tool.uv.sources]` value, which uv accepts either +// as one table or as a list of tables carrying environment markers. +func sourceEntries(declared any) []map[string]any { + switch value := declared.(type) { + case map[string]any: + return []map[string]any{value} + case []any: + var entries []map[string]any + for _, element := range value { + if entry, ok := element.(map[string]any); ok { + entries = append(entries, entry) + } + } + return entries + default: + return nil + } +} + +// repositoryRoot walks up from a service's directory to the repository that +// owns it. The walk stops at the first marker rather than the outermost one, so +// a module nested in a workspace of checkouts is bounded by its own checkout. +func repositoryRoot(serviceRoot string) (string, error) { + for directory := serviceRoot; ; { + for _, marker := range repositoryMarkers { + if _, err := os.Lstat(filepath.Join(directory, marker)); err == nil { + return directory, nil + } + } + parent := filepath.Dir(directory) + if parent == directory { + return "", fmt.Errorf("no repository owns the service at %s, so a path source cannot be contained", serviceRoot) + } + directory = parent + } +} + +// copySourceTree copies the tree at src into dst, preserving file modes. +// +// Symlinks and other irregular files are skipped: the recipe inventory rejects +// symlinks outright, so a copied symlink would fail plan generation. Build +// outputs and tool caches are skipped for the same reason a lockfile is not +// regenerated here — they are local state, not build inputs. +// +// output is the recipe directory the walk must never descend into: it sits +// inside the service directory in the committed layout, so the walk would +// otherwise copy the growing recipe into itself. Directory identity, not a path +// prefix, decides. +func copySourceTree(src, dst, output string) error { + outputInfo, err := os.Stat(output) + if err != nil { + return err + } + return filepath.WalkDir(src, func(p string, entry os.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + if entry.IsDir() { + info, infoErr := entry.Info() + if infoErr != nil { + return infoErr + } + if os.SameFile(info, outputInfo) { + return filepath.SkipDir + } + if p != src && ephemeralSourceDirectories[entry.Name()] { + return filepath.SkipDir + } + } + relative, relErr := filepath.Rel(src, p) + if relErr != nil { + return relErr + } + target := filepath.Join(dst, relative) + if entry.IsDir() { + return os.MkdirAll(target, 0o755) + } + if !entry.Type().IsRegular() { + return nil + } + return copyFile(p, target) + }) +} + +func copyFile(src, dst string) error { + info, err := os.Stat(src) + if err != nil { + return err + } + if err = os.MkdirAll(filepath.Dir(dst), 0o755); err != nil { + return err + } + in, err := os.Open(src) + if err != nil { + return err + } + defer func() { _ = in.Close() }() + out, err := os.OpenFile(dst, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, info.Mode().Perm()) + if err != nil { + return err + } + if _, err = io.Copy(out, in); err != nil { + _ = out.Close() + return err + } + return out.Close() +} + +// within reports whether path is root or sits under it. +func within(root, path string) bool { + relative, err := filepath.Rel(root, path) + if err != nil { + return false + } + return relative == "." || filepath.IsLocal(relative) +}