diff --git a/AGENTS.md b/AGENTS.md index 6bc61b1..de3b4b0 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -112,6 +112,7 @@ anything from it. | `main.go` | settings, agent advertisement, composition of the generic Python layer | | `runtime.go` | Init/Start/Stop/Destroy, uvicorn supervision, hot reload, the lock order | | `builder.go` | `Create` scaffolding, OpenAPI endpoint loading, Docker + kustomize recipes | +| `uvsources.go` | the assembled build context: `[tool.uv.sources]` paths leaving the project | | `probes.go`, `probe.py` | endpoint health translated to Kubernetes probes / a local probe | | `templates/` | **what ships** — embedded via `go:embed`, rendered on `Create` | | `base/` | a checked-in sample service, embedded nowhere | @@ -135,6 +136,15 @@ into `templates/factory/code/pyproject.toml.tmpl` yourself. - **Readiness is the endpoint's declared health check, never an open port.** Probe intents (readiness / startup / liveness) stay separate through `resources.PlanEndpointProbes`; collapsing them silently drops a predicate. +- **The build recipe owns its uv, and its build context.** The emitted + Dockerfile copies a pinned uv (`uvVersion` in `main.go`) rather than running + whichever one the base image was built with — a project written against a + newer `[tool.uv]` key otherwise resolves locally and fails in the builder + stage. And a `[tool.uv.sources]` path leaving the project makes the recipe + assemble its own context, mirroring the repository-relative layout so the + same relative path resolves there; that plan claims the whole tree + (`RECIPE_INVENTORY_SCOPE_TREE`), the Dockerfile-only one claims just what it + emitted. Never rewrite `pyproject.toml` or `uv.lock` to make a path resolve. - **The runtime image is pinned.** `docker-image` rejects a bare name and `:latest`. The default `codeflydev/python` companion is built by core and repinned on release — prefer it. diff --git a/builder.go b/builder.go index 5eb541c..f497566 100644 --- a/builder.go +++ b/builder.go @@ -167,6 +167,18 @@ type DockerTemplating struct { Components []string RuntimePackages []string Envs []Env + + // UVImage is the pinned uv distribution the builder stage copies its uv + // from, so the tool that reads pyproject.toml and uv.lock is a declared + // input of the recipe rather than whatever the base image was built with. + UVImage string + + // ServicePrefix, ProjectWorkdir and Carried describe the build context. + // They are empty / defaulted when the recipe builds the service directory + // itself, which renders the Dockerfile this agent has always rendered. + ServicePrefix string + ProjectWorkdir string + Carried []CarriedSource } func (s *Builder) BuildCapabilities(context.Context, *builderv0.BuildCapabilitiesRequest) (*builderv0.BuildCapabilitiesResponse, error) { @@ -183,19 +195,69 @@ func (s *Builder) Build(ctx context.Context, req *builderv0.BuildRequest) (*buil return s.Base.Builder.BuildError(err) } ctx = s.Wool.Inject(ctx) - docker := DockerTemplating{ - Builder: runtimeImage.FullName(), - Components: requirements.All(), - } outputDir := req.GetOutputDirectory() emitted, err := services.PrepareRecipeDestination(builderFS, outputDir) if err != nil { return s.Base.Builder.BuildError(err) } + // A `[tool.uv.sources]` path resolves against the project directory, and the + // default context is the service directory and nothing beside it: carry what + // leaves the project into an assembled context so the builder stage resolves + // what the host resolves. A service with no such path assembles nothing. + assembled, err := assembleUVContext(s.Location, s.SourceLocation, outputDir) + if err != nil { + return s.Base.Builder.BuildError(err) + } + docker := DockerTemplating{ + Builder: runtimeImage.FullName(), + Components: requirements.All(), + UVImage: uvImage, + ProjectWorkdir: defaultProjectWorkdir, + } + if assembled != nil { + docker.ServicePrefix = assembled.ServicePrefix + docker.ProjectWorkdir = assembled.ProjectWorkdir + docker.Carried = assembled.Carried + } if err := s.Base.Templates(ctx, docker, services.WithBuilder(builderFS).WithDestination("%s", outputDir)); err != nil { return s.Base.Builder.BuildError(err) } - return s.Base.Builder.SingleImageBuildResponse(req, s.DockerImage(dockerRequest).FullName(), emitted) + image := s.DockerImage(dockerRequest).FullName() + if assembled == nil { + // Only the Dockerfile and its ignore were written, and the application's + // inputs are the service's own tree: the plan claims what it emitted and + // the caller builds the service directory. + return s.Base.Builder.SingleImageBuildResponse(req, image, emitted) + } + // The whole destination was assembled by this build, so the plan claims all + // of it and the caller builds the assembled tree rather than the service + // directory — which is what RECIPE_INVENTORY_SCOPE_TREE declares. + plan, err := services.BuildDockerBuildPlan(outputDir, []*builderv0.DockerBuildRecipe{{ + Name: "app", + Dockerfile: "Dockerfile", + Context: ".", + Dockerignore: dockerignoreOf(emitted), + Image: image, + Platforms: services.RecipeBuildPlatforms(), + }}) + if err != nil { + return s.Base.Builder.BuildError(err) + } + s.Base.Builder.WithBuildPlan(plan) + return s.Base.Builder.BuildResponse() +} + +// dockerignoreOf names the ignore file only when this build rendered one. +// Probing the destination instead would adopt a stale file: the executor copies +// the referenced ignore to the path buildx discovers, so one left behind by an +// older template set would silently keep files out of the image. +func dockerignoreOf(emitted []string) string { + for _, name := range emitted { + if name == "dockerignore" { + return name + } + } + return "" } // SBOM serves both evidence scopes. Source inventory stays with the generic diff --git a/go.mod b/go.mod index d5e7046..aeeac3f 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.27.0 require ( github.com/codefly-dev/core v0.5.1 github.com/codefly-dev/service-python v0.0.66 + github.com/pelletier/go-toml/v2 v2.4.3 github.com/stretchr/testify v1.12.1 google.golang.org/grpc v1.83.2 gopkg.in/yaml.v3 v3.0.1 @@ -92,7 +93,6 @@ require ( github.com/oklog/ulid/v2 v2.1.2 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.1.1 // indirect - github.com/pelletier/go-toml/v2 v2.4.3 // indirect github.com/pjbgf/sha1cd v0.6.0 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/power-devops/perfstat v0.0.0-20260805114148-88456608a4f6 // indirect diff --git a/main.go b/main.go index a885442..bcf76e0 100644 --- a/main.go +++ b/main.go @@ -94,6 +94,26 @@ type Settings struct { // other codefly-built image. var runtimeImage = &resources.DockerImage{Name: "codeflydev/python", Tag: "0.0.1"} +// uvVersion is the uv the emitted build recipe resolves its lockfile with. +// +// The base image carries a uv of its own, installed when that image was built +// and stated nowhere a project or this agent can read. A build recipe is meant +// to be durable and reproducible, and a recipe whose resolver version is a +// property of its base image is neither: a project written against a newer +// `[tool.uv]` key resolves on the developer's machine and fails inside the +// builder stage with a TOML parse error naming a field the older uv does not +// know. Pinning it here is the same contract the Go recipe already keeps for +// its own toolchain, which states its Go and Alpine versions outright. +// +// Bump it deliberately, together with uvImage. +const uvVersion = "0.12.16" + +// uvImage is the distribution the builder stage copies that uv from. astral +// publishes each release as a multi-architecture image holding the static +// binary, so the copy needs no package manager and no network beyond the +// registry the build already talks to. +const uvImage = "ghcr.io/astral-sh/uv:" + uvVersion + // Service is the FastAPI specialization. It embeds the generic Python // Service so methods defined on *pythonservice.Service (and transitively // *services.Base: Wool, Logger, Location, Identity, …) are promoted. diff --git a/templates/builder/Dockerfile.tmpl b/templates/builder/Dockerfile.tmpl index c087a80..9c5ffb1 100644 --- a/templates/builder/Dockerfile.tmpl +++ b/templates/builder/Dockerfile.tmpl @@ -1,12 +1,29 @@ # The builder image — installs the venv via uv, then we copy it into the runtime. FROM {{.Builder}} as builder -WORKDIR /app - -COPY code/pyproject.toml code/uv.lock ./ - -# Install deps to a project-local .venv. --frozen requires uv.lock to match -# pyproject.toml; this catches drift at build time rather than at runtime. +# uv is part of what the recipe has to state. A base image carries whichever uv +# it happened to be built with, so a project written against a newer `[tool.uv]` +# key resolves on the developer's machine and fails inside the builder stage +# with a TOML parse error naming a field that uv there does not know. Copying a +# pinned uv in makes the tool that reads pyproject.toml and uv.lock a declared +# input of the recipe instead of a property of the base image. +COPY --from={{.UVImage}} /uv /usr/local/bin/uv + +WORKDIR {{.ProjectWorkdir}} + +COPY {{.ServicePrefix}}code/pyproject.toml {{.ServicePrefix}}code/uv.lock ./ +{{- if .Carried}} + +# A `[tool.uv.sources]` path leaving the project, placed at the same +# repository-relative path the developer resolves it from, so the directive and +# the lockfile entry that record it resolve here to what they resolve to there. +{{- range .Carried}} +COPY {{.Context}} {{.Image}} +{{- end}} +{{- end}} + +# Install deps to a project-local .venv. --frozen installs the locked resolution +# as it stands instead of re-resolving it at build time. ENV UV_PROJECT_ENVIRONMENT=/app/.venv \ UV_LINK_MODE=copy RUN uv sync --frozen --no-dev --no-install-project @@ -27,10 +44,10 @@ ENV VIRTUAL_ENV=/app/.venv \ COPY --from=builder ${VIRTUAL_ENV} ${VIRTUAL_ENV} -COPY --chown=appuser service.codefly.yaml . +COPY --chown=appuser {{.ServicePrefix}}service.codefly.yaml . {{ range .Components}} -COPY --chown=appuser {{.}} {{.}} +COPY --chown=appuser {{$.ServicePrefix}}{{.}} {{.}} {{end}} {{ range .Envs}} diff --git a/uv_sources_recipe_test.go b/uv_sources_recipe_test.go new file mode 100644 index 0000000..f279fbf --- /dev/null +++ b/uv_sources_recipe_test.go @@ -0,0 +1,299 @@ +package main + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/codefly-dev/core/agents/services" + builderv0 "github.com/codefly-dev/core/generated/go/codefly/services/builder/v0" + "github.com/stretchr/testify/require" +) + +// writeFile writes content at path, creating parents. +func writeFile(t *testing.T, path, content string) { + t.Helper() + require.NoError(t, os.MkdirAll(filepath.Dir(path), 0o755)) + require.NoError(t, os.WriteFile(path, []byte(content), 0o644)) +} + +// siblingLibrary writes a minimal installable project at directory — the shape +// a second codefly service's Python project has when a first one depends on it +// by path. +func siblingLibrary(t *testing.T, directory string) { + t.Helper() + writeFile(t, filepath.Join(directory, "pyproject.toml"), + "[project]\nname = \"example-lib\"\nversion = \"0.1.0\"\nrequires-python = \">=3.10\"\ndependencies = []\n") + writeFile(t, filepath.Join(directory, "src", "__init__.py"), "NAME = \"example-lib\"\n") +} + +// pathSourceProject replaces the scaffold's project with one whose only +// dependency is resolved from declared, so the recipe's behaviour is what the +// test exercises and not a dependency download. +func pathSourceProject(t *testing.T, builder *Builder, declared string) { + t.Helper() + writeFile(t, filepath.Join(builder.Location, "code", "pyproject.toml"), fmt.Sprintf(`[project] +name = "example-worker" +version = "0.1.0" +requires-python = ">=3.10" +dependencies = ["example-lib"] + +[tool.uv] +package = false + +[tool.uv.sources] +example-lib = { path = %q } +`, declared)) +} + +// escapingSourceBuilder is a created service whose project depends on a library +// in a sibling service of the same repository — the layout the default context, +// which is the service directory and nothing beside it, cannot build. It +// reports whether the project could be locked, which needs uv on the host. +func escapingSourceBuilder(t *testing.T) (*Builder, bool) { + t.Helper() + builder, _ := createdBuilder(t) + siblingLibrary(t, filepath.Join(filepath.Dir(builder.Location), "lib", "code")) + pathSourceProject(t, builder, "../../lib/code") + if _, err := exec.LookPath("uv"); err != nil { + return builder, false + } + lock := exec.Command("uv", "lock") + lock.Dir = filepath.Join(builder.Location, "code") + output, err := lock.CombinedOutput() + // uv is installed, so a failure here is a broken fixture, not an + // unequipped host. + require.NoError(t, err, "uv lock: %s", output) + return builder, true +} + +// emitRecipe runs Build into a fresh directory and returns the plan and that +// directory. +func emitRecipe(t *testing.T, builder *Builder, outputDir string) *builderv0.DockerBuildPlan { + t.Helper() + resp, err := builder.Build(t.Context(), recipeBuildRequest(outputDir)) + require.NoError(t, err) + require.Equal(t, builderv0.BuildStatus_SUCCESS, resp.GetState().GetState(), resp.GetState().GetMessage()) + plan := resp.GetResult().GetDockerBuildPlan() + require.NotNil(t, plan) + return plan +} + +// dockerfileOf reads the Dockerfile a plan's single recipe names. +func dockerfileOf(t *testing.T, outputDir string, plan *builderv0.DockerBuildPlan) string { + t.Helper() + require.Len(t, plan.GetRecipes(), 1) + content, err := os.ReadFile(filepath.Join(outputDir, plan.GetRecipes()[0].GetDockerfile())) + require.NoError(t, err) + return string(content) +} + +// TestBuildRecipeCarriesEscapingUVPathSource is the defect this recipe had: a +// project whose `[tool.uv.sources]` path leaves its own directory emitted a +// context without the directory it names, and the builder stage failed on +// `uv sync --frozen`, while the identical resolution succeeded on the +// developer's machine. +// +// It proves both halves against the real tool: the context as the old recipe +// built it — the service directory alone — does not resolve the path source, +// and the context this recipe assembles does, in a location where the original +// relative path leads nowhere. +func TestBuildRecipeCarriesEscapingUVPathSource(t *testing.T) { + builder, locked := escapingSourceBuilder(t) + outputDir := t.TempDir() + plan := emitRecipe(t, builder, outputDir) + + // The whole destination is this build's output, so the plan claims all of + // it and the CLI roots the docker context there instead of at the service. + require.Equal(t, builderv0.RecipeInventoryScope_RECIPE_INVENTORY_SCOPE_TREE, plan.GetScope()) + require.NoError(t, services.VerifyDockerBuildPlan(outputDir, plan)) + + service, err := filepath.Rel(repositoryOf(t, builder.Location), evalSymlinks(t, builder.Location)) + require.NoError(t, err) + library := filepath.Join(filepath.Dir(service), "lib", "code") + for _, rel := range []string{ + filepath.Join("context", service, "code", "pyproject.toml"), + filepath.Join("context", service, "service.codefly.yaml"), + filepath.Join("context", library, "pyproject.toml"), + filepath.Join("context", library, "src", "__init__.py"), + } { + require.FileExists(t, filepath.Join(outputDir, rel)) + } + + dockerfile := dockerfileOf(t, outputDir, plan) + slashedService := filepath.ToSlash(service) + slashedLibrary := filepath.ToSlash(library) + require.Contains(t, dockerfile, "WORKDIR /src/"+slashedService+"/code") + require.Contains(t, dockerfile, + fmt.Sprintf("COPY context/%s/code/pyproject.toml context/%s/code/uv.lock ./", slashedService, slashedService)) + require.Contains(t, dockerfile, + fmt.Sprintf("COPY context/%s /src/%s", slashedLibrary, slashedLibrary)) + require.Contains(t, dockerfile, + fmt.Sprintf("COPY --chown=appuser context/%s/service.codefly.yaml .", slashedService)) + require.Contains(t, dockerfile, + fmt.Sprintf("COPY --chown=appuser context/%s/code/src code/src", slashedService)) + + if !locked { + t.Skip("uv is not installed: the emitted context cannot be resolved against the real tool") + } + require.FileExists(t, filepath.Join(outputDir, "context", service, "code", "uv.lock")) + + // The old recipe, reconstructed: the service directory alone, in a place + // where the path source's original relative path leads nowhere. + old := filepath.Join(t.TempDir(), "app") + copyTree(t, filepath.Join(outputDir, "context", service, "code"), old) + output, err := runUVSync(t, old) + require.Error(t, err, "the old context must not resolve its path source, got:\n%s", output) + require.Contains(t, strings.ToLower(output), "lib/code", + "expected the path-resolution failure this fix is for, got:\n%s", output) + + // The emitted context, placed the way the builder stage places it. + source := filepath.Join(t.TempDir(), "src") + copyTree(t, filepath.Join(outputDir, "context"), source) + output, err = runUVSync(t, filepath.Join(source, service, "code")) + require.NoError(t, err, "uv sync in the emitted context:\n%s", output) +} + +// TestBuildRecipeWithoutEscapingPathSourceIsUnchanged holds the no-op case: a +// service whose project stays inside its own directory emits exactly the recipe +// it emitted before — only the Dockerfile and its ignore, claimed as such, with +// the context left to the caller's service directory. +func TestBuildRecipeWithoutEscapingPathSourceIsUnchanged(t *testing.T) { + builder, _ := createdBuilder(t) + outputDir := t.TempDir() + plan := emitRecipe(t, builder, outputDir) + + require.Equal(t, builderv0.RecipeInventoryScope_RECIPE_INVENTORY_SCOPE_EMITTED, plan.GetScope()) + require.NoDirExists(t, filepath.Join(outputDir, "context")) + + dockerfile := dockerfileOf(t, outputDir, plan) + require.Contains(t, dockerfile, "WORKDIR /app") + require.Contains(t, dockerfile, "COPY code/pyproject.toml code/uv.lock ./") + require.Contains(t, dockerfile, "COPY --chown=appuser service.codefly.yaml .") + require.Contains(t, dockerfile, "COPY --chown=appuser code/src code/src") + require.NotContains(t, dockerfile, "context/") +} + +// TestBuildRecipeCarryIsIdempotentInAReusedDirectory holds the layout the CLI +// hands the agent: one recipe directory per service, reused from build to +// build. A tree an earlier emission carried must not outlive the declaration +// that asked for it, and the second emission must not trip over its own output. +func TestBuildRecipeCarryIsIdempotentInAReusedDirectory(t *testing.T) { + builder, _ := escapingSourceBuilder(t) + outputDir := t.TempDir() + stale := filepath.Join(outputDir, "context", "gone", "leftover.py") + + for attempt := 1; attempt <= 2; attempt++ { + plan := emitRecipe(t, builder, outputDir) + require.Equal(t, builderv0.RecipeInventoryScope_RECIPE_INVENTORY_SCOPE_TREE, plan.GetScope(), + "emission %d", attempt) + require.NoError(t, services.VerifyDockerBuildPlan(outputDir, plan), "emission %d", attempt) + if attempt == 1 { + writeFile(t, stale, "# carried for a declaration that no longer exists\n") + } + } + require.NoFileExists(t, stale, "a tree no declaration asks for survived re-emission") +} + +// TestBuildRecipeRefusesPathSourceOutsideRepository proves a path source that +// leaves the repository is reported, naming the declaration. Copying it would +// put one machine's filesystem into an image; skipping it silently would build +// the image against a different dependency than the developer builds against. +func TestBuildRecipeRefusesPathSourceOutsideRepository(t *testing.T) { + outside := filepath.Join(t.TempDir(), "lib", "code") + siblingLibrary(t, outside) + builder, _ := createdBuilder(t) + pathSourceProject(t, builder, outside) + + resp, err := builder.Build(t.Context(), recipeBuildRequest(t.TempDir())) + require.NoError(t, err) + require.Equal(t, builderv0.BuildStatus_ERROR, resp.GetState().GetState()) + require.Contains(t, resp.GetState().GetMessage(), "[tool.uv.sources] example-lib") + require.Contains(t, resp.GetState().GetMessage(), "outside the repository") +} + +// TestBuildRecipeReportsAMissingPathSource proves a declaration naming a +// directory that is not there is reported as such, rather than emitting a +// recipe that fails inside the builder stage. +func TestBuildRecipeReportsAMissingPathSource(t *testing.T) { + builder, _ := createdBuilder(t) + pathSourceProject(t, builder, "../../absent/code") + + resp, err := builder.Build(t.Context(), recipeBuildRequest(t.TempDir())) + require.NoError(t, err) + require.Equal(t, builderv0.BuildStatus_ERROR, resp.GetState().GetState()) + require.Contains(t, resp.GetState().GetMessage(), "does not exist") +} + +// runUVSync runs the builder stage's install command in directory, with its own +// environment so the test never writes into a project's own venv. +func runUVSync(t *testing.T, directory string) (string, error) { + t.Helper() + command := exec.Command("uv", "sync", "--frozen", "--no-dev", "--no-install-project") + command.Dir = directory + command.Env = append(os.Environ(), "UV_PROJECT_ENVIRONMENT="+t.TempDir()) + output, err := command.CombinedOutput() + return string(output), err +} + +// copyTree copies src to dst the way the Dockerfile's COPY does. +func copyTree(t *testing.T, src, dst string) { + t.Helper() + require.NoError(t, os.MkdirAll(filepath.Dir(dst), 0o755)) + output, err := exec.Command("cp", "-R", src, dst).CombinedOutput() + require.NoError(t, err, "copy %s to %s: %s", src, dst, output) +} + +func evalSymlinks(t *testing.T, path string) string { + t.Helper() + resolved, err := filepath.EvalSymlinks(path) + require.NoError(t, err) + return resolved +} + +func repositoryOf(t *testing.T, path string) string { + t.Helper() + root, err := repositoryRoot(evalSymlinks(t, path)) + require.NoError(t, err) + return root +} + +// TestAssembledContextBuildsTheImage is the end-to-end half: the context this +// recipe assembles is built by docker from the root the plan's TREE scope tells +// the caller to build, and the image that comes out carries the dependency the +// escaping path source names. A Dockerfile whose COPY paths were right on paper +// and wrong against the assembled tree fails here and nowhere else. +func TestAssembledContextBuildsTheImage(t *testing.T) { + for _, tool := range []string{"docker", "uv"} { + if _, err := exec.LookPath(tool); err != nil { + t.Skipf("%s is not installed: the assembled context cannot be built", tool) + } + } + builder, locked := escapingSourceBuilder(t) + require.True(t, locked) + outputDir := t.TempDir() + plan := emitRecipe(t, builder, outputDir) + require.Equal(t, builderv0.RecipeInventoryScope_RECIPE_INVENTORY_SCOPE_TREE, plan.GetScope()) + + tag := fmt.Sprintf("codefly-fastapi-uv-context-test:%d", time.Now().UnixMilli()) + // The context root a TREE plan declares is the assembled output directory, + // not the service directory — which is what the caller resolves from the + // scope, and what this build has to be exercised against. + build := exec.CommandContext(t.Context(), "docker", "build", + "-f", filepath.Join(outputDir, plan.GetRecipes()[0].GetDockerfile()), + "-t", tag, outputDir) + if out, err := build.CombinedOutput(); err != nil { + t.Fatalf("the assembled context did not build: %v\n%s", err, out) + } + t.Cleanup(func() { _ = exec.Command("docker", "rmi", "-f", tag).Run() }) + + installed, err := exec.CommandContext(t.Context(), "docker", "run", "--rm", tag, + "python", "-c", "import importlib.metadata as m; print(m.version('example-lib'))").CombinedOutput() + require.NoError(t, err, "%s", installed) + require.Contains(t, string(installed), "0.1.0", + "the image does not carry the dependency the path source names") +} diff --git a/uv_toolchain_recipe_test.go b/uv_toolchain_recipe_test.go new file mode 100644 index 0000000..d5b531a --- /dev/null +++ b/uv_toolchain_recipe_test.go @@ -0,0 +1,109 @@ +package main + +import ( + "context" + "fmt" + "os/exec" + "path/filepath" + "regexp" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +// buildConstraintProject is a project declaring a `[tool.uv]` key that only a +// current uv knows. It is the shape that resolved on the developer's machine +// and failed inside the builder stage, reported there as +// +// TOML parse error at line N, column 1 +// unknown field `build-constraint-dependencies` +// +// because the uv the builder stage ran was the one the base image was built +// with, not the one the project was written against. +const buildConstraintProject = `[project] +name = "example-worker" +version = "0.1.0" +requires-python = ">=3.10" +dependencies = [] + +[tool.uv] +package = false +build-constraint-dependencies = ["setuptools==80.9.0"] +` + +// TestRecipePinsTheUVItResolvesWith holds the toolchain half of the recipe +// contract: the version of the tool that reads pyproject.toml and uv.lock is +// stated by the recipe, at an exact version, and copied in before anything +// resolves — not inherited from whichever uv the base image happens to carry. +func TestRecipePinsTheUVItResolvesWith(t *testing.T) { + require.Regexp(t, regexp.MustCompile(`^\d+\.\d+\.\d+$`), uvVersion, + "uv must be pinned to an exact version; a range or a moving tag is not a reproducible recipe") + require.Equal(t, "ghcr.io/astral-sh/uv:"+uvVersion, uvImage) + + builder, _ := createdBuilder(t) + outputDir := t.TempDir() + plan := emitRecipe(t, builder, outputDir) + dockerfile := dockerfileOf(t, outputDir, plan) + + copied := fmt.Sprintf("COPY --from=%s /uv /usr/local/bin/uv", uvImage) + require.Contains(t, dockerfile, copied) + require.Less(t, strings.Index(dockerfile, copied), strings.Index(dockerfile, "uv sync --frozen"), + "the pinned uv must be in place before the recipe resolves anything with it") +} + +// TestRecipeBuilderStageResolvesCurrentUVKeys is the behavioural half, and the +// regression for the failure this pin exists for: a project declaring a +// `[tool.uv]` key introduced after the base image was built resolves in the +// builder stage the recipe emits, and the uv that resolved it is the pinned +// one. +// +// It also records what the base image's own uv makes of the same project, so +// the evidence for the pin is in the run rather than in a claim about it. +func TestRecipeBuilderStageResolvesCurrentUVKeys(t *testing.T) { + for _, tool := range []string{"docker", "uv"} { + if _, err := exec.LookPath(tool); err != nil { + t.Skipf("%s is not installed: the builder stage cannot be built", tool) + } + } + + builder, _ := createdBuilder(t) + code := filepath.Join(builder.Location, "code") + writeFile(t, filepath.Join(code, "pyproject.toml"), buildConstraintProject) + + ctx, cancel := context.WithTimeout(t.Context(), 20*time.Minute) + defer cancel() + + lock := exec.CommandContext(ctx, "uv", "lock") + lock.Dir = code + output, err := lock.CombinedOutput() + require.NoError(t, err, "uv lock: %s", output) + + // What the base image's uv makes of the same project, named rather than + // assumed. This is evidence, not a gate: a newer base image knowing the key + // is an improvement, never a regression of the pin. + base := exec.CommandContext(ctx, "docker", "run", "--rm", "-v", code+":/probe", "-w", "/probe", + runtimeImage.FullName(), "sh", "-c", "uv --version; uv sync --frozen --no-dev --no-install-project 2>&1 | head -20") + probe, _ := base.CombinedOutput() + t.Logf("base image %s resolving the same project:\n%s", runtimeImage.FullName(), probe) + + outputDir := t.TempDir() + plan := emitRecipe(t, builder, outputDir) + + tag := fmt.Sprintf("codefly-fastapi-uv-pin-test:%d", time.Now().UnixMilli()) + build := exec.CommandContext(ctx, "docker", "build", "--target", "builder", + "-f", filepath.Join(outputDir, plan.GetRecipes()[0].GetDockerfile()), + "-t", tag, builder.Location) + if out, buildErr := build.CombinedOutput(); buildErr != nil { + // docker is installed and the project resolves on this host, so this is + // the builder stage refusing the project — the failure the pin is for. + t.Fatalf("the builder stage did not resolve a project using current uv keys: %v\n%s", buildErr, out) + } + t.Cleanup(func() { _ = exec.Command("docker", "rmi", "-f", tag).Run() }) + + version, err := exec.CommandContext(ctx, "docker", "run", "--rm", tag, "uv", "--version").CombinedOutput() + require.NoError(t, err, "%s", version) + require.Contains(t, string(version), uvVersion, + "the builder stage resolved with a uv the recipe does not name") +} diff --git a/uvsources.go b/uvsources.go new file mode 100644 index 0000000..74b180d --- /dev/null +++ b/uvsources.go @@ -0,0 +1,387 @@ +package main + +import ( + "errors" + "fmt" + "io" + "os" + "path/filepath" + "sort" + + "github.com/pelletier/go-toml/v2" +) + +// The recipe emitted for a service whose pyproject declares a +// `[tool.uv.sources]` path outside its own directory has to assemble its own +// build context. uv resolves such a path against the directory of the +// pyproject.toml that declares it, and it records the same relative path in +// uv.lock. The default context is the service directory and nothing beside it, +// so a path leaving the service is absent from the builder stage and +// `uv sync --frozen` fails there with +// +// error: Could not compute absolute path from workspace root and lockfile path +// +// while the identical resolution succeeds on the developer's machine. Two +// services sharing one Python project in a repository is a layout this recipe +// has to build; it is not something a consumer should restructure around. +// +// The context is assembled so that every relative path resolves in the builder +// stage exactly as it resolves on the host: the service's own tree and each +// escaping target are placed at their repository-relative paths under one +// root, and the builder stage's working directory is the project's own +// repository-relative path under that root. Nothing in pyproject.toml or +// uv.lock is rewritten — a rewrite would have to keep two files agreeing about +// a path uv reads from both, and a lockfile is a resolver's output, not an +// input a recipe gets to edit. +// +// Containment is the rule: an escaping path must resolve inside the repository +// that owns the service. A path pointing outside it names a directory true on +// one machine only, and is reported as an error naming the declaration rather +// than skipped, because a silently dropped source would build the image against +// a different dependency than the developer builds against. + +const ( + // assembledContextDirectory holds every tree the recipe assembles, under + // the caller-owned output directory. It is output, not source: each + // emission replaces it wholesale, so a tree carried for a declaration that + // has since been removed cannot outlive it. + assembledContextDirectory = "context" + + // builderSourceRoot is where the assembled context is placed in the builder + // stage. Mirroring the repository-relative layout under it is what makes a + // relative path resolve to the same directory it resolves to on the host, + // with no depth left to chance — resolving through the filesystem root + // would clamp two different paths onto one directory. + builderSourceRoot = "/src" + + // defaultProjectWorkdir is the builder-stage working directory when no + // context is assembled: the recipe's context is the service directory and + // the project is copied straight to it, exactly as before. + defaultProjectWorkdir = "/app" +) + +// repositoryMarkers name the root of the repository that owns a service. A +// codefly module ships as one package from one repository, so its manifest +// marks the same boundary a checkout does — and it still marks it after the CLI +// has resolved the module into its cache, where there is no checkout metadata. +var repositoryMarkers = []string{".git", "module.codefly.yaml", "workspace.codefly.yaml"} + +// ephemeralSourceDirectories are Python build outputs and tool caches. They are +// never inputs to an image build, and copying them would make the recipe's +// digest a function of whatever the developer last ran locally — a virtual +// environment also carries absolute paths and symlinks the recipe inventory +// rejects outright. +var ephemeralSourceDirectories = map[string]bool{ + ".venv": true, + "venv": true, + "__pycache__": true, + ".pytest_cache": true, + ".mypy_cache": true, + ".ruff_cache": true, + ".tox": true, + ".git": true, + ".codefly": true, + "node_modules": true, +} + +// CarriedSource is one COPY the builder stage performs for a tree the recipe +// carried: Context is the path inside the emitted build context, Image the +// absolute path the builder stage places it at. +type CarriedSource struct { + Context string + Image string +} + +// assembledContext describes a build context this recipe assembled, and is nil +// when the service declares no path source that leaves its own directory. +type assembledContext struct { + // ServicePrefix is the context-relative directory holding the service's own + // tree, with a trailing separator. It is empty when no context is + // assembled, which renders the recipe's COPY paths exactly as before. + ServicePrefix string + // ProjectWorkdir is the builder stage's working directory for the project. + ProjectWorkdir string + // Carried are the trees the builder stage must place beside the project. + Carried []CarriedSource +} + +// pathSource is one `[tool.uv.sources]` entry that names a directory. +type pathSource struct { + name string + declared string + resolved string +} + +// declaration renders a path source the way pyproject.toml spells it, so an +// error names the line the reader has to change. +func (p pathSource) declaration() string { + return fmt.Sprintf("pyproject.toml `[tool.uv.sources] %s = { path = %q }`", p.name, p.declared) +} + +// assembleUVContext makes the recipe's build context self-contained for a +// service whose project declares a `[tool.uv.sources]` path outside its own +// directory. +// +// serviceDir is the service root (the directory holding service.codefly.yaml), +// sourceDir the Python project inside it, and outputDir the caller-owned recipe +// directory the assembled tree is written to. It returns nil when nothing +// escapes: the recipe then emits only its Dockerfile and dockerignore, claims +// only those, and builds the service directory exactly as before. +func assembleUVContext(serviceDir, sourceDir, outputDir string) (*assembledContext, error) { + escaping, err := escapingPathSources(sourceDir) + if err != nil { + return nil, err + } + if len(escaping) == 0 { + return nil, nil + } + + serviceRoot, err := filepath.EvalSymlinks(serviceDir) + if err != nil { + return nil, err + } + repository, err := repositoryRoot(serviceRoot) + if err != nil { + return nil, fmt.Errorf("%w: %s", err, escaping[0].declaration()) + } + serviceRelative, err := filepath.Rel(repository, serviceRoot) + if err != nil { + return nil, err + } + + // The recipe directory is reused from build to build and the copy only ever + // adds to it, so a tree an earlier emission left behind would outlive the + // declaration that asked for it. + root := filepath.Join(outputDir, assembledContextDirectory) + if err = os.RemoveAll(root); err != nil { + return nil, err + } + serviceContext := filepath.Join(root, serviceRelative) + if err = copySourceTree(serviceRoot, serviceContext, outputDir); err != nil { + return nil, err + } + + context := &assembledContext{ + ServicePrefix: assembledContextDirectory + "/" + filepath.ToSlash(serviceRelative) + "/", + ProjectWorkdir: builderSourceRoot + "/" + filepath.ToSlash(filepath.Join(serviceRelative, filepath.Base(sourceDir))), + } + + placed := map[string]bool{} + for _, source := range escaping { + if within(source.resolved, serviceRoot) { + return nil, fmt.Errorf("%s resolves to %s, which contains the service itself: the image build context cannot include it", + source.declaration(), source.resolved) + } + if !within(repository, source.resolved) { + return nil, fmt.Errorf("%s resolves to %s, outside the repository %s that owns the service: the image build context cannot include it", + source.declaration(), source.resolved, repository) + } + relative, relErr := filepath.Rel(repository, source.resolved) + if relErr != nil { + return nil, relErr + } + slashed := filepath.ToSlash(relative) + if placed[slashed] { + // Two declarations may name one directory; carrying it once is enough. + continue + } + placed[slashed] = true + destination := filepath.Join(root, relative) + // A target inside the service was copied with it; one outside still has + // to be carried. Either way the builder stage places it explicitly, + // because it copies the project's manifests rather than the whole tree. + if !within(serviceRoot, source.resolved) { + if err = copySourceTree(source.resolved, destination, outputDir); err != nil { + return nil, fmt.Errorf("carry %s into the build context: %w", source.declaration(), err) + } + } + context.Carried = append(context.Carried, CarriedSource{ + Context: assembledContextDirectory + "/" + slashed, + Image: builderSourceRoot + "/" + slashed, + }) + } + return context, nil +} + +// escapingPathSources reads the project's pyproject.toml and returns every +// `[tool.uv.sources]` entry naming a directory outside the project, sorted by +// dependency name so an emission is deterministic. +// +// Only `path` entries are resolved. A `[tool.uv.workspace]` member is relative +// to the project too, and one inside the service resolves in the assembled +// context because the whole service tree is placed there; a member naming a +// directory outside the service is not handled here. +func escapingPathSources(sourceDir string) ([]pathSource, error) { + manifest := filepath.Join(sourceDir, "pyproject.toml") + content, err := os.ReadFile(manifest) + if errors.Is(err, os.ErrNotExist) { + return nil, nil + } + if err != nil { + return nil, err + } + var project struct { + Tool struct { + UV struct { + Sources map[string]any `toml:"sources"` + } `toml:"uv"` + } `toml:"tool"` + } + if err = toml.Unmarshal(content, &project); err != nil { + return nil, fmt.Errorf("parse %s: %w", manifest, err) + } + + projectRoot, err := filepath.EvalSymlinks(sourceDir) + if err != nil { + return nil, err + } + var escaping []pathSource + for name, declared := range project.Tool.UV.Sources { + for _, entry := range sourceEntries(declared) { + declaredPath, ok := entry["path"].(string) + if !ok || declaredPath == "" { + // A source resolved from an index, a version control system or + // the workspace carries no directory of its own. + continue + } + target := declaredPath + if !filepath.IsAbs(target) { + target = filepath.Join(projectRoot, filepath.FromSlash(target)) + } + resolved, resolveErr := filepath.EvalSymlinks(target) + if resolveErr != nil { + return nil, fmt.Errorf("pyproject.toml `[tool.uv.sources] %s = { path = %q }` resolves to %s, which does not exist", + name, declaredPath, target) + } + if within(projectRoot, resolved) { + // Inside the project: the default context already carries it. + continue + } + escaping = append(escaping, pathSource{name: name, declared: declaredPath, resolved: resolved}) + } + } + sort.Slice(escaping, func(i, j int) bool { + if escaping[i].name != escaping[j].name { + return escaping[i].name < escaping[j].name + } + return escaping[i].declared < escaping[j].declared + }) + return escaping, nil +} + +// sourceEntries normalizes a `[tool.uv.sources]` value, which uv accepts either +// as one table or as a list of tables carrying environment markers. +func sourceEntries(declared any) []map[string]any { + switch value := declared.(type) { + case map[string]any: + return []map[string]any{value} + case []any: + var entries []map[string]any + for _, element := range value { + if entry, ok := element.(map[string]any); ok { + entries = append(entries, entry) + } + } + return entries + default: + return nil + } +} + +// repositoryRoot walks up from a service's directory to the repository that +// owns it. The walk stops at the first marker rather than the outermost one, so +// a module nested in a workspace of checkouts is bounded by its own checkout. +func repositoryRoot(serviceRoot string) (string, error) { + for directory := serviceRoot; ; { + for _, marker := range repositoryMarkers { + if _, err := os.Lstat(filepath.Join(directory, marker)); err == nil { + return directory, nil + } + } + parent := filepath.Dir(directory) + if parent == directory { + return "", fmt.Errorf("no repository owns the service at %s, so a path source cannot be contained", serviceRoot) + } + directory = parent + } +} + +// copySourceTree copies the tree at src into dst, preserving file modes. +// +// Symlinks and other irregular files are skipped: the recipe inventory rejects +// symlinks outright, so a copied symlink would fail plan generation. Build +// outputs and tool caches are skipped for the same reason a lockfile is not +// regenerated here — they are local state, not build inputs. +// +// output is the recipe directory the walk must never descend into: it sits +// inside the service directory in the committed layout, so the walk would +// otherwise copy the growing recipe into itself. Directory identity, not a path +// prefix, decides. +func copySourceTree(src, dst, output string) error { + outputInfo, err := os.Stat(output) + if err != nil { + return err + } + return filepath.WalkDir(src, func(p string, entry os.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + if entry.IsDir() { + info, infoErr := entry.Info() + if infoErr != nil { + return infoErr + } + if os.SameFile(info, outputInfo) { + return filepath.SkipDir + } + if p != src && ephemeralSourceDirectories[entry.Name()] { + return filepath.SkipDir + } + } + relative, relErr := filepath.Rel(src, p) + if relErr != nil { + return relErr + } + target := filepath.Join(dst, relative) + if entry.IsDir() { + return os.MkdirAll(target, 0o755) + } + if !entry.Type().IsRegular() { + return nil + } + return copyFile(p, target) + }) +} + +func copyFile(src, dst string) error { + info, err := os.Stat(src) + if err != nil { + return err + } + if err = os.MkdirAll(filepath.Dir(dst), 0o755); err != nil { + return err + } + in, err := os.Open(src) + if err != nil { + return err + } + defer func() { _ = in.Close() }() + out, err := os.OpenFile(dst, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, info.Mode().Perm()) + if err != nil { + return err + } + if _, err = io.Copy(out, in); err != nil { + _ = out.Close() + return err + } + return out.Close() +} + +// within reports whether path is root or sits under it. +func within(root, path string) bool { + relative, err := filepath.Rel(root, path) + if err != nil { + return false + } + return relative == "." || filepath.IsLocal(relative) +}