From 0a70dffdcda50d3d82e5fb3c2228df796a40f80f Mon Sep 17 00:00:00 2001 From: Antoine Toussaint <312536060+antoinetoussaint-byte@users.noreply.github.com> Date: Sun, 27 Sep 2026 18:57:45 -0400 Subject: [PATCH] fix(build): build a uv workspace member project, with the interpreter it requires A service whose project (code/) is a member of a uv workspace keeps its one uv.lock, and its sibling members, at the workspace root in the service. `uv sync` from code/ resolves against that root on the host, which is what the native runner does, but the image recipe copied code/pyproject.toml and code/uv.lock only, so the build failed on a lock that is not in code/. A library whose members name each other `{ workspace = true }` has to be laid out this way: uv honours that source only under an ancestor root. The recipe now detects the enclosing workspace, copies it whole (ignoring virtual environments and caches), and syncs the project from its own directory. A root outside the service, a root without a lock, or a path source leaving the service is refused with the reason. `--no-editable` installs members and path sources into the venv as packages, since the runtime stage carries only the venv. A project whose requires-python the base image's Python does not satisfy gets the interpreter uv installs for it, as on the host; it is now installed where the runtime stage copies it from, so the venv's interpreter exists at runtime. Without that copy the venv's python dangled and the base interpreter ran without the project's packages. Co-Authored-By: Claude Opus 5.5 (1M context) --- builder.go | 39 +++++- templates/builder/Dockerfile.tmpl | 24 +++- templates/builder/dockerignore.tmpl | 3 + uv_workspace_recipe_test.go | 205 ++++++++++++++++++++++++++++ uvworkspace.go | 178 ++++++++++++++++++++++++ 5 files changed, 441 insertions(+), 8 deletions(-) create mode 100644 uv_workspace_recipe_test.go create mode 100644 uvworkspace.go diff --git a/builder.go b/builder.go index 44650d0..f1803ef 100644 --- a/builder.go +++ b/builder.go @@ -183,6 +183,15 @@ type DockerTemplating struct { ServicePrefix string ProjectWorkdir string Carried []CarriedSource + + // UVWorkspace is set when the project is a member of a uv workspace rooted + // in the service: the recipe copies that workspace and syncs the project + // from MemberWorkdir, its directory in the builder stage. + UVWorkspace *uvWorkspaceMembership + MemberWorkdir string + // IgnorePatterns are the dockerignore entries of a recipe that copies the + // service tree whole. + IgnorePatterns []string } func (s *Builder) BuildCapabilities(context.Context, *builderv0.BuildCapabilitiesRequest) (*builderv0.BuildCapabilitiesResponse, error) { @@ -204,14 +213,29 @@ func (s *Builder) Build(ctx context.Context, req *builderv0.BuildRequest) (*buil if err != nil { return s.Base.Builder.BuildError(err) } - // A `[tool.uv.sources]` path resolves against the project directory, and the - // default context is the service directory and nothing beside it: carry what - // leaves the project into an assembled context so the builder stage resolves - // what the host resolves. A service with no such path assembles nothing. - assembled, err := assembleUVContext(s.Location, s.SourceLocation, outputDir) + // A project that is a member of a uv workspace in the service is built from + // that workspace, copied whole (see enclosingUVWorkspace); its path sources + // must then stay inside the service the context carries. + membership, err := enclosingUVWorkspace(s.Location, s.SourceLocation) if err != nil { return s.Base.Builder.BuildError(err) } + var assembled *assembledContext + if membership != nil { + if err := pathSourcesStayInService(s.Location, s.SourceLocation, membership); err != nil { + return s.Base.Builder.BuildError(err) + } + } else { + // A `[tool.uv.sources]` path resolves against the project directory, and + // the default context is the service directory and nothing beside it: + // carry what leaves the project into an assembled context so the builder + // stage resolves what the host resolves. A service with no such path + // assembles nothing. + assembled, err = assembleUVContext(s.Location, s.SourceLocation, outputDir) + if err != nil { + return s.Base.Builder.BuildError(err) + } + } components, err := imageComponents(s.SourceLocation) if err != nil { return s.Base.Builder.BuildError(err) @@ -227,6 +251,11 @@ func (s *Builder) Build(ctx context.Context, req *builderv0.BuildRequest) (*buil docker.ProjectWorkdir = assembled.ProjectWorkdir docker.Carried = assembled.Carried } + if membership != nil { + docker.UVWorkspace = membership + docker.MemberWorkdir = path.Join(defaultProjectWorkdir, membership.Root, membership.Member) + docker.IgnorePatterns = ephemeralIgnorePatterns() + } if err := s.Base.Templates(ctx, docker, services.WithBuilder(builderFS).WithDestination("%s", outputDir)); err != nil { return s.Base.Builder.BuildError(err) } diff --git a/templates/builder/Dockerfile.tmpl b/templates/builder/Dockerfile.tmpl index 9c5ffb1..651778d 100644 --- a/templates/builder/Dockerfile.tmpl +++ b/templates/builder/Dockerfile.tmpl @@ -10,6 +10,15 @@ FROM {{.Builder}} as builder COPY --from={{.UVImage}} /uv /usr/local/bin/uv WORKDIR {{.ProjectWorkdir}} +{{- if .UVWorkspace}} + +# The project is a member of the uv workspace rooted at "{{.UVWorkspace.Root}}" +# in the service. uv reads the lock and every member from that root, so the +# workspace is copied whole and the project is synced from its own directory, +# which is how `uv sync` resolves it on the host. +COPY {{.UVWorkspace.Root}} {{.UVWorkspace.Root}} +WORKDIR {{.MemberWorkdir}} +{{- else}} COPY {{.ServicePrefix}}code/pyproject.toml {{.ServicePrefix}}code/uv.lock ./ {{- if .Carried}} @@ -21,12 +30,20 @@ COPY {{.ServicePrefix}}code/pyproject.toml {{.ServicePrefix}}code/uv.lock ./ COPY {{.Context}} {{.Image}} {{- end}} {{- end}} +{{- end}} # Install deps to a project-local .venv. --frozen installs the locked resolution -# as it stands instead of re-resolving it at build time. +# as it stands instead of re-resolving it at build time. --no-editable installs +# workspace members and path sources as packages in the venv rather than links +# to builder-stage sources, which the runtime stage does not carry. +# +# A project whose requires-python the base image's Python does not satisfy gets +# the interpreter uv installs for it, as on the host. It is installed where the +# runtime stage copies it from, so the venv's interpreter exists there too. ENV UV_PROJECT_ENVIRONMENT=/app/.venv \ - UV_LINK_MODE=copy -RUN uv sync --frozen --no-dev --no-install-project + UV_LINK_MODE=copy \ + UV_PYTHON_INSTALL_DIR=/opt/uv/python +RUN mkdir -p /opt/uv/python && uv sync --frozen --no-dev --no-install-project --no-editable # Runtime @@ -42,6 +59,7 @@ USER appuser ENV VIRTUAL_ENV=/app/.venv \ PATH="/app/.venv/bin:$PATH" +COPY --from=builder /opt/uv/python /opt/uv/python COPY --from=builder ${VIRTUAL_ENV} ${VIRTUAL_ENV} COPY --chown=appuser {{.ServicePrefix}}service.codefly.yaml . diff --git a/templates/builder/dockerignore.tmpl b/templates/builder/dockerignore.tmpl index e69de29..2a55543 100644 --- a/templates/builder/dockerignore.tmpl +++ b/templates/builder/dockerignore.tmpl @@ -0,0 +1,3 @@ +{{- range .IgnorePatterns}} +{{.}} +{{- end}} diff --git a/uv_workspace_recipe_test.go b/uv_workspace_recipe_test.go new file mode 100644 index 0000000..cc902a1 --- /dev/null +++ b/uv_workspace_recipe_test.go @@ -0,0 +1,205 @@ +package main + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/codefly-dev/core/agents/services" + builderv0 "github.com/codefly-dev/core/generated/go/codefly/services/builder/v0" + "github.com/stretchr/testify/require" +) + +// workspaceRequiresPython is outside the Python the base image carries, so the +// image build has to use the interpreter uv installs for the project, as the +// host does. +const workspaceRequiresPython = ">=3.11,<3.12" + +// uvWorkspaceService lays a service out the way a uv workspace member project +// is: the service directory is the workspace root and holds the one uv.lock, +// `code/` is a member depending on a library member `{ workspace = true }` — +// which uv honours only under an ancestor root — and `code/` has no lock of +// its own. +func uvWorkspaceService(t *testing.T, serviceDir, requiresPython string) { + t.Helper() + _ = os.Remove(filepath.Join(serviceDir, "code", "uv.lock")) + writeFile(t, filepath.Join(serviceDir, "pyproject.toml"), `[tool.uv.workspace] +members = ["code", "packages/*"] +exclude = ["packages/scratch"] +`) + writeFile(t, filepath.Join(serviceDir, "packages", "greeting", "pyproject.toml"), fmt.Sprintf(`[project] +name = "greeting" +version = "0.1.0" +requires-python = %q +dependencies = [] + +[build-system] +requires = ["uv_build>=0.8,<0.13"] +build-backend = "uv_build" +`, requiresPython)) + writeFile(t, filepath.Join(serviceDir, "packages", "greeting", "src", "greeting", "__init__.py"), "TEXT = \"hello from a workspace member\"\n") + writeFile(t, filepath.Join(serviceDir, "code", "pyproject.toml"), fmt.Sprintf(`[project] +name = "example-service" +version = "0.1.0" +requires-python = %q +dependencies = ["greeting"] + +[tool.uv] +package = false + +[tool.uv.sources] +greeting = { workspace = true } +`, requiresPython)) +} + +func lockWorkspace(t *testing.T, serviceDir string) bool { + t.Helper() + if _, err := exec.LookPath("uv"); err != nil { + return false + } + lock := exec.Command("uv", "lock") + lock.Dir = serviceDir + output, err := lock.CombinedOutput() + require.NoError(t, err, "uv lock: %s", output) + return true +} + +func TestEnclosingUVWorkspaceFindsTheMembership(t *testing.T) { + service := t.TempDir() + writeFile(t, filepath.Join(service, "module.codefly.yaml"), "name: mod\n") + uvWorkspaceService(t, service, workspaceRequiresPython) + writeFile(t, filepath.Join(service, "uv.lock"), "version = 1\n") + + membership, err := enclosingUVWorkspace(service, filepath.Join(service, "code")) + require.NoError(t, err) + require.Equal(t, &uvWorkspaceMembership{Root: ".", Member: "code"}, membership) +} + +func TestEnclosingUVWorkspaceLeavesAProjectWithItsOwnLock(t *testing.T) { + service := t.TempDir() + uvWorkspaceService(t, service, workspaceRequiresPython) + writeFile(t, filepath.Join(service, "uv.lock"), "version = 1\n") + writeFile(t, filepath.Join(service, "code", "uv.lock"), "version = 1\n") + + membership, err := enclosingUVWorkspace(service, filepath.Join(service, "code")) + require.NoError(t, err) + require.Nil(t, membership, "a project with its own lock is its own root") +} + +func TestEnclosingUVWorkspaceHonoursMembersAndExclude(t *testing.T) { + workspace := &uvWorkspaceTable{Members: []string{"code", "packages/*"}, Exclude: []string{"packages/scratch"}} + require.True(t, workspace.lists("code")) + require.True(t, workspace.lists("packages/greeting")) + require.False(t, workspace.lists("packages/scratch"), "an excluded directory is not a member") + require.False(t, workspace.lists("tools"), "an unlisted directory is not a member") + + service := t.TempDir() + writeFile(t, filepath.Join(service, "pyproject.toml"), "[tool.uv.workspace]\nmembers = [\"packages/*\"]\n") + writeFile(t, filepath.Join(service, "uv.lock"), "version = 1\n") + writeFile(t, filepath.Join(service, "code", "pyproject.toml"), "[project]\nname = \"x\"\nversion = \"0\"\n") + membership, err := enclosingUVWorkspace(service, filepath.Join(service, "code")) + require.NoError(t, err) + require.Nil(t, membership, "uv treats an unlisted project as standalone") +} + +func TestEnclosingUVWorkspaceRefusesAWorkspaceWithoutALock(t *testing.T) { + service := t.TempDir() + uvWorkspaceService(t, service, workspaceRequiresPython) + _, err := enclosingUVWorkspace(service, filepath.Join(service, "code")) + require.ErrorContains(t, err, "has no uv.lock") +} + +func TestEnclosingUVWorkspaceRefusesARootOutsideTheService(t *testing.T) { + repository := t.TempDir() + writeFile(t, filepath.Join(repository, "module.codefly.yaml"), "name: mod\n") + writeFile(t, filepath.Join(repository, "pyproject.toml"), "[tool.uv.workspace]\nmembers = [\"services/*/code\"]\n") + writeFile(t, filepath.Join(repository, "uv.lock"), "version = 1\n") + service := filepath.Join(repository, "services", "api") + writeFile(t, filepath.Join(service, "code", "pyproject.toml"), "[project]\nname = \"api\"\nversion = \"0\"\n") + + _, err := enclosingUVWorkspace(service, filepath.Join(service, "code")) + require.ErrorContains(t, err, "outside the service") +} + +// TestWorkspaceMemberRecipeCopiesTheWorkspace pins the recipe a workspace member +// gets: the workspace copied from its root, the project synced from its own +// directory, the developer's virtual environments and caches ignored, and no +// reference to a `code/uv.lock` that does not exist. +func TestWorkspaceMemberRecipeCopiesTheWorkspace(t *testing.T) { + builder, _ := createdBuilder(t) + uvWorkspaceService(t, builder.Location, workspaceRequiresPython) + if !lockWorkspace(t, builder.Location) { + t.Skip("uv is not installed: the workspace cannot be locked") + } + outputDir := t.TempDir() + plan := emitRecipe(t, builder, outputDir) + require.NoError(t, services.VerifyDockerBuildPlan(outputDir, plan)) + require.Equal(t, builderv0.RecipeInventoryScope_RECIPE_INVENTORY_SCOPE_EMITTED, plan.GetScope(), + "the service directory is the context: the recipe assembles nothing") + + dockerfile := dockerfileOf(t, outputDir, plan) + require.Contains(t, dockerfile, "COPY . .") + require.Contains(t, dockerfile, "WORKDIR /app/code") + require.NotContains(t, dockerfile, "code/uv.lock") + require.Less(t, strings.Index(dockerfile, "WORKDIR /app/code"), strings.Index(dockerfile, "uv sync --frozen")) + require.Contains(t, dockerfile, "--no-editable") + + ignore, err := os.ReadFile(filepath.Join(outputDir, plan.GetRecipes()[0].GetDockerignore())) + require.NoError(t, err) + require.Contains(t, string(ignore), "**/.venv") +} + +// TestAStandaloneProjectRecipeIsUnchangedInShape keeps the default recipe what +// it was for a project that owns its lock: its two files copied, not the tree. +func TestAStandaloneProjectRecipeIsUnchangedInShape(t *testing.T) { + builder, _ := createdBuilder(t) + outputDir := t.TempDir() + plan := emitRecipe(t, builder, outputDir) + dockerfile := dockerfileOf(t, outputDir, plan) + require.Contains(t, dockerfile, "COPY code/pyproject.toml code/uv.lock ./") + require.NotContains(t, dockerfile, "COPY . .") +} + +// TestWorkspaceMemberBuildsTheImage is the claim end to end: a member project +// whose lock is at the workspace root, and whose requires-python the base +// image's Python does not satisfy, builds, and its image imports the workspace +// library with the interpreter the project asked for. +func TestWorkspaceMemberBuildsTheImage(t *testing.T) { + for _, tool := range []string{"docker", "uv"} { + if _, err := exec.LookPath(tool); err != nil { + t.Skipf("%s is not installed: the workspace image cannot be built", tool) + } + } + builder, _ := createdBuilder(t) + uvWorkspaceService(t, builder.Location, workspaceRequiresPython) + require.True(t, lockWorkspace(t, builder.Location)) + // A developer's environment in the tree must not reach the image. + writeFile(t, filepath.Join(builder.Location, ".venv", "pyvenv.cfg"), "home = /nowhere\n") + + outputDir := t.TempDir() + plan := emitRecipe(t, builder, outputDir) + recipe := plan.GetRecipes()[0] + tag := fmt.Sprintf("codefly-fastapi-uv-workspace-test:%d", time.Now().UnixMilli()) + build := exec.CommandContext(t.Context(), "docker", "build", + "-f", filepath.Join(outputDir, recipe.GetDockerfile()), + "-t", tag, builder.Location) + // The CLI applies the recipe's ignore file to the context; plain docker + // reads .dockerignore beside the Dockerfile. + ignore, err := os.ReadFile(filepath.Join(outputDir, recipe.GetDockerignore())) + require.NoError(t, err) + writeFile(t, filepath.Join(outputDir, recipe.GetDockerfile()+".dockerignore"), string(ignore)) + if out, err := build.CombinedOutput(); err != nil { + t.Fatalf("the workspace member did not build: %v\n%s", err, out) + } + t.Cleanup(func() { _ = exec.Command("docker", "rmi", "-f", tag).Run() }) + + out, err := exec.CommandContext(t.Context(), "docker", "run", "--rm", tag, + "python", "-c", "import sys, greeting; print(sys.version_info[:2], greeting.TEXT)").CombinedOutput() + require.NoError(t, err, "%s", out) + require.Contains(t, string(out), "(3, 11)", "the image runs the interpreter the project requires") + require.Contains(t, string(out), "hello from a workspace member", "the image carries the workspace library") +} diff --git a/uvworkspace.go b/uvworkspace.go new file mode 100644 index 0000000..a7af380 --- /dev/null +++ b/uvworkspace.go @@ -0,0 +1,178 @@ +package main + +import ( + "errors" + "fmt" + "os" + "path" + "path/filepath" + "sort" + + "github.com/pelletier/go-toml/v2" +) + +// A service's project (`/code`) may be a member of a uv workspace +// rather than a workspace of its own. uv then keeps one lock, and the +// definitions of every member, at the workspace root: `uv sync` run from the +// project discovers the root above it and resolves against that lock, which is +// what the agent's native runner does. A packaged library whose members name +// each other `{ workspace = true }` has to be laid out this way, because uv +// honours that source only under an ancestor root. +// +// The image recipe used to copy `code/pyproject.toml` and `code/uv.lock` and +// nothing else, so such a project, which resolved and ran on the host, failed +// the image build on a lock that is not in `code/`. The recipe now builds it +// the way the host resolves it: the workspace is copied whole and the project +// is synced from its own directory. +// +// The workspace root must be inside the service, which the image build context +// is. A root above it would make the image depend on files the service does +// not own, and is reported rather than built around. + +// uvWorkspaceMembership places a project in the uv workspace it belongs to. +type uvWorkspaceMembership struct { + // Root is the workspace root, relative to the service directory, in slash + // form ("." when the service directory is the root). + Root string + // Member is the project's directory relative to Root, in slash form. + Member string +} + +type uvWorkspaceTable struct { + Members []string `toml:"members"` + Exclude []string `toml:"exclude"` +} + +// enclosingUVWorkspace returns the uv workspace the project at sourceDir is a +// member of, or nil when the project is its own root: it has its own uv.lock, or +// no workspace above it inside the repository lists it. +func enclosingUVWorkspace(serviceDir, sourceDir string) (*uvWorkspaceMembership, error) { + project, err := filepath.EvalSymlinks(sourceDir) + if err != nil { + return nil, err + } + if _, err := os.Stat(filepath.Join(project, "uv.lock")); err == nil { + return nil, nil + } + service, err := filepath.EvalSymlinks(serviceDir) + if err != nil { + return nil, err + } + // A project outside any repository has no workspace to discover; its own + // directory is the only candidate, and it has no lock. + boundary, err := repositoryRoot(service) + if err != nil { + boundary = service + } + for directory := filepath.Dir(project); within(boundary, directory); directory = filepath.Dir(directory) { + workspace, found, err := readUVWorkspace(directory) + if err != nil { + return nil, err + } + if found { + member, err := filepath.Rel(directory, project) + if err != nil { + return nil, err + } + if !workspace.lists(filepath.ToSlash(member)) { + // uv treats a project a workspace does not list as standalone. + return nil, nil + } + if !within(service, directory) { + return nil, fmt.Errorf("the project %s is a member of the uv workspace rooted at %s, outside the service %s: the image build context cannot include it", + project, directory, service) + } + if _, err := os.Stat(filepath.Join(directory, "uv.lock")); err != nil { + return nil, fmt.Errorf("the uv workspace rooted at %s, which the project %s is a member of, has no uv.lock: run `uv lock` there", directory, project) + } + root, err := filepath.Rel(service, directory) + if err != nil { + return nil, err + } + return &uvWorkspaceMembership{Root: filepath.ToSlash(root), Member: filepath.ToSlash(member)}, nil + } + if directory == boundary { + break + } + } + return nil, nil +} + +// readUVWorkspace reads the `[tool.uv.workspace]` table of directory's +// pyproject.toml, reporting whether the directory is a workspace root. +func readUVWorkspace(directory string) (*uvWorkspaceTable, bool, error) { + manifest := filepath.Join(directory, "pyproject.toml") + content, err := os.ReadFile(manifest) + if errors.Is(err, os.ErrNotExist) { + return nil, false, nil + } + if err != nil { + return nil, false, err + } + var document struct { + Tool struct { + UV struct { + Workspace *uvWorkspaceTable `toml:"workspace"` + } `toml:"uv"` + } `toml:"tool"` + } + if err := toml.Unmarshal(content, &document); err != nil { + return nil, false, fmt.Errorf("parse %s: %w", manifest, err) + } + if document.Tool.UV.Workspace == nil { + return nil, false, nil + } + return document.Tool.UV.Workspace, true, nil +} + +// lists reports whether the workspace includes member (slash form, relative to +// its root): a `members` glob matches it and no `exclude` glob does. Globs match +// one path at a time, as uv's do for the `*`, `?` and `[...]` forms. +func (workspace *uvWorkspaceTable) lists(member string) bool { + matches := func(patterns []string) bool { + for _, pattern := range patterns { + if ok, _ := path.Match(path.Clean(pattern), member); ok { + return true + } + } + return false + } + return matches(workspace.Members) && !matches(workspace.Exclude) +} + +// ephemeralIgnorePatterns are the dockerignore patterns for +// ephemeralSourceDirectories at any depth, in a stable order. A workspace +// recipe copies the service tree whole, and a developer's virtual environment +// or cache in it is never an input to the image. +func ephemeralIgnorePatterns() []string { + patterns := make([]string, 0, len(ephemeralSourceDirectories)) + for name := range ephemeralSourceDirectories { + patterns = append(patterns, "**/"+name) + } + sort.Strings(patterns) + return patterns +} + +// pathSourcesStayInService checks that every `[tool.uv.sources]` path the +// project or its workspace root declares resolves inside the service. A +// workspace recipe copies the service tree whole, so a path inside it is +// present in the builder stage; one outside it would not be. +func pathSourcesStayInService(serviceDir, sourceDir string, membership *uvWorkspaceMembership) error { + service, err := filepath.EvalSymlinks(serviceDir) + if err != nil { + return err + } + for _, declaring := range []string{sourceDir, filepath.Join(service, filepath.FromSlash(membership.Root))} { + sources, err := escapingPathSources(declaring) + if err != nil { + return err + } + for _, source := range sources { + if !within(service, source.resolved) { + return fmt.Errorf("%s resolves to %s, outside the service %s whose uv workspace the image copies: the image build context cannot include it", + source.declaration(), source.resolved, service) + } + } + } + return nil +}