diff --git a/builder.go b/builder.go index 44650d0..f1803ef 100644 --- a/builder.go +++ b/builder.go @@ -183,6 +183,15 @@ type DockerTemplating struct { ServicePrefix string ProjectWorkdir string Carried []CarriedSource + + // UVWorkspace is set when the project is a member of a uv workspace rooted + // in the service: the recipe copies that workspace and syncs the project + // from MemberWorkdir, its directory in the builder stage. + UVWorkspace *uvWorkspaceMembership + MemberWorkdir string + // IgnorePatterns are the dockerignore entries of a recipe that copies the + // service tree whole. + IgnorePatterns []string } func (s *Builder) BuildCapabilities(context.Context, *builderv0.BuildCapabilitiesRequest) (*builderv0.BuildCapabilitiesResponse, error) { @@ -204,14 +213,29 @@ func (s *Builder) Build(ctx context.Context, req *builderv0.BuildRequest) (*buil if err != nil { return s.Base.Builder.BuildError(err) } - // A `[tool.uv.sources]` path resolves against the project directory, and the - // default context is the service directory and nothing beside it: carry what - // leaves the project into an assembled context so the builder stage resolves - // what the host resolves. A service with no such path assembles nothing. - assembled, err := assembleUVContext(s.Location, s.SourceLocation, outputDir) + // A project that is a member of a uv workspace in the service is built from + // that workspace, copied whole (see enclosingUVWorkspace); its path sources + // must then stay inside the service the context carries. + membership, err := enclosingUVWorkspace(s.Location, s.SourceLocation) if err != nil { return s.Base.Builder.BuildError(err) } + var assembled *assembledContext + if membership != nil { + if err := pathSourcesStayInService(s.Location, s.SourceLocation, membership); err != nil { + return s.Base.Builder.BuildError(err) + } + } else { + // A `[tool.uv.sources]` path resolves against the project directory, and + // the default context is the service directory and nothing beside it: + // carry what leaves the project into an assembled context so the builder + // stage resolves what the host resolves. A service with no such path + // assembles nothing. + assembled, err = assembleUVContext(s.Location, s.SourceLocation, outputDir) + if err != nil { + return s.Base.Builder.BuildError(err) + } + } components, err := imageComponents(s.SourceLocation) if err != nil { return s.Base.Builder.BuildError(err) @@ -227,6 +251,11 @@ func (s *Builder) Build(ctx context.Context, req *builderv0.BuildRequest) (*buil docker.ProjectWorkdir = assembled.ProjectWorkdir docker.Carried = assembled.Carried } + if membership != nil { + docker.UVWorkspace = membership + docker.MemberWorkdir = path.Join(defaultProjectWorkdir, membership.Root, membership.Member) + docker.IgnorePatterns = ephemeralIgnorePatterns() + } if err := s.Base.Templates(ctx, docker, services.WithBuilder(builderFS).WithDestination("%s", outputDir)); err != nil { return s.Base.Builder.BuildError(err) } diff --git a/templates/builder/Dockerfile.tmpl b/templates/builder/Dockerfile.tmpl index 9c5ffb1..651778d 100644 --- a/templates/builder/Dockerfile.tmpl +++ b/templates/builder/Dockerfile.tmpl @@ -10,6 +10,15 @@ FROM {{.Builder}} as builder COPY --from={{.UVImage}} /uv /usr/local/bin/uv WORKDIR {{.ProjectWorkdir}} +{{- if .UVWorkspace}} + +# The project is a member of the uv workspace rooted at "{{.UVWorkspace.Root}}" +# in the service. uv reads the lock and every member from that root, so the +# workspace is copied whole and the project is synced from its own directory, +# which is how `uv sync` resolves it on the host. +COPY {{.UVWorkspace.Root}} {{.UVWorkspace.Root}} +WORKDIR {{.MemberWorkdir}} +{{- else}} COPY {{.ServicePrefix}}code/pyproject.toml {{.ServicePrefix}}code/uv.lock ./ {{- if .Carried}} @@ -21,12 +30,20 @@ COPY {{.ServicePrefix}}code/pyproject.toml {{.ServicePrefix}}code/uv.lock ./ COPY {{.Context}} {{.Image}} {{- end}} {{- end}} +{{- end}} # Install deps to a project-local .venv. --frozen installs the locked resolution -# as it stands instead of re-resolving it at build time. +# as it stands instead of re-resolving it at build time. --no-editable installs +# workspace members and path sources as packages in the venv rather than links +# to builder-stage sources, which the runtime stage does not carry. +# +# A project whose requires-python the base image's Python does not satisfy gets +# the interpreter uv installs for it, as on the host. It is installed where the +# runtime stage copies it from, so the venv's interpreter exists there too. ENV UV_PROJECT_ENVIRONMENT=/app/.venv \ - UV_LINK_MODE=copy -RUN uv sync --frozen --no-dev --no-install-project + UV_LINK_MODE=copy \ + UV_PYTHON_INSTALL_DIR=/opt/uv/python +RUN mkdir -p /opt/uv/python && uv sync --frozen --no-dev --no-install-project --no-editable # Runtime @@ -42,6 +59,7 @@ USER appuser ENV VIRTUAL_ENV=/app/.venv \ PATH="/app/.venv/bin:$PATH" +COPY --from=builder /opt/uv/python /opt/uv/python COPY --from=builder ${VIRTUAL_ENV} ${VIRTUAL_ENV} COPY --chown=appuser {{.ServicePrefix}}service.codefly.yaml . diff --git a/templates/builder/dockerignore.tmpl b/templates/builder/dockerignore.tmpl index e69de29..2a55543 100644 --- a/templates/builder/dockerignore.tmpl +++ b/templates/builder/dockerignore.tmpl @@ -0,0 +1,3 @@ +{{- range .IgnorePatterns}} +{{.}} +{{- end}} diff --git a/uv_workspace_recipe_test.go b/uv_workspace_recipe_test.go new file mode 100644 index 0000000..cc902a1 --- /dev/null +++ b/uv_workspace_recipe_test.go @@ -0,0 +1,205 @@ +package main + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/codefly-dev/core/agents/services" + builderv0 "github.com/codefly-dev/core/generated/go/codefly/services/builder/v0" + "github.com/stretchr/testify/require" +) + +// workspaceRequiresPython is outside the Python the base image carries, so the +// image build has to use the interpreter uv installs for the project, as the +// host does. +const workspaceRequiresPython = ">=3.11,<3.12" + +// uvWorkspaceService lays a service out the way a uv workspace member project +// is: the service directory is the workspace root and holds the one uv.lock, +// `code/` is a member depending on a library member `{ workspace = true }` — +// which uv honours only under an ancestor root — and `code/` has no lock of +// its own. +func uvWorkspaceService(t *testing.T, serviceDir, requiresPython string) { + t.Helper() + _ = os.Remove(filepath.Join(serviceDir, "code", "uv.lock")) + writeFile(t, filepath.Join(serviceDir, "pyproject.toml"), `[tool.uv.workspace] +members = ["code", "packages/*"] +exclude = ["packages/scratch"] +`) + writeFile(t, filepath.Join(serviceDir, "packages", "greeting", "pyproject.toml"), fmt.Sprintf(`[project] +name = "greeting" +version = "0.1.0" +requires-python = %q +dependencies = [] + +[build-system] +requires = ["uv_build>=0.8,<0.13"] +build-backend = "uv_build" +`, requiresPython)) + writeFile(t, filepath.Join(serviceDir, "packages", "greeting", "src", "greeting", "__init__.py"), "TEXT = \"hello from a workspace member\"\n") + writeFile(t, filepath.Join(serviceDir, "code", "pyproject.toml"), fmt.Sprintf(`[project] +name = "example-service" +version = "0.1.0" +requires-python = %q +dependencies = ["greeting"] + +[tool.uv] +package = false + +[tool.uv.sources] +greeting = { workspace = true } +`, requiresPython)) +} + +func lockWorkspace(t *testing.T, serviceDir string) bool { + t.Helper() + if _, err := exec.LookPath("uv"); err != nil { + return false + } + lock := exec.Command("uv", "lock") + lock.Dir = serviceDir + output, err := lock.CombinedOutput() + require.NoError(t, err, "uv lock: %s", output) + return true +} + +func TestEnclosingUVWorkspaceFindsTheMembership(t *testing.T) { + service := t.TempDir() + writeFile(t, filepath.Join(service, "module.codefly.yaml"), "name: mod\n") + uvWorkspaceService(t, service, workspaceRequiresPython) + writeFile(t, filepath.Join(service, "uv.lock"), "version = 1\n") + + membership, err := enclosingUVWorkspace(service, filepath.Join(service, "code")) + require.NoError(t, err) + require.Equal(t, &uvWorkspaceMembership{Root: ".", Member: "code"}, membership) +} + +func TestEnclosingUVWorkspaceLeavesAProjectWithItsOwnLock(t *testing.T) { + service := t.TempDir() + uvWorkspaceService(t, service, workspaceRequiresPython) + writeFile(t, filepath.Join(service, "uv.lock"), "version = 1\n") + writeFile(t, filepath.Join(service, "code", "uv.lock"), "version = 1\n") + + membership, err := enclosingUVWorkspace(service, filepath.Join(service, "code")) + require.NoError(t, err) + require.Nil(t, membership, "a project with its own lock is its own root") +} + +func TestEnclosingUVWorkspaceHonoursMembersAndExclude(t *testing.T) { + workspace := &uvWorkspaceTable{Members: []string{"code", "packages/*"}, Exclude: []string{"packages/scratch"}} + require.True(t, workspace.lists("code")) + require.True(t, workspace.lists("packages/greeting")) + require.False(t, workspace.lists("packages/scratch"), "an excluded directory is not a member") + require.False(t, workspace.lists("tools"), "an unlisted directory is not a member") + + service := t.TempDir() + writeFile(t, filepath.Join(service, "pyproject.toml"), "[tool.uv.workspace]\nmembers = [\"packages/*\"]\n") + writeFile(t, filepath.Join(service, "uv.lock"), "version = 1\n") + writeFile(t, filepath.Join(service, "code", "pyproject.toml"), "[project]\nname = \"x\"\nversion = \"0\"\n") + membership, err := enclosingUVWorkspace(service, filepath.Join(service, "code")) + require.NoError(t, err) + require.Nil(t, membership, "uv treats an unlisted project as standalone") +} + +func TestEnclosingUVWorkspaceRefusesAWorkspaceWithoutALock(t *testing.T) { + service := t.TempDir() + uvWorkspaceService(t, service, workspaceRequiresPython) + _, err := enclosingUVWorkspace(service, filepath.Join(service, "code")) + require.ErrorContains(t, err, "has no uv.lock") +} + +func TestEnclosingUVWorkspaceRefusesARootOutsideTheService(t *testing.T) { + repository := t.TempDir() + writeFile(t, filepath.Join(repository, "module.codefly.yaml"), "name: mod\n") + writeFile(t, filepath.Join(repository, "pyproject.toml"), "[tool.uv.workspace]\nmembers = [\"services/*/code\"]\n") + writeFile(t, filepath.Join(repository, "uv.lock"), "version = 1\n") + service := filepath.Join(repository, "services", "api") + writeFile(t, filepath.Join(service, "code", "pyproject.toml"), "[project]\nname = \"api\"\nversion = \"0\"\n") + + _, err := enclosingUVWorkspace(service, filepath.Join(service, "code")) + require.ErrorContains(t, err, "outside the service") +} + +// TestWorkspaceMemberRecipeCopiesTheWorkspace pins the recipe a workspace member +// gets: the workspace copied from its root, the project synced from its own +// directory, the developer's virtual environments and caches ignored, and no +// reference to a `code/uv.lock` that does not exist. +func TestWorkspaceMemberRecipeCopiesTheWorkspace(t *testing.T) { + builder, _ := createdBuilder(t) + uvWorkspaceService(t, builder.Location, workspaceRequiresPython) + if !lockWorkspace(t, builder.Location) { + t.Skip("uv is not installed: the workspace cannot be locked") + } + outputDir := t.TempDir() + plan := emitRecipe(t, builder, outputDir) + require.NoError(t, services.VerifyDockerBuildPlan(outputDir, plan)) + require.Equal(t, builderv0.RecipeInventoryScope_RECIPE_INVENTORY_SCOPE_EMITTED, plan.GetScope(), + "the service directory is the context: the recipe assembles nothing") + + dockerfile := dockerfileOf(t, outputDir, plan) + require.Contains(t, dockerfile, "COPY . .") + require.Contains(t, dockerfile, "WORKDIR /app/code") + require.NotContains(t, dockerfile, "code/uv.lock") + require.Less(t, strings.Index(dockerfile, "WORKDIR /app/code"), strings.Index(dockerfile, "uv sync --frozen")) + require.Contains(t, dockerfile, "--no-editable") + + ignore, err := os.ReadFile(filepath.Join(outputDir, plan.GetRecipes()[0].GetDockerignore())) + require.NoError(t, err) + require.Contains(t, string(ignore), "**/.venv") +} + +// TestAStandaloneProjectRecipeIsUnchangedInShape keeps the default recipe what +// it was for a project that owns its lock: its two files copied, not the tree. +func TestAStandaloneProjectRecipeIsUnchangedInShape(t *testing.T) { + builder, _ := createdBuilder(t) + outputDir := t.TempDir() + plan := emitRecipe(t, builder, outputDir) + dockerfile := dockerfileOf(t, outputDir, plan) + require.Contains(t, dockerfile, "COPY code/pyproject.toml code/uv.lock ./") + require.NotContains(t, dockerfile, "COPY . .") +} + +// TestWorkspaceMemberBuildsTheImage is the claim end to end: a member project +// whose lock is at the workspace root, and whose requires-python the base +// image's Python does not satisfy, builds, and its image imports the workspace +// library with the interpreter the project asked for. +func TestWorkspaceMemberBuildsTheImage(t *testing.T) { + for _, tool := range []string{"docker", "uv"} { + if _, err := exec.LookPath(tool); err != nil { + t.Skipf("%s is not installed: the workspace image cannot be built", tool) + } + } + builder, _ := createdBuilder(t) + uvWorkspaceService(t, builder.Location, workspaceRequiresPython) + require.True(t, lockWorkspace(t, builder.Location)) + // A developer's environment in the tree must not reach the image. + writeFile(t, filepath.Join(builder.Location, ".venv", "pyvenv.cfg"), "home = /nowhere\n") + + outputDir := t.TempDir() + plan := emitRecipe(t, builder, outputDir) + recipe := plan.GetRecipes()[0] + tag := fmt.Sprintf("codefly-fastapi-uv-workspace-test:%d", time.Now().UnixMilli()) + build := exec.CommandContext(t.Context(), "docker", "build", + "-f", filepath.Join(outputDir, recipe.GetDockerfile()), + "-t", tag, builder.Location) + // The CLI applies the recipe's ignore file to the context; plain docker + // reads .dockerignore beside the Dockerfile. + ignore, err := os.ReadFile(filepath.Join(outputDir, recipe.GetDockerignore())) + require.NoError(t, err) + writeFile(t, filepath.Join(outputDir, recipe.GetDockerfile()+".dockerignore"), string(ignore)) + if out, err := build.CombinedOutput(); err != nil { + t.Fatalf("the workspace member did not build: %v\n%s", err, out) + } + t.Cleanup(func() { _ = exec.Command("docker", "rmi", "-f", tag).Run() }) + + out, err := exec.CommandContext(t.Context(), "docker", "run", "--rm", tag, + "python", "-c", "import sys, greeting; print(sys.version_info[:2], greeting.TEXT)").CombinedOutput() + require.NoError(t, err, "%s", out) + require.Contains(t, string(out), "(3, 11)", "the image runs the interpreter the project requires") + require.Contains(t, string(out), "hello from a workspace member", "the image carries the workspace library") +} diff --git a/uvworkspace.go b/uvworkspace.go new file mode 100644 index 0000000..a7af380 --- /dev/null +++ b/uvworkspace.go @@ -0,0 +1,178 @@ +package main + +import ( + "errors" + "fmt" + "os" + "path" + "path/filepath" + "sort" + + "github.com/pelletier/go-toml/v2" +) + +// A service's project (`/code`) may be a member of a uv workspace +// rather than a workspace of its own. uv then keeps one lock, and the +// definitions of every member, at the workspace root: `uv sync` run from the +// project discovers the root above it and resolves against that lock, which is +// what the agent's native runner does. A packaged library whose members name +// each other `{ workspace = true }` has to be laid out this way, because uv +// honours that source only under an ancestor root. +// +// The image recipe used to copy `code/pyproject.toml` and `code/uv.lock` and +// nothing else, so such a project, which resolved and ran on the host, failed +// the image build on a lock that is not in `code/`. The recipe now builds it +// the way the host resolves it: the workspace is copied whole and the project +// is synced from its own directory. +// +// The workspace root must be inside the service, which the image build context +// is. A root above it would make the image depend on files the service does +// not own, and is reported rather than built around. + +// uvWorkspaceMembership places a project in the uv workspace it belongs to. +type uvWorkspaceMembership struct { + // Root is the workspace root, relative to the service directory, in slash + // form ("." when the service directory is the root). + Root string + // Member is the project's directory relative to Root, in slash form. + Member string +} + +type uvWorkspaceTable struct { + Members []string `toml:"members"` + Exclude []string `toml:"exclude"` +} + +// enclosingUVWorkspace returns the uv workspace the project at sourceDir is a +// member of, or nil when the project is its own root: it has its own uv.lock, or +// no workspace above it inside the repository lists it. +func enclosingUVWorkspace(serviceDir, sourceDir string) (*uvWorkspaceMembership, error) { + project, err := filepath.EvalSymlinks(sourceDir) + if err != nil { + return nil, err + } + if _, err := os.Stat(filepath.Join(project, "uv.lock")); err == nil { + return nil, nil + } + service, err := filepath.EvalSymlinks(serviceDir) + if err != nil { + return nil, err + } + // A project outside any repository has no workspace to discover; its own + // directory is the only candidate, and it has no lock. + boundary, err := repositoryRoot(service) + if err != nil { + boundary = service + } + for directory := filepath.Dir(project); within(boundary, directory); directory = filepath.Dir(directory) { + workspace, found, err := readUVWorkspace(directory) + if err != nil { + return nil, err + } + if found { + member, err := filepath.Rel(directory, project) + if err != nil { + return nil, err + } + if !workspace.lists(filepath.ToSlash(member)) { + // uv treats a project a workspace does not list as standalone. + return nil, nil + } + if !within(service, directory) { + return nil, fmt.Errorf("the project %s is a member of the uv workspace rooted at %s, outside the service %s: the image build context cannot include it", + project, directory, service) + } + if _, err := os.Stat(filepath.Join(directory, "uv.lock")); err != nil { + return nil, fmt.Errorf("the uv workspace rooted at %s, which the project %s is a member of, has no uv.lock: run `uv lock` there", directory, project) + } + root, err := filepath.Rel(service, directory) + if err != nil { + return nil, err + } + return &uvWorkspaceMembership{Root: filepath.ToSlash(root), Member: filepath.ToSlash(member)}, nil + } + if directory == boundary { + break + } + } + return nil, nil +} + +// readUVWorkspace reads the `[tool.uv.workspace]` table of directory's +// pyproject.toml, reporting whether the directory is a workspace root. +func readUVWorkspace(directory string) (*uvWorkspaceTable, bool, error) { + manifest := filepath.Join(directory, "pyproject.toml") + content, err := os.ReadFile(manifest) + if errors.Is(err, os.ErrNotExist) { + return nil, false, nil + } + if err != nil { + return nil, false, err + } + var document struct { + Tool struct { + UV struct { + Workspace *uvWorkspaceTable `toml:"workspace"` + } `toml:"uv"` + } `toml:"tool"` + } + if err := toml.Unmarshal(content, &document); err != nil { + return nil, false, fmt.Errorf("parse %s: %w", manifest, err) + } + if document.Tool.UV.Workspace == nil { + return nil, false, nil + } + return document.Tool.UV.Workspace, true, nil +} + +// lists reports whether the workspace includes member (slash form, relative to +// its root): a `members` glob matches it and no `exclude` glob does. Globs match +// one path at a time, as uv's do for the `*`, `?` and `[...]` forms. +func (workspace *uvWorkspaceTable) lists(member string) bool { + matches := func(patterns []string) bool { + for _, pattern := range patterns { + if ok, _ := path.Match(path.Clean(pattern), member); ok { + return true + } + } + return false + } + return matches(workspace.Members) && !matches(workspace.Exclude) +} + +// ephemeralIgnorePatterns are the dockerignore patterns for +// ephemeralSourceDirectories at any depth, in a stable order. A workspace +// recipe copies the service tree whole, and a developer's virtual environment +// or cache in it is never an input to the image. +func ephemeralIgnorePatterns() []string { + patterns := make([]string, 0, len(ephemeralSourceDirectories)) + for name := range ephemeralSourceDirectories { + patterns = append(patterns, "**/"+name) + } + sort.Strings(patterns) + return patterns +} + +// pathSourcesStayInService checks that every `[tool.uv.sources]` path the +// project or its workspace root declares resolves inside the service. A +// workspace recipe copies the service tree whole, so a path inside it is +// present in the builder stage; one outside it would not be. +func pathSourcesStayInService(serviceDir, sourceDir string, membership *uvWorkspaceMembership) error { + service, err := filepath.EvalSymlinks(serviceDir) + if err != nil { + return err + } + for _, declaring := range []string{sourceDir, filepath.Join(service, filepath.FromSlash(membership.Root))} { + sources, err := escapingPathSources(declaring) + if err != nil { + return err + } + for _, source := range sources { + if !within(service, source.resolved) { + return fmt.Errorf("%s resolves to %s, outside the service %s whose uv workspace the image copies: the image build context cannot include it", + source.declaration(), source.resolved, service) + } + } + } + return nil +}