diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f1236f2..89746f2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,7 +12,7 @@ concurrency: jobs: ci: - uses: codefly-dev/core/.github/workflows/go-service-ci.yml@v0.19.4 + uses: codefly-dev/core/.github/workflows/go-service-ci.yml@v0.19.6 with: # The Python runtime tests drive `uv` on the host (dependency sync, # pytest); it isn't on the runner by default. diff --git a/AGENTS.md b/AGENTS.md index 2ed90c3..0259e4a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -67,9 +67,9 @@ within reach. ## Build and test Derived from `.github/workflows/ci.yml`, which delegates to core's shared -`go-service-ci.yml`, pinned to the linked core's tag (`@v0.19.4`; move it with +`go-service-ci.yml`, pinned to the linked core's tag (`@v0.19.6`; move it with the core bump), and adds a `race` job of its own. Go comes from `go.mod` -(1.27.2). All of it runs locally: +(1.27.3). All of it runs locally: ```bash go build ./... # CI: build job @@ -126,6 +126,7 @@ anything from it. | `runtime.go` | Init/Start/Stop/Destroy, uvicorn supervision, hot reload, the lock order | | `builder.go` | `Create` scaffolding, OpenAPI endpoint loading, Docker + kustomize recipes | | `uvsources.go` | the assembled build context: `[tool.uv.sources]` paths leaving the project | +| `uvgitsources.go` | the git commits `uv.lock` pins, declared for the caller to mirror into the build | | `probes.go`, `probe.py` | endpoint health translated to Kubernetes probes / a local probe | | `templates/` | **what ships** — embedded via `go:embed`, rendered on `Create` | | `base/` | a checked-in sample service, embedded nowhere | @@ -158,6 +159,13 @@ into `templates/factory/code/pyproject.toml.tmpl` yourself. same relative path resolves there; that plan claims the whole tree (`RECIPE_INVENTORY_SCOPE_TREE`), the Dockerfile-only one claims just what it emitted. Never rewrite `pyproject.toml` or `uv.lock` to make a path resolve. + A git source is declared in the recipe (`git_source_downloads`) by the URL + exactly as `uv.lock` records it up to its query, which is what uv hands git; + the caller fetches it with its own credentials and the Dockerfile maps that + URL onto the mirror; a normalized URL would miss it silently. A source core + refuses, or whose URL begins an undeclared locked one (insteadOf is a + prefix match), is not declared; a warning says why. An image whose own git + config already rewrites a declared URL keeps that rewrite (git prefers it). - **The runtime image is pinned.** `docker-image` rejects a bare name and `:latest`. The default `ghcr.io/codefly-dev/python` companion is built from core and pinned by the digest of its multi-platform index — prefer it. Its tag diff --git a/builder.go b/builder.go index d1bca72..a5e6575 100644 --- a/builder.go +++ b/builder.go @@ -192,6 +192,9 @@ type DockerTemplating struct { // IgnorePatterns are the dockerignore entries of a recipe that copies the // service tree whole. IgnorePatterns []string + // GitSources are the repositories of the git commits the lock pins, which + // the builder stage reads from the caller's mirrors (uvgitsources.go). + GitSources []uvGitMirror } func (s *Builder) BuildCapabilities(context.Context, *builderv0.BuildCapabilitiesRequest) (*builderv0.BuildCapabilitiesResponse, error) { @@ -248,11 +251,25 @@ func (s *Builder) Build(ctx context.Context, req *builderv0.BuildRequest) (*buil if err != nil { return s.Base.Builder.BuildError(err) } + gitSources, unmirrored, err := lockedGitSources(uvLockPath(s.Location, s.SourceLocation, membership)) + if err != nil { + return s.Base.Builder.BuildError(err) + } + for _, source := range unmirrored { + // The URL is not logged: a lock may still carry a credential in it. + s.Wool.Warn("git source is not mirrored: the image build fetches it itself", + wool.Field("package", source.Package), wool.Field("reason", source.Reason)) + } + var options []services.RecipeOption + if len(gitSources) > 0 { + options = append(options, services.WithGitSourceDownloads(gitSources...)) + } docker := DockerTemplating{ Builder: runtimeImage.FullName(), Components: components, UVImage: uvImage, ProjectWorkdir: defaultProjectWorkdir, + GitSources: uvGitMirrors(gitSources), } if assembled != nil { docker.ServicePrefix = assembled.ServicePrefix @@ -272,19 +289,23 @@ func (s *Builder) Build(ctx context.Context, req *builderv0.BuildRequest) (*buil // Only the Dockerfile and its ignore were written, and the application's // inputs are the service's own tree: the plan claims what it emitted and // the caller builds the service directory. - return s.Base.Builder.SingleImageBuildResponse(req, image, emitted) + return s.Base.Builder.SingleImageBuildResponse(req, image, emitted, options...) } // The whole destination was assembled by this build, so the plan claims all // of it and the caller builds the assembled tree rather than the service // directory — which is what RECIPE_INVENTORY_SCOPE_TREE declares. - plan, err := services.BuildDockerBuildPlan(outputDir, []*builderv0.DockerBuildRecipe{{ + recipe := &builderv0.DockerBuildRecipe{ Name: "app", Dockerfile: "Dockerfile", Context: ".", Dockerignore: dockerignoreOf(emitted), Image: image, Platforms: services.RecipeBuildPlatforms(), - }}) + } + for _, option := range options { + option(recipe) + } + plan, err := services.BuildDockerBuildPlan(outputDir, []*builderv0.DockerBuildRecipe{recipe}) if err != nil { return s.Base.Builder.BuildError(err) } diff --git a/go.mod b/go.mod index 6a95443..a90dcd9 100644 --- a/go.mod +++ b/go.mod @@ -1,9 +1,9 @@ module github.com/codefly-dev/service-python-fastapi -go 1.27.2 +go 1.27.3 require ( - github.com/codefly-dev/core v0.19.4 + github.com/codefly-dev/core v0.19.6 github.com/codefly-dev/service-python v0.0.66 github.com/pelletier/go-toml/v2 v2.4.3 github.com/stretchr/testify v1.12.1 diff --git a/go.sum b/go.sum index 8c49f7c..4630295 100644 --- a/go.sum +++ b/go.sum @@ -43,8 +43,8 @@ github.com/clipperhouse/uax29/v2 v2.7.0 h1:+gs4oBZ2gPfVrKPthwbMzWZDaAFPGYK72F0NJ github.com/clipperhouse/uax29/v2 v2.7.0/go.mod h1:EFJ2TJMRUaplDxHKj1qAEhCtQPW2tJSwu5BF98AuoVM= github.com/cloudflare/circl v1.6.5 h1:O64F26HEqNhznd/hrC5KZXVKYuKM2rx4deZDTc4ihQA= github.com/cloudflare/circl v1.6.5/go.mod h1:h5LNyxAc5nTue9DS5jT+48en2PSDYt3zdGnz5OstK6c= -github.com/codefly-dev/core v0.19.4 h1:vUvaFrWkB8B3ksQY/qlgBTVj3jMNhdEpJca8aofaals= -github.com/codefly-dev/core v0.19.4/go.mod h1:0uX/7lTBVaGV0RI2Mb3mpEr/PPdx/jZqXHym/U8FVDI= +github.com/codefly-dev/core v0.19.6 h1:ZkCChK/BioK+B7zZ0fp+/rXDCy0k+lsdFqHjyc5xFIg= +github.com/codefly-dev/core v0.19.6/go.mod h1:1ina3kP+YSAL4tlZvnEYd4P8bFGnJECfkP86ojeeF1A= github.com/codefly-dev/gortk v0.2.0 h1:7bOlS5valYz2zil+fZctQNcPCYBcPj86abcw9N8h1hQ= github.com/codefly-dev/gortk v0.2.0/go.mod h1:dDWUMFgAP063OCGhTEpV7FFUj9+zTcxxO/nV80VKEwg= github.com/codefly-dev/service-python v0.0.66 h1:Y74QvKAYMaOnZcsfRxuD9dpQo3//7Mjo12RYELyrp5o= diff --git a/templates/builder/Dockerfile.tmpl b/templates/builder/Dockerfile.tmpl index 7424678..fb84146 100644 --- a/templates/builder/Dockerfile.tmpl +++ b/templates/builder/Dockerfile.tmpl @@ -1,3 +1,13 @@ +{{if .GitSources -}} +# syntax=docker/dockerfile:1@sha256:ecfaec9ed6d810b56388c508f4121597bfbba70d41a6dfeee4d8cad5f295fc32 +# The git commits uv.lock pins (git_source_downloads in the recipe), fetched by +# the caller on its own machine, with its own credentials, before any image +# build, and supplied here as the build context of this name: one bare mirror +# per repository. Without a caller supplying it, the stage is empty and uv +# fetches them from the network as it always did. +FROM scratch AS uvsources + +{{end -}} # The builder image — installs the venv via uv, then we copy it into the runtime. FROM {{.Builder}} as builder @@ -43,7 +53,30 @@ COPY {{.Context}} {{.Image}} ENV UV_PROJECT_ENVIRONMENT=/app/.venv \ UV_LINK_MODE=copy \ UV_PYTHON_INSTALL_DIR=/opt/uv/python +{{- if .GitSources}} +# +# With the caller's mirrors, git reads each locked repository from its mirror +# instead of the URL the lock records (uv hands git that URL unchanged), so no +# credential is needed, and a commit missing from the mirror fails here rather +# than prompting for one. The mappings are appended after the git configuration +# the environment already carries, when its GIT_CONFIG_COUNT is a plain count; +# otherwise the configuration is left alone and the sources are fetched as +# without a mirror. An image whose own git configuration (system, global or +# environment) already rewrites a declared repository's URL keeps its rewrite: +# git prefers it, and the build fetches that source as it does without a mirror. +RUN --mount=type=bind,from=uvsources,target=/uvsources \ + if [ -n "$(ls -A /uvsources)" ]; then \ + n=${GIT_CONFIG_COUNT:-0}; n=${n#+}; \ + case $n in ''|*[!0-9]*) n= ;; *) while [ "${n#0}" != "$n" ] && [ -n "${n#0}" ]; do n=${n#0}; done; [ ${#n} -le 4 ] || n= ;; esac; \ + if [ -z "$n" ]; then echo "inherited GIT_CONFIG_COUNT is not a plain count; git sources are fetched without the mirror"; \ + else export GIT_TERMINAL_PROMPT=0{{range $i, $source := .GitSources}} \ + "GIT_CONFIG_KEY_$((n+{{$i}}))=url.file:///uvsources/{{$source.Path}}.insteadOf" "GIT_CONFIG_VALUE_$((n+{{$i}}))="'{{$source.Repository}}'{{end}} \ + GIT_CONFIG_COUNT=$((n+{{len .GitSources}})); fi; \ + fi && \ + mkdir -p /opt/uv/python && uv sync --frozen --no-dev --no-install-project --no-editable +{{- else}} RUN mkdir -p /opt/uv/python && uv sync --frozen --no-dev --no-install-project --no-editable +{{- end}} # Runtime diff --git a/testdata/dockerfile/git-sources.golden b/testdata/dockerfile/git-sources.golden new file mode 100644 index 0000000..3c49c69 --- /dev/null +++ b/testdata/dockerfile/git-sources.golden @@ -0,0 +1,90 @@ +# syntax=docker/dockerfile:1@sha256:ecfaec9ed6d810b56388c508f4121597bfbba70d41a6dfeee4d8cad5f295fc32 +# The git commits uv.lock pins (git_source_downloads in the recipe), fetched by +# the caller on its own machine, with its own credentials, before any image +# build, and supplied here as the build context of this name: one bare mirror +# per repository. Without a caller supplying it, the stage is empty and uv +# fetches them from the network as it always did. +FROM scratch AS uvsources + +# The builder image — installs the venv via uv, then we copy it into the runtime. +FROM BUILDER_IMAGE as builder + +# uv is part of what the recipe has to state. A base image carries whichever uv +# it happened to be built with, so a project written against a newer `[tool.uv]` +# key resolves on the developer's machine and fails inside the builder stage +# with a TOML parse error naming a field that uv there does not know. Copying a +# pinned uv in makes the tool that reads pyproject.toml and uv.lock a declared +# input of the recipe instead of a property of the base image. +COPY --from=UV_IMAGE /uv /usr/local/bin/uv + +WORKDIR /app + +COPY code/pyproject.toml code/uv.lock ./ + +# Install deps to a project-local .venv. --frozen installs the locked resolution +# as it stands instead of re-resolving it at build time. --no-editable installs +# workspace members and path sources as packages in the venv rather than links +# to builder-stage sources, which the runtime stage does not carry. +# +# A project whose requires-python the base image's Python does not satisfy gets +# the interpreter uv installs for it, as on the host. It is installed where the +# runtime stage copies it from, so the venv's interpreter exists there too. +ENV UV_PROJECT_ENVIRONMENT=/app/.venv \ + UV_LINK_MODE=copy \ + UV_PYTHON_INSTALL_DIR=/opt/uv/python +# +# With the caller's mirrors, git reads each locked repository from its mirror +# instead of the URL the lock records (uv hands git that URL unchanged), so no +# credential is needed, and a commit missing from the mirror fails here rather +# than prompting for one. The mappings are appended after the git configuration +# the environment already carries, when its GIT_CONFIG_COUNT is a plain count; +# otherwise the configuration is left alone and the sources are fetched as +# without a mirror. An image whose own git configuration (system, global or +# environment) already rewrites a declared repository's URL keeps its rewrite: +# git prefers it, and the build fetches that source as it does without a mirror. +RUN --mount=type=bind,from=uvsources,target=/uvsources \ + if [ -n "$(ls -A /uvsources)" ]; then \ + n=${GIT_CONFIG_COUNT:-0}; n=${n#+}; \ + case $n in ''|*[!0-9]*) n= ;; *) while [ "${n#0}" != "$n" ] && [ -n "${n#0}" ]; do n=${n#0}; done; [ ${#n} -le 4 ] || n= ;; esac; \ + if [ -z "$n" ]; then echo "inherited GIT_CONFIG_COUNT is not a plain count; git sources are fetched without the mirror"; \ + else export GIT_TERMINAL_PROMPT=0 \ + "GIT_CONFIG_KEY_$((n+0))=url.file:///uvsources/c9fa8363b851c2643d648edad1c8c7892904f717385511fe4771422d0817a55e.git.insteadOf" "GIT_CONFIG_VALUE_$((n+0))="'https://git.example.invalid/acme/mono' \ + "GIT_CONFIG_KEY_$((n+1))=url.file:///uvsources/d2b987abe743521016c33e82dba28324380629b9d2544f9e5213ee4299a3af6c.git.insteadOf" "GIT_CONFIG_VALUE_$((n+1))="'ssh://git@git.example.invalid:2222/acme/tools.git' \ + GIT_CONFIG_COUNT=$((n+2)); fi; \ + fi && \ + mkdir -p /opt/uv/python && uv sync --frozen --no-dev --no-install-project --no-editable + + +# Runtime +FROM BUILDER_IMAGE as runtime + +WORKDIR /app + +# UID pinned to match runAsUser in the kustomize Deployment: Kubernetes' +# runAsNonRoot check rejects a container whose image user is non-numeric. +RUN adduser -D -u 1000 appuser +USER appuser + +ENV VIRTUAL_ENV=/app/.venv \ + PATH="/app/.venv/bin:$PATH" + +COPY --from=builder /opt/uv/python /opt/uv/python +COPY --from=builder ${VIRTUAL_ENV} ${VIRTUAL_ENV} + +COPY --chown=appuser service.codefly.yaml . + + +COPY --chown=appuser code/src code/src + + + + +WORKDIR /app/code + +# uvicorn's logging configuration and the filter it names (uvicorn_logging.go): +# health probes are dropped from the access log, everything else is as uvicorn +# writes it. The builder writes both into the project before the build. +COPY --chown=appuser code/uvicorn-log-config.json code/codefly_uvicorn_logging.py ./ + +# Default REST port (overridden by codefly at runtime). +CMD ["uvicorn", "src.main:app", "--host", "0.0.0.0", "--port", "8080", "--log-config", "uvicorn-log-config.json"] diff --git a/testdata/dockerfile/no-git-sources.golden b/testdata/dockerfile/no-git-sources.golden new file mode 100644 index 0000000..c6b44d1 --- /dev/null +++ b/testdata/dockerfile/no-git-sources.golden @@ -0,0 +1,62 @@ +# The builder image — installs the venv via uv, then we copy it into the runtime. +FROM BUILDER_IMAGE as builder + +# uv is part of what the recipe has to state. A base image carries whichever uv +# it happened to be built with, so a project written against a newer `[tool.uv]` +# key resolves on the developer's machine and fails inside the builder stage +# with a TOML parse error naming a field that uv there does not know. Copying a +# pinned uv in makes the tool that reads pyproject.toml and uv.lock a declared +# input of the recipe instead of a property of the base image. +COPY --from=UV_IMAGE /uv /usr/local/bin/uv + +WORKDIR /app + +COPY code/pyproject.toml code/uv.lock ./ + +# Install deps to a project-local .venv. --frozen installs the locked resolution +# as it stands instead of re-resolving it at build time. --no-editable installs +# workspace members and path sources as packages in the venv rather than links +# to builder-stage sources, which the runtime stage does not carry. +# +# A project whose requires-python the base image's Python does not satisfy gets +# the interpreter uv installs for it, as on the host. It is installed where the +# runtime stage copies it from, so the venv's interpreter exists there too. +ENV UV_PROJECT_ENVIRONMENT=/app/.venv \ + UV_LINK_MODE=copy \ + UV_PYTHON_INSTALL_DIR=/opt/uv/python +RUN mkdir -p /opt/uv/python && uv sync --frozen --no-dev --no-install-project --no-editable + + +# Runtime +FROM BUILDER_IMAGE as runtime + +WORKDIR /app + +# UID pinned to match runAsUser in the kustomize Deployment: Kubernetes' +# runAsNonRoot check rejects a container whose image user is non-numeric. +RUN adduser -D -u 1000 appuser +USER appuser + +ENV VIRTUAL_ENV=/app/.venv \ + PATH="/app/.venv/bin:$PATH" + +COPY --from=builder /opt/uv/python /opt/uv/python +COPY --from=builder ${VIRTUAL_ENV} ${VIRTUAL_ENV} + +COPY --chown=appuser service.codefly.yaml . + + +COPY --chown=appuser code/src code/src + + + + +WORKDIR /app/code + +# uvicorn's logging configuration and the filter it names (uvicorn_logging.go): +# health probes are dropped from the access log, everything else is as uvicorn +# writes it. The builder writes both into the project before the build. +COPY --chown=appuser code/uvicorn-log-config.json code/codefly_uvicorn_logging.py ./ + +# Default REST port (overridden by codefly at runtime). +CMD ["uvicorn", "src.main:app", "--host", "0.0.0.0", "--port", "8080", "--log-config", "uvicorn-log-config.json"] diff --git a/uv_git_sources_recipe_test.go b/uv_git_sources_recipe_test.go new file mode 100644 index 0000000..be23b68 --- /dev/null +++ b/uv_git_sources_recipe_test.go @@ -0,0 +1,390 @@ +package main + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/codefly-dev/core/agents/services" + builderv0 "github.com/codefly-dev/core/generated/go/codefly/services/builder/v0" + "github.com/stretchr/testify/require" +) + +const ( + commitA = "8b58bb4786656900153af99dd758fc496b9c3c96" + commitB = "90db9b742baa9c07532cc08a90d58e790b881312" +) + +// gitSourcesLock is a lock in the form uv writes it: two packages built from +// subdirectories of one repository at one commit, the same repository at a +// second commit, an ssh source with a port, sources a recipe cannot declare +// (http, file, a credential, percent-encoding, "@" in the path), and a +// registry package. +var gitSourcesLock = fmt.Sprintf(`version = 1 +requires-python = ">=3.12" + +[[package]] +name = "liba" +version = "0.1.0" +source = { git = "https://git.example.invalid/acme/mono?subdirectory=liba&rev=%[1]s#%[1]s" } + +[[package]] +name = "libb" +version = "0.1.0" +source = { git = "https://git.example.invalid/acme/mono?subdirectory=libb&rev=%[1]s#%[1]s" } + +[[package]] +name = "libc" +version = "0.2.0" +source = { git = "https://git.example.invalid/acme/mono?subdirectory=libc&rev=%[2]s#%[2]s" } + +[[package]] +name = "libd" +version = "0.1.0" +source = { git = "ssh://git@git.example.invalid:2222/acme/tools.git?rev=main#%[2]s" } + +[[package]] +name = "libe" +version = "0.1.0" +source = { git = "http://token@git.example.invalid/acme/plain?rev=main#%[1]s" } + +[[package]] +name = "libf" +version = "0.1.0" +source = { git = "file:///srv/local?rev=main#%[1]s" } + +[[package]] +name = "libg" +version = "0.1.0" +source = { git = "https://TOKEN@git.example.invalid/acme/secret?rev=main#%[1]s" } + +[[package]] +name = "libh" +version = "0.1.0" +source = { git = "https://git.example.invalid/Team%%20Project/repo?rev=main#%[1]s" } + +[[package]] +name = "libi" +version = "0.1.0" +source = { git = "https://git.example.invalid/acme/lib@v1?rev=main#%[1]s" } + +[[package]] +name = "six" +version = "1.17.0" +source = { registry = "https://pypi.org/simple" } +`, commitA, commitB) + +var gitSourcesDeclared = []*builderv0.GitSourceDownload{ + {Repository: "https://git.example.invalid/acme/mono", Commit: commitA, MirrorContext: "uvsources"}, + {Repository: "https://git.example.invalid/acme/mono", Commit: commitB, MirrorContext: "uvsources"}, + {Repository: "ssh://git@git.example.invalid:2222/acme/tools.git", Commit: commitB, MirrorContext: "uvsources"}, +} + +func requireDeclares(t *testing.T, want, got []*builderv0.GitSourceDownload) { + t.Helper() + require.Len(t, got, len(want)) + for i := range want { + require.Equal(t, want[i].String(), got[i].String()) + } +} + +// TestLockedGitSourcesDeclaresEachLockedCommitOnce reads the URL uv hands git +// (the recorded one up to its query) and the locked commit, sorted, with a +// repository at one commit declared once however many packages it builds. The +// http and file sources are left out, named by package for the warning, and +// left to the build to fetch as before. +func TestLockedGitSourcesDeclaresEachLockedCommitOnce(t *testing.T) { + lock := filepath.Join(t.TempDir(), "uv.lock") + writeFile(t, lock, gitSourcesLock) + declared, unmirrored, err := lockedGitSources(lock) + require.NoError(t, err) + requireDeclares(t, gitSourcesDeclared, declared) + require.Equal(t, []unmirroredSource{ + {Package: "libe", Reason: "repository must start with https:// or ssh://"}, + {Package: "libf", Reason: "repository must start with https:// or ssh://"}, + {Package: "libg", Reason: "repository must carry no user information over https"}, + {Package: "libh", Reason: "repository must carry no percent-encoding"}, + {Package: "libi", Reason: "repository must carry no @ in the path"}, + }, unmirrored) + + mirrors := uvGitMirrors(declared) + require.Len(t, mirrors, 2, "one mirror per repository") + require.Equal(t, services.GitSourceMirrorPath("https://git.example.invalid/acme/mono"), mirrors[0].Path) +} + +// TestLockedGitSourcesRefusesAnUnpinnedSource: uv always records the full +// commit id, so a git source without one is a malformed lock. The error names +// the package, never the URL, which may carry a credential. +func TestLockedGitSourcesRefusesAnUnpinnedSource(t *testing.T) { + for _, source := range []string{"https://token@git.example.invalid/acme/mono?rev=main", "https://token@git.example.invalid/acme/mono#8b58bb4"} { + lock := filepath.Join(t.TempDir(), "uv.lock") + writeFile(t, lock, fmt.Sprintf("version = 1\n\n[[package]]\nname = \"liba\"\nversion = \"0.1.0\"\nsource = { git = %q }\n", source)) + _, _, err := lockedGitSources(lock) + require.ErrorContains(t, err, `package "liba"`) + require.NotContains(t, err.Error(), "token") + } +} + +// TestBuildDeclaresTheLockedGitSources holds the recipe: exactly the lock's git +// sources, the v6 contract, and the mirror mapping in the Dockerfile; without +// git sources, no declaration, the contract and Dockerfile of before. +func TestBuildDeclaresTheLockedGitSources(t *testing.T) { + builder, _ := createdBuilder(t) + outputDir := t.TempDir() + plan := emitRecipe(t, builder, outputDir) + require.Empty(t, plan.GetRecipes()[0].GetGitSourceDownloads()) + require.Equal(t, services.DockerBuildRecipeContractVersion, plan.GetContractVersion()) + requireGolden(t, "no-git-sources", dockerfileOf(t, outputDir, plan)) + + writeFile(t, filepath.Join(builder.SourceLocation, "uv.lock"), gitSourcesLock) + outputDir = t.TempDir() + plan = emitRecipe(t, builder, outputDir) + require.NoError(t, services.VerifyDockerBuildPlan(outputDir, plan)) + requireDeclares(t, gitSourcesDeclared, plan.GetRecipes()[0].GetGitSourceDownloads()) + require.Equal(t, services.DockerBuildRecipeGitSourcesContractVersion, plan.GetContractVersion()) + requireGolden(t, "git-sources", dockerfileOf(t, outputDir, plan)) + // What a recipe cannot declare reaches neither the plan nor the Dockerfile. + for _, rendered := range []string{plan.String(), dockerfileOf(t, outputDir, plan)} { + for _, refused := range []string{"TOKEN", "Team", "lib@v1", "acme/plain", "/srv/local"} { + require.NotContains(t, rendered, refused) + } + } +} + +// TestWorkspaceMemberDeclaresTheRootLocksGitSources reads the lock uv syncs a +// workspace member from: the workspace root's. +func TestWorkspaceMemberDeclaresTheRootLocksGitSources(t *testing.T) { + builder, _ := createdBuilder(t) + uvWorkspaceService(t, builder.Location, workspaceRequiresPython) + writeFile(t, filepath.Join(builder.Location, "uv.lock"), gitSourcesLock) + plan := emitRecipe(t, builder, t.TempDir()) + requireDeclares(t, gitSourcesDeclared, plan.GetRecipes()[0].GetGitSourceDownloads()) +} + +// requireGolden compares a rendered Dockerfile, its pinned images replaced, to +// testdata/dockerfile/.golden. The no-git-sources golden is the +// Dockerfile rendered before git sources were declared, byte for byte. +// UPDATE_GOLDEN=1 rewrites them after an intended template change. +func requireGolden(t *testing.T, name, dockerfile string) { + t.Helper() + dockerfile = strings.ReplaceAll(dockerfile, runtimeImage.FullName(), "BUILDER_IMAGE") + dockerfile = strings.ReplaceAll(dockerfile, uvImage, "UV_IMAGE") + path := filepath.Join("testdata", "dockerfile", name+".golden") + if os.Getenv("UPDATE_GOLDEN") != "" { + require.NoError(t, os.WriteFile(path, []byte(dockerfile), 0o644)) + } + golden, err := os.ReadFile(path) + require.NoError(t, err) + require.Equal(t, string(golden), dockerfile) +} + +// run runs a command in dir and fails the test on error. +func run(t *testing.T, dir string, env []string, name string, args ...string) string { + t.Helper() + command := exec.CommandContext(t.Context(), name, args...) + command.Dir = dir + command.Env = append(os.Environ(), env...) + out, err := command.CombinedOutput() + require.NoError(t, err, "%s %v: %s", name, args, out) + return strings.TrimSpace(string(out)) +} + +// TestLockedGitSourceBuildsFromTheSuppliedMirror is the claim end to end. The +// project depends on a package in a subdirectory of a git repository whose URL +// the build cannot reach, standing in for one it has no credential for; the +// host's uv locks it through a host-side rewrite, as a host credential would +// let it. Built with the mirror context the caller supplies, the image carries +// the package; built without it, the build fails on the unreachable URL. +func TestLockedGitSourceBuildsFromTheSuppliedMirror(t *testing.T) { + for _, tool := range []string{"docker", "uv", "git"} { + if _, err := exec.LookPath(tool); err != nil { + t.Skipf("%s is not installed: the git source image cannot be built", tool) + } + } + const repository = "https://git.example.invalid/example/libraries" + work := t.TempDir() + library := filepath.Join(work, "libraries") + writeFile(t, filepath.Join(library, "lib", "pyproject.toml"), `[project] +name = "example-git-lib" +version = "0.3.0" +requires-python = ">=3.10" + +[build-system] +requires = ["uv_build>=0.8,<0.13"] +build-backend = "uv_build" +`) + writeFile(t, filepath.Join(library, "lib", "src", "example_git_lib", "__init__.py"), "") + identity := []string{"GIT_AUTHOR_NAME=test", "GIT_AUTHOR_EMAIL=test@example.com", "GIT_COMMITTER_NAME=test", "GIT_COMMITTER_EMAIL=test@example.com"} + run(t, library, nil, "git", "init", "-q") + run(t, library, nil, "git", "add", "-A") + run(t, library, identity, "git", "commit", "-qm", "library") + commit := run(t, library, nil, "git", "rev-parse", "HEAD") + served := filepath.Join(work, "served.git") + run(t, work, nil, "git", "clone", "-q", "--bare", library, served) + + builder, _ := createdBuilder(t) + writeFile(t, filepath.Join(builder.SourceLocation, "pyproject.toml"), fmt.Sprintf(`[project] +name = "example-service" +version = "0.1.0" +requires-python = ">=3.10" +dependencies = ["example-git-lib"] + +[tool.uv] +package = false + +[tool.uv.sources] +example-git-lib = { git = %q, subdirectory = "lib", rev = %q } +`, repository, commit)) + rewrite := []string{"GIT_CONFIG_COUNT=1", "GIT_CONFIG_KEY_0=url.file://" + served + ".insteadOf", "GIT_CONFIG_VALUE_0=" + repository, "UV_CACHE_DIR=" + filepath.Join(work, "uv-cache")} + run(t, builder.SourceLocation, rewrite, "uv", "lock", "-q") + lock, err := os.ReadFile(filepath.Join(builder.SourceLocation, "uv.lock")) + require.NoError(t, err) + require.Contains(t, string(lock), fmt.Sprintf("%s?subdirectory=lib&rev=%s#%s", repository, commit, commit)) + + outputDir := t.TempDir() + plan := emitRecipe(t, builder, outputDir) + recipe := plan.GetRecipes()[0] + requireDeclares(t, []*builderv0.GitSourceDownload{{Repository: repository, Commit: commit, MirrorContext: "uvsources"}}, recipe.GetGitSourceDownloads()) + + // The mirror context as the caller fills it. + mirrors := filepath.Join(work, "mirrors") + mirror := filepath.Join(mirrors, services.GitSourceMirrorPath(repository)) + run(t, work, nil, "git", "init", "-q", "--bare", mirror) + run(t, mirror, nil, "git", "fetch", "-q", served, "+"+commit+":"+services.GitSourceMirrorRef(commit)) + + tag := fmt.Sprintf("codefly-fastapi-uv-git-source-test:%d", time.Now().UnixMilli()) + build := func(args ...string) ([]byte, error) { + args = append([]string{"build", "-f", filepath.Join(outputDir, recipe.GetDockerfile())}, args...) + return exec.CommandContext(t.Context(), "docker", append(args, builder.Location)...).CombinedOutput() + } + out, err := build("-t", tag, "--build-context", "uvsources="+mirrors) + require.NoError(t, err, "the mirrored git source did not build:\n%s", out) + t.Cleanup(func() { _ = exec.Command("docker", "rmi", "-f", tag).Run() }) + installed, err := exec.CommandContext(t.Context(), "docker", "run", "--rm", tag, + "python", "-c", "import importlib.metadata as m; print(m.version('example-git-lib'))").CombinedOutput() + require.NoError(t, err, "%s", installed) + require.Contains(t, string(installed), "0.3.0", "the image does not carry the git source") + + out, err = build() + require.Error(t, err, "without the mirror the unreachable URL must fail the build:\n%s", out) + require.Contains(t, string(out), "git.example.invalid") +} + +// lockWith is a uv-form lock with one package per git source, all at commitA. +func lockWith(sources ...string) string { + lock := "version = 1\n" + for i, source := range sources { + lock += fmt.Sprintf("\n[[package]]\nname = \"lib%d\"\nversion = \"0.1.0\"\nsource = { git = \"%s?rev=main#%s\" }\n", i, source, commitA) + } + return lock +} + +// TestLockedGitSourcesLeavesOutAPrefixOfAnUndeclaredSource: git's insteadOf +// rewrites every URL that begins with the declared one, so a declared URL +// that begins another locked source's undeclared URL would send that source to +// the wrong mirror. It is left out too, until no such capture remains. +func TestLockedGitSourcesLeavesOutAPrefixOfAnUndeclaredSource(t *testing.T) { + lock := filepath.Join(t.TempDir(), "uv.lock") + writeFile(t, lock, lockWith("https://h.example/acme/mono", "https://h.example/acme/mono@v1", + "https://h.example/acme/x", "https://h.example/acme/xy", "https://h.example/acme/xyz@v1", "https://h.example/acme/kept")) + declared, unmirrored, err := lockedGitSources(lock) + require.NoError(t, err) + requireDeclares(t, []*builderv0.GitSourceDownload{{Repository: "https://h.example/acme/kept", Commit: commitA, MirrorContext: "uvsources"}}, declared) + var names []string + for _, source := range unmirrored { + names = append(names, source.Package) + } + require.ElementsMatch(t, []string{"lib0", "lib1", "lib2", "lib3", "lib4"}, names) + require.Contains(t, unmirrored, unmirroredSource{Package: "lib0", Reason: prefixCaptureReason}) + require.Contains(t, unmirrored, unmirroredSource{Package: "lib2", Reason: prefixCaptureReason}) +} + +// TestLockedGitSourcesKeepsALongerDeclaredSource: git applies the longest +// matching insteadOf, so two declared URLs, one beginning the other, each +// reach their own mirror. +func TestLockedGitSourcesKeepsALongerDeclaredSource(t *testing.T) { + lock := filepath.Join(t.TempDir(), "uv.lock") + writeFile(t, lock, lockWith("https://h.example/acme/mono", "https://h.example/acme/mono.git")) + declared, unmirrored, err := lockedGitSources(lock) + require.NoError(t, err) + require.Empty(t, unmirrored) + require.Len(t, declared, 2) +} + +// runMapping renders the recipe for lock and runs its mirror mapping, then +// script, in the builder base image with env set, as a base image or an +// earlier ENV may set it. +func runMapping(t *testing.T, lock string, env []string, script string) string { + t.Helper() + builder, _ := createdBuilder(t) + writeFile(t, filepath.Join(builder.SourceLocation, "uv.lock"), lock) + outputDir := t.TempDir() + dockerfile := dockerfileOf(t, outputDir, emitRecipe(t, builder, outputDir)) + run := "RUN --mount=type=bind,from=uvsources,target=/uvsources \\\n" + start := strings.Index(dockerfile, run) + end := strings.Index(dockerfile, "mkdir -p /opt/uv/python && uv sync") + require.True(t, start >= 0 && end > start, "the rendered sync step has no mirror mapping") + mapping := strings.ReplaceAll(dockerfile[start+len(run):end], "\\\n", "") + + mirrors := t.TempDir() + writeFile(t, filepath.Join(mirrors, "present"), "") + args := []string{"run", "--rm", "--network=none", "-w", "/tmp", "-v", mirrors + ":/uvsources:ro"} + for _, variable := range env { + args = append(args, "-e", variable) + } + out, err := exec.CommandContext(t.Context(), "docker", append(args, runtimeImage.FullName(), "sh", "-c", mapping+script)...).CombinedOutput() + require.NoError(t, err, "%s", out) + return string(out) +} + +// TestGitSourceMappingInTheBuilderImage runs the rendered mapping where it +// runs: appended after the git configuration the environment carries when its +// count is plain, leaving that configuration alone otherwise, and never +// rewriting a source left out of the declaration. +func TestGitSourceMappingInTheBuilderImage(t *testing.T) { + if _, err := exec.LookPath("docker"); err != nil { + t.Skip("docker is not installed: the mapping cannot be run") + } + const mono = "https://git.example.invalid/acme/mono" + mirror := "file:///uvsources/" + services.GitSourceMirrorPath(mono) + inherited := []string{"GIT_CONFIG_KEY_0=user.name", "GIT_CONFIG_VALUE_0=inherited"} + for _, count := range []string{"", "1", "01", "+1"} { + t.Run("count "+count, func(t *testing.T) { + env := append([]string{"GIT_CONFIG_COUNT=" + count}, inherited...) + if count == "" { + env = nil + } + out := runMapping(t, gitSourcesLock, env, "git ls-remote --get-url "+mono+" && { git config --get user.name || true; } && echo count=$GIT_CONFIG_COUNT") + want := mirror + "\ninherited\ncount=3\n" + if count == "" { + want = mirror + "\ncount=2\n" + } + require.Equal(t, want, out) + }) + } + for _, count := range []string{" 1", "-0", "99999", "abc"} { + t.Run("count "+count, func(t *testing.T) { + env := append([]string{"GIT_CONFIG_COUNT=" + count}, inherited...) + out := runMapping(t, gitSourcesLock, env, `echo "count=[$GIT_CONFIG_COUNT] keys=$(env | grep -c '^GIT_CONFIG_KEY_') prompt=[$GIT_TERMINAL_PROMPT]"`) + require.Equal(t, "inherited GIT_CONFIG_COUNT is not a plain count; git sources are fetched without the mirror\ncount=["+count+"] keys=1 prompt=[]\n", out) + }) + } + // The limitation, as it is today without a mirror: an image whose own git + // configuration rewrites a declared URL keeps its rewrite, which git + // prefers, so the source is fetched where the image says, not broken. + t.Run("inherited rewrite of a declared URL", func(t *testing.T) { + out := runMapping(t, gitSourcesLock, []string{"GIT_CONFIG_COUNT=1", + "GIT_CONFIG_KEY_0=url.https://proxy.invalid/.insteadOf", "GIT_CONFIG_VALUE_0=" + mono}, + "git ls-remote --get-url "+mono+" && echo count=$GIT_CONFIG_COUNT") + require.Equal(t, "https://proxy.invalid/\ncount=3\n", out) + }) + t.Run("an undeclared source the declared URL begins", func(t *testing.T) { + out := runMapping(t, lockWith(mono, mono+"@v1", "https://git.example.invalid/acme/tools"), nil, + "git ls-remote --get-url "+mono+"@v1 && git ls-remote --get-url "+mono) + require.Equal(t, mono+"@v1\n"+mono+"\n", out) + }) +} diff --git a/uvgitsources.go b/uvgitsources.go new file mode 100644 index 0000000..567fed4 --- /dev/null +++ b/uvgitsources.go @@ -0,0 +1,175 @@ +package main + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "sort" + "strings" + + "github.com/codefly-dev/core/agents/services" + builderv0 "github.com/codefly-dev/core/generated/go/codefly/services/builder/v0" + "github.com/pelletier/go-toml/v2" +) + +// A git dependency that needs a credential (a private repository) resolves on +// the developer's machine, whose git has the credential, and fails in the +// builder stage, which has none. The recipe therefore declares every git +// commit the lock pins (DockerBuildRecipe.git_source_downloads): the caller +// fetches each on its own machine before the build and supplies them as the +// uvsourcesContext build context, one bare mirror per repository, and the +// builder stage points git at those mirrors. uv.lock is read, never rewritten. +// +// uv records a git source as `source = { git = "?#" }` and +// asks git for the commit from by the refspec `+:refs/commit/`, +// where is the recorded string up to its query, byte for byte: uv has +// already lowercased the scheme and host and dropped a `git+` prefix when it +// wrote the lock, and keeps a `.git` suffix, a trailing slash, a port and an +// ssh login as they are. git's `url..insteadOf=` matches that +// string by prefix, so the declaration carries it exactly; a normalized form +// would miss the mirror and silently go to the network. + +// uvSourcesContext is the build context the Dockerfile reads the caller's git +// mirrors from (the `uvsources` stage in the template). +const uvSourcesContext = "uvsources" + +// uvGitMirror is one repository the builder stage maps onto its mirror. +type uvGitMirror struct { + // Path is the mirror's directory inside the uvsources context. + Path string + // Repository is the URL as uv hands it to git. + Repository string +} + +// unmirroredSource is a locked git source the recipe does not declare. +type unmirroredSource struct { + Package string + // Reason is core's, or the prefix capture; neither quotes the URL. + Reason string +} + +// uvLockPath is the lock uv syncs the project from: the workspace root's for a +// member of a workspace in the service, the project's own otherwise. +func uvLockPath(serviceDir, sourceDir string, membership *uvWorkspaceMembership) string { + if membership != nil { + return filepath.Join(serviceDir, filepath.FromSlash(membership.Root), "uv.lock") + } + return filepath.Join(sourceDir, "uv.lock") +} + +// lockedGitSources returns the git commits the lock at lockPath pins, sorted +// by repository and commit, each (repository, commit) once: packages built +// from subdirectories of one repository at one commit share a declaration. A +// project with no lock declares nothing; its build fails on the missing lock as +// it always has. +// +// A source core would refuse to declare (services.GitSourceRepositoryProblem: +// http, file, a credential, percent-encoding, "@" in the path, ...) is +// returned as unmirrored, with core's reason, and is neither declared nor +// rendered: the build fetches it itself, as it always has. So is a source +// whose URL begins another locked source's undeclared URL, because git's +// insteadOf rewrites by prefix and would send that source to this mirror. A +// source whose commit is missing or not a full object id makes the lock +// malformed. +func lockedGitSources(lockPath string) (downloads []*builderv0.GitSourceDownload, unmirrored []unmirroredSource, err error) { + content, err := os.ReadFile(lockPath) + if errors.Is(err, os.ErrNotExist) { + return nil, nil, nil + } + if err != nil { + return nil, nil, err + } + var lock struct { + Package []struct { + Name string `toml:"name"` + Source struct { + Git string `toml:"git"` + } `toml:"source"` + } `toml:"package"` + } + if err := toml.Unmarshal(content, &lock); err != nil { + return nil, nil, fmt.Errorf("parse %s: %w", lockPath, err) + } + type accepted struct{ name, repository, commit string } + var candidates []accepted + var locked []string + for _, pkg := range lock.Package { + if pkg.Source.Git == "" { + continue + } + // The URL is not quoted in errors: a lock written from a source that + // named a credential may still carry one. + source, commit, found := strings.Cut(pkg.Source.Git, "#") + if !found || !services.IsGitSourceCommit(commit) { + return nil, nil, fmt.Errorf("%s: package %q is locked to a git source without a full commit id: run `uv lock`", lockPath, pkg.Name) + } + repository, _, _ := strings.Cut(source, "?") + locked = append(locked, repository) + if reason := services.GitSourceRepositoryProblem(repository); reason != "" { + unmirrored = append(unmirrored, unmirroredSource{Package: pkg.Name, Reason: "repository " + reason}) + continue + } + candidates = append(candidates, accepted{pkg.Name, repository, commit}) + } + // Leave out every declarable URL that begins an undeclared one, until none + // does: leaving one out can make it the undeclared URL a shorter one begins. + for changed := true; changed; { + changed = false + declared := map[string]bool{} + for _, candidate := range candidates { + declared[candidate.repository] = true + } + kept := candidates[:0] + for _, candidate := range candidates { + captures := false + for _, other := range locked { + if !declared[other] && strings.HasPrefix(other, candidate.repository) { + captures = true + break + } + } + if captures { + unmirrored = append(unmirrored, unmirroredSource{Package: candidate.name, Reason: prefixCaptureReason}) + changed = true + continue + } + kept = append(kept, candidate) + } + candidates = kept + } + seen := map[[2]string]struct{}{} + for _, candidate := range candidates { + key := [2]string{candidate.repository, candidate.commit} + if _, ok := seen[key]; ok { + continue + } + seen[key] = struct{}{} + downloads = append(downloads, &builderv0.GitSourceDownload{Repository: candidate.repository, Commit: candidate.commit, MirrorContext: uvSourcesContext}) + } + sort.Slice(downloads, func(i, j int) bool { + if downloads[i].GetRepository() != downloads[j].GetRepository() { + return downloads[i].GetRepository() < downloads[j].GetRepository() + } + return downloads[i].GetCommit() < downloads[j].GetCommit() + }) + return downloads, unmirrored, nil +} + +// uvGitMirrors is each declared repository once, in declaration order, with +// the mirror the caller fetches it into. +func uvGitMirrors(downloads []*builderv0.GitSourceDownload) []uvGitMirror { + var mirrors []uvGitMirror + for _, download := range downloads { + repository := download.GetRepository() + if len(mirrors) > 0 && mirrors[len(mirrors)-1].Repository == repository { + continue + } + mirrors = append(mirrors, uvGitMirror{Path: services.GitSourceMirrorPath(repository), Repository: repository}) + } + return mirrors +} + +// prefixCaptureReason is why a declarable source is left out when git's +// prefix rewrite for it would also capture an undeclared locked source. +const prefixCaptureReason = "another locked source's URL begins with it, and git's rewrite would capture that source"