From f3a47be6597608aa8b39424015fc12fd40faa074 Mon Sep 17 00:00:00 2001 From: Saurabh Maheshwari <242053479+saurabhmaheshwari-obin@users.noreply.github.com> Date: Wed, 21 Oct 2026 17:25:46 -0400 Subject: [PATCH 01/11] build(deps): pin core to the git source downloads change Pins github.com/codefly-dev/core to the open change that adds DockerBuildRecipe.git_source_downloads (contract v6), by pseudo-version, until it is released; the pin moves to the core tag, together with the shared workflow in ci.yml, after the release. Co-Authored-By: Claude Opus 5.5 --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 6a95443..3d1707b 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/codefly-dev/service-python-fastapi go 1.27.2 require ( - github.com/codefly-dev/core v0.19.4 + github.com/codefly-dev/core v0.19.6-0.20261021204332-519efae17f9e github.com/codefly-dev/service-python v0.0.66 github.com/pelletier/go-toml/v2 v2.4.3 github.com/stretchr/testify v1.12.1 diff --git a/go.sum b/go.sum index 8c49f7c..bcf6da7 100644 --- a/go.sum +++ b/go.sum @@ -43,8 +43,8 @@ github.com/clipperhouse/uax29/v2 v2.7.0 h1:+gs4oBZ2gPfVrKPthwbMzWZDaAFPGYK72F0NJ github.com/clipperhouse/uax29/v2 v2.7.0/go.mod h1:EFJ2TJMRUaplDxHKj1qAEhCtQPW2tJSwu5BF98AuoVM= github.com/cloudflare/circl v1.6.5 h1:O64F26HEqNhznd/hrC5KZXVKYuKM2rx4deZDTc4ihQA= github.com/cloudflare/circl v1.6.5/go.mod h1:h5LNyxAc5nTue9DS5jT+48en2PSDYt3zdGnz5OstK6c= -github.com/codefly-dev/core v0.19.4 h1:vUvaFrWkB8B3ksQY/qlgBTVj3jMNhdEpJca8aofaals= -github.com/codefly-dev/core v0.19.4/go.mod h1:0uX/7lTBVaGV0RI2Mb3mpEr/PPdx/jZqXHym/U8FVDI= +github.com/codefly-dev/core v0.19.6-0.20261021204332-519efae17f9e h1:fbASQDxSfXFKR8CL6IwF2YJCmawmsKNc80irNZSaho4= +github.com/codefly-dev/core v0.19.6-0.20261021204332-519efae17f9e/go.mod h1:0uX/7lTBVaGV0RI2Mb3mpEr/PPdx/jZqXHym/U8FVDI= github.com/codefly-dev/gortk v0.2.0 h1:7bOlS5valYz2zil+fZctQNcPCYBcPj86abcw9N8h1hQ= github.com/codefly-dev/gortk v0.2.0/go.mod h1:dDWUMFgAP063OCGhTEpV7FFUj9+zTcxxO/nV80VKEwg= github.com/codefly-dev/service-python v0.0.66 h1:Y74QvKAYMaOnZcsfRxuD9dpQo3//7Mjo12RYELyrp5o= From 394d7bc373db4647e02962586ee67022b9c53aa0 Mon Sep 17 00:00:00 2001 From: Saurabh Maheshwari <242053479+saurabhmaheshwari-obin@users.noreply.github.com> Date: Wed, 21 Oct 2026 17:25:46 -0400 Subject: [PATCH 02/11] feat(builder): declare uv.lock git sources for the caller to mirror A git dependency that needs a credential resolves on the host and fails in the builder stage, which has none. The recipe now declares every git commit uv.lock pins (git_source_downloads, mirror context "uvsources"): the caller fetches them on its own machine before the build and supplies one bare mirror per repository, and the builder stage maps each locked URL onto its mirror with url..insteadOf before the unchanged uv sync --frozen. The declared repository is the lock's URL up to its query, which is the string uv hands git, so the insteadOf match is exact. The workspace root's lock is read for a workspace member. A lock with no git sources renders the Dockerfile and plan contract it rendered before, and a plain docker build without the context fetches from the network as before. Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 5 + builder.go | 22 +- templates/builder/Dockerfile.tmpl | 21 ++ testdata/dockerfile/git-sources.golden | 78 +++++++ testdata/dockerfile/no-git-sources.golden | 62 ++++++ uv_git_sources_recipe_test.go | 238 ++++++++++++++++++++++ uvgitsources.go | 122 +++++++++++ 7 files changed, 545 insertions(+), 3 deletions(-) create mode 100644 testdata/dockerfile/git-sources.golden create mode 100644 testdata/dockerfile/no-git-sources.golden create mode 100644 uv_git_sources_recipe_test.go create mode 100644 uvgitsources.go diff --git a/AGENTS.md b/AGENTS.md index 2ed90c3..35b5292 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -126,6 +126,7 @@ anything from it. | `runtime.go` | Init/Start/Stop/Destroy, uvicorn supervision, hot reload, the lock order | | `builder.go` | `Create` scaffolding, OpenAPI endpoint loading, Docker + kustomize recipes | | `uvsources.go` | the assembled build context: `[tool.uv.sources]` paths leaving the project | +| `uvgitsources.go` | the git commits `uv.lock` pins, declared for the caller to mirror into the build | | `probes.go`, `probe.py` | endpoint health translated to Kubernetes probes / a local probe | | `templates/` | **what ships** — embedded via `go:embed`, rendered on `Create` | | `base/` | a checked-in sample service, embedded nowhere | @@ -158,6 +159,10 @@ into `templates/factory/code/pyproject.toml.tmpl` yourself. same relative path resolves there; that plan claims the whole tree (`RECIPE_INVENTORY_SCOPE_TREE`), the Dockerfile-only one claims just what it emitted. Never rewrite `pyproject.toml` or `uv.lock` to make a path resolve. + A git source is declared in the recipe (`git_source_downloads`) by the URL + exactly as `uv.lock` records it up to its query, which is what uv hands git; + the caller fetches it with its own credentials and the Dockerfile maps that + URL onto the mirror. A normalized URL would miss the mirror silently. - **The runtime image is pinned.** `docker-image` rejects a bare name and `:latest`. The default `ghcr.io/codefly-dev/python` companion is built from core and pinned by the digest of its multi-platform index — prefer it. Its tag diff --git a/builder.go b/builder.go index d1bca72..ef56ee8 100644 --- a/builder.go +++ b/builder.go @@ -192,6 +192,9 @@ type DockerTemplating struct { // IgnorePatterns are the dockerignore entries of a recipe that copies the // service tree whole. IgnorePatterns []string + // GitSources are the repositories of the git commits the lock pins, which + // the builder stage reads from the caller's mirrors (uvgitsources.go). + GitSources []uvGitMirror } func (s *Builder) BuildCapabilities(context.Context, *builderv0.BuildCapabilitiesRequest) (*builderv0.BuildCapabilitiesResponse, error) { @@ -248,11 +251,20 @@ func (s *Builder) Build(ctx context.Context, req *builderv0.BuildRequest) (*buil if err != nil { return s.Base.Builder.BuildError(err) } + gitSources, err := lockedGitSources(uvLockPath(s.Location, s.SourceLocation, membership)) + if err != nil { + return s.Base.Builder.BuildError(err) + } + var options []services.RecipeOption + if len(gitSources) > 0 { + options = append(options, services.WithGitSourceDownloads(gitSources...)) + } docker := DockerTemplating{ Builder: runtimeImage.FullName(), Components: components, UVImage: uvImage, ProjectWorkdir: defaultProjectWorkdir, + GitSources: uvGitMirrors(gitSources), } if assembled != nil { docker.ServicePrefix = assembled.ServicePrefix @@ -272,19 +284,23 @@ func (s *Builder) Build(ctx context.Context, req *builderv0.BuildRequest) (*buil // Only the Dockerfile and its ignore were written, and the application's // inputs are the service's own tree: the plan claims what it emitted and // the caller builds the service directory. - return s.Base.Builder.SingleImageBuildResponse(req, image, emitted) + return s.Base.Builder.SingleImageBuildResponse(req, image, emitted, options...) } // The whole destination was assembled by this build, so the plan claims all // of it and the caller builds the assembled tree rather than the service // directory — which is what RECIPE_INVENTORY_SCOPE_TREE declares. - plan, err := services.BuildDockerBuildPlan(outputDir, []*builderv0.DockerBuildRecipe{{ + recipe := &builderv0.DockerBuildRecipe{ Name: "app", Dockerfile: "Dockerfile", Context: ".", Dockerignore: dockerignoreOf(emitted), Image: image, Platforms: services.RecipeBuildPlatforms(), - }}) + } + for _, option := range options { + option(recipe) + } + plan, err := services.BuildDockerBuildPlan(outputDir, []*builderv0.DockerBuildRecipe{recipe}) if err != nil { return s.Base.Builder.BuildError(err) } diff --git a/templates/builder/Dockerfile.tmpl b/templates/builder/Dockerfile.tmpl index 7424678..b459001 100644 --- a/templates/builder/Dockerfile.tmpl +++ b/templates/builder/Dockerfile.tmpl @@ -1,3 +1,12 @@ +{{if .GitSources -}} +# The git commits uv.lock pins (git_source_downloads in the recipe), fetched by +# the caller on its own machine, with its own credentials, before any image +# build, and supplied here as the build context of this name: one bare mirror +# per repository. Without a caller supplying it, the stage is empty and uv +# fetches them from the network as it always did. +FROM scratch AS uvsources + +{{end -}} # The builder image — installs the venv via uv, then we copy it into the runtime. FROM {{.Builder}} as builder @@ -43,7 +52,19 @@ COPY {{.Context}} {{.Image}} ENV UV_PROJECT_ENVIRONMENT=/app/.venv \ UV_LINK_MODE=copy \ UV_PYTHON_INSTALL_DIR=/opt/uv/python +{{- if .GitSources}} +# +# With the caller's mirrors, git reads each locked repository from its mirror +# instead of the URL the lock records (uv hands git that URL unchanged), so no +# credential is needed, and a commit missing from the mirror fails here rather +# than prompting for one. +RUN --mount=type=bind,from=uvsources,target=/uvsources \ + if [ -n "$(ls -A /uvsources)" ]; then export GIT_TERMINAL_PROMPT=0 GIT_CONFIG_COUNT={{len .GitSources}}{{range $i, $source := .GitSources}} \ + GIT_CONFIG_KEY_{{$i}}='url.file:///uvsources/{{$source.Path}}.insteadOf' GIT_CONFIG_VALUE_{{$i}}='{{$source.Repository}}'{{end}}; fi && \ + mkdir -p /opt/uv/python && uv sync --frozen --no-dev --no-install-project --no-editable +{{- else}} RUN mkdir -p /opt/uv/python && uv sync --frozen --no-dev --no-install-project --no-editable +{{- end}} # Runtime diff --git a/testdata/dockerfile/git-sources.golden b/testdata/dockerfile/git-sources.golden new file mode 100644 index 0000000..5b5308c --- /dev/null +++ b/testdata/dockerfile/git-sources.golden @@ -0,0 +1,78 @@ +# The git commits uv.lock pins (git_source_downloads in the recipe), fetched by +# the caller on its own machine, with its own credentials, before any image +# build, and supplied here as the build context of this name: one bare mirror +# per repository. Without a caller supplying it, the stage is empty and uv +# fetches them from the network as it always did. +FROM scratch AS uvsources + +# The builder image — installs the venv via uv, then we copy it into the runtime. +FROM BUILDER_IMAGE as builder + +# uv is part of what the recipe has to state. A base image carries whichever uv +# it happened to be built with, so a project written against a newer `[tool.uv]` +# key resolves on the developer's machine and fails inside the builder stage +# with a TOML parse error naming a field that uv there does not know. Copying a +# pinned uv in makes the tool that reads pyproject.toml and uv.lock a declared +# input of the recipe instead of a property of the base image. +COPY --from=UV_IMAGE /uv /usr/local/bin/uv + +WORKDIR /app + +COPY code/pyproject.toml code/uv.lock ./ + +# Install deps to a project-local .venv. --frozen installs the locked resolution +# as it stands instead of re-resolving it at build time. --no-editable installs +# workspace members and path sources as packages in the venv rather than links +# to builder-stage sources, which the runtime stage does not carry. +# +# A project whose requires-python the base image's Python does not satisfy gets +# the interpreter uv installs for it, as on the host. It is installed where the +# runtime stage copies it from, so the venv's interpreter exists there too. +ENV UV_PROJECT_ENVIRONMENT=/app/.venv \ + UV_LINK_MODE=copy \ + UV_PYTHON_INSTALL_DIR=/opt/uv/python +# +# With the caller's mirrors, git reads each locked repository from its mirror +# instead of the URL the lock records (uv hands git that URL unchanged), so no +# credential is needed, and a commit missing from the mirror fails here rather +# than prompting for one. +RUN --mount=type=bind,from=uvsources,target=/uvsources \ + if [ -n "$(ls -A /uvsources)" ]; then export GIT_TERMINAL_PROMPT=0 GIT_CONFIG_COUNT=2 \ + GIT_CONFIG_KEY_0='url.file:///uvsources/c9fa8363b851c2643d648edad1c8c7892904f717385511fe4771422d0817a55e.git.insteadOf' GIT_CONFIG_VALUE_0='https://git.example.invalid/acme/mono' \ + GIT_CONFIG_KEY_1='url.file:///uvsources/d2b987abe743521016c33e82dba28324380629b9d2544f9e5213ee4299a3af6c.git.insteadOf' GIT_CONFIG_VALUE_1='ssh://git@git.example.invalid:2222/acme/tools.git'; fi && \ + mkdir -p /opt/uv/python && uv sync --frozen --no-dev --no-install-project --no-editable + + +# Runtime +FROM BUILDER_IMAGE as runtime + +WORKDIR /app + +# UID pinned to match runAsUser in the kustomize Deployment: Kubernetes' +# runAsNonRoot check rejects a container whose image user is non-numeric. +RUN adduser -D -u 1000 appuser +USER appuser + +ENV VIRTUAL_ENV=/app/.venv \ + PATH="/app/.venv/bin:$PATH" + +COPY --from=builder /opt/uv/python /opt/uv/python +COPY --from=builder ${VIRTUAL_ENV} ${VIRTUAL_ENV} + +COPY --chown=appuser service.codefly.yaml . + + +COPY --chown=appuser code/src code/src + + + + +WORKDIR /app/code + +# uvicorn's logging configuration and the filter it names (uvicorn_logging.go): +# health probes are dropped from the access log, everything else is as uvicorn +# writes it. The builder writes both into the project before the build. +COPY --chown=appuser code/uvicorn-log-config.json code/codefly_uvicorn_logging.py ./ + +# Default REST port (overridden by codefly at runtime). +CMD ["uvicorn", "src.main:app", "--host", "0.0.0.0", "--port", "8080", "--log-config", "uvicorn-log-config.json"] diff --git a/testdata/dockerfile/no-git-sources.golden b/testdata/dockerfile/no-git-sources.golden new file mode 100644 index 0000000..c6b44d1 --- /dev/null +++ b/testdata/dockerfile/no-git-sources.golden @@ -0,0 +1,62 @@ +# The builder image — installs the venv via uv, then we copy it into the runtime. +FROM BUILDER_IMAGE as builder + +# uv is part of what the recipe has to state. A base image carries whichever uv +# it happened to be built with, so a project written against a newer `[tool.uv]` +# key resolves on the developer's machine and fails inside the builder stage +# with a TOML parse error naming a field that uv there does not know. Copying a +# pinned uv in makes the tool that reads pyproject.toml and uv.lock a declared +# input of the recipe instead of a property of the base image. +COPY --from=UV_IMAGE /uv /usr/local/bin/uv + +WORKDIR /app + +COPY code/pyproject.toml code/uv.lock ./ + +# Install deps to a project-local .venv. --frozen installs the locked resolution +# as it stands instead of re-resolving it at build time. --no-editable installs +# workspace members and path sources as packages in the venv rather than links +# to builder-stage sources, which the runtime stage does not carry. +# +# A project whose requires-python the base image's Python does not satisfy gets +# the interpreter uv installs for it, as on the host. It is installed where the +# runtime stage copies it from, so the venv's interpreter exists there too. +ENV UV_PROJECT_ENVIRONMENT=/app/.venv \ + UV_LINK_MODE=copy \ + UV_PYTHON_INSTALL_DIR=/opt/uv/python +RUN mkdir -p /opt/uv/python && uv sync --frozen --no-dev --no-install-project --no-editable + + +# Runtime +FROM BUILDER_IMAGE as runtime + +WORKDIR /app + +# UID pinned to match runAsUser in the kustomize Deployment: Kubernetes' +# runAsNonRoot check rejects a container whose image user is non-numeric. +RUN adduser -D -u 1000 appuser +USER appuser + +ENV VIRTUAL_ENV=/app/.venv \ + PATH="/app/.venv/bin:$PATH" + +COPY --from=builder /opt/uv/python /opt/uv/python +COPY --from=builder ${VIRTUAL_ENV} ${VIRTUAL_ENV} + +COPY --chown=appuser service.codefly.yaml . + + +COPY --chown=appuser code/src code/src + + + + +WORKDIR /app/code + +# uvicorn's logging configuration and the filter it names (uvicorn_logging.go): +# health probes are dropped from the access log, everything else is as uvicorn +# writes it. The builder writes both into the project before the build. +COPY --chown=appuser code/uvicorn-log-config.json code/codefly_uvicorn_logging.py ./ + +# Default REST port (overridden by codefly at runtime). +CMD ["uvicorn", "src.main:app", "--host", "0.0.0.0", "--port", "8080", "--log-config", "uvicorn-log-config.json"] diff --git a/uv_git_sources_recipe_test.go b/uv_git_sources_recipe_test.go new file mode 100644 index 0000000..be0c97a --- /dev/null +++ b/uv_git_sources_recipe_test.go @@ -0,0 +1,238 @@ +package main + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/codefly-dev/core/agents/services" + builderv0 "github.com/codefly-dev/core/generated/go/codefly/services/builder/v0" + "github.com/stretchr/testify/require" +) + +const ( + commitA = "8b58bb4786656900153af99dd758fc496b9c3c96" + commitB = "90db9b742baa9c07532cc08a90d58e790b881312" +) + +// gitSourcesLock is a lock in the form uv writes it: two packages built from +// subdirectories of one repository at one commit, the same repository at a +// second commit, an ssh source with a port, and a registry package. +var gitSourcesLock = fmt.Sprintf(`version = 1 +requires-python = ">=3.12" + +[[package]] +name = "liba" +version = "0.1.0" +source = { git = "https://git.example.invalid/acme/mono?subdirectory=liba&rev=%[1]s#%[1]s" } + +[[package]] +name = "libb" +version = "0.1.0" +source = { git = "https://git.example.invalid/acme/mono?subdirectory=libb&rev=%[1]s#%[1]s" } + +[[package]] +name = "libc" +version = "0.2.0" +source = { git = "https://git.example.invalid/acme/mono?subdirectory=libc&rev=%[2]s#%[2]s" } + +[[package]] +name = "libd" +version = "0.1.0" +source = { git = "ssh://git@git.example.invalid:2222/acme/tools.git?rev=main#%[2]s" } + +[[package]] +name = "six" +version = "1.17.0" +source = { registry = "https://pypi.org/simple" } +`, commitA, commitB) + +var gitSourcesDeclared = []*builderv0.GitSourceDownload{ + {Repository: "https://git.example.invalid/acme/mono", Commit: commitA, MirrorContext: "uvsources"}, + {Repository: "https://git.example.invalid/acme/mono", Commit: commitB, MirrorContext: "uvsources"}, + {Repository: "ssh://git@git.example.invalid:2222/acme/tools.git", Commit: commitB, MirrorContext: "uvsources"}, +} + +func requireDeclares(t *testing.T, want, got []*builderv0.GitSourceDownload) { + t.Helper() + require.Len(t, got, len(want)) + for i := range want { + require.Equal(t, want[i].String(), got[i].String()) + } +} + +// TestLockedGitSourcesDeclaresEachLockedCommitOnce reads the URL uv hands git +// (the recorded one up to its query) and the locked commit, sorted, with a +// repository at one commit declared once however many packages it builds. +func TestLockedGitSourcesDeclaresEachLockedCommitOnce(t *testing.T) { + lock := filepath.Join(t.TempDir(), "uv.lock") + writeFile(t, lock, gitSourcesLock) + declared, err := lockedGitSources(lock) + require.NoError(t, err) + requireDeclares(t, gitSourcesDeclared, declared) + + mirrors := uvGitMirrors(declared) + require.Len(t, mirrors, 2, "one mirror per repository") + require.Equal(t, services.GitSourceMirrorPath("https://git.example.invalid/acme/mono"), mirrors[0].Path) +} + +// TestLockedGitSourcesRefusesWhatTheBuildCannotBeSupplied names the package, +// never the URL, which may carry a credential. +func TestLockedGitSourcesRefusesWhatTheBuildCannotBeSupplied(t *testing.T) { + for name, source := range map[string]string{ + "insecure": "http://token@git.example.invalid/acme/mono?rev=main#" + commitA, + "local": "file:///srv/mono#" + commitA, + "unpinned": "https://token@git.example.invalid/acme/mono?rev=main", + } { + t.Run(name, func(t *testing.T) { + lock := filepath.Join(t.TempDir(), "uv.lock") + writeFile(t, lock, fmt.Sprintf("version = 1\n\n[[package]]\nname = \"liba\"\nversion = \"0.1.0\"\nsource = { git = %q }\n", source)) + _, err := lockedGitSources(lock) + require.ErrorContains(t, err, `package "liba"`) + require.NotContains(t, err.Error(), "token") + }) + } +} + +// TestBuildDeclaresTheLockedGitSources holds the recipe: exactly the lock's git +// sources, the v6 contract, and the mirror mapping in the Dockerfile; without +// git sources, no declaration, the contract and Dockerfile of before. +func TestBuildDeclaresTheLockedGitSources(t *testing.T) { + builder, _ := createdBuilder(t) + outputDir := t.TempDir() + plan := emitRecipe(t, builder, outputDir) + require.Empty(t, plan.GetRecipes()[0].GetGitSourceDownloads()) + require.Equal(t, services.DockerBuildRecipeContractVersion, plan.GetContractVersion()) + requireGolden(t, "no-git-sources", dockerfileOf(t, outputDir, plan)) + + writeFile(t, filepath.Join(builder.SourceLocation, "uv.lock"), gitSourcesLock) + outputDir = t.TempDir() + plan = emitRecipe(t, builder, outputDir) + require.NoError(t, services.VerifyDockerBuildPlan(outputDir, plan)) + requireDeclares(t, gitSourcesDeclared, plan.GetRecipes()[0].GetGitSourceDownloads()) + require.Equal(t, services.DockerBuildRecipeGitSourcesContractVersion, plan.GetContractVersion()) + requireGolden(t, "git-sources", dockerfileOf(t, outputDir, plan)) +} + +// TestWorkspaceMemberDeclaresTheRootLocksGitSources reads the lock uv syncs a +// workspace member from: the workspace root's. +func TestWorkspaceMemberDeclaresTheRootLocksGitSources(t *testing.T) { + builder, _ := createdBuilder(t) + uvWorkspaceService(t, builder.Location, workspaceRequiresPython) + writeFile(t, filepath.Join(builder.Location, "uv.lock"), gitSourcesLock) + plan := emitRecipe(t, builder, t.TempDir()) + requireDeclares(t, gitSourcesDeclared, plan.GetRecipes()[0].GetGitSourceDownloads()) +} + +// requireGolden compares a rendered Dockerfile, its pinned images replaced, to +// testdata/dockerfile/.golden. The no-git-sources golden is the +// Dockerfile rendered before git sources were declared, byte for byte. +// UPDATE_GOLDEN=1 rewrites them after an intended template change. +func requireGolden(t *testing.T, name, dockerfile string) { + t.Helper() + dockerfile = strings.ReplaceAll(dockerfile, runtimeImage.FullName(), "BUILDER_IMAGE") + dockerfile = strings.ReplaceAll(dockerfile, uvImage, "UV_IMAGE") + path := filepath.Join("testdata", "dockerfile", name+".golden") + if os.Getenv("UPDATE_GOLDEN") != "" { + require.NoError(t, os.WriteFile(path, []byte(dockerfile), 0o644)) + } + golden, err := os.ReadFile(path) + require.NoError(t, err) + require.Equal(t, string(golden), dockerfile) +} + +// run runs a command in dir and fails the test on error. +func run(t *testing.T, dir string, env []string, name string, args ...string) string { + t.Helper() + command := exec.CommandContext(t.Context(), name, args...) + command.Dir = dir + command.Env = append(os.Environ(), env...) + out, err := command.CombinedOutput() + require.NoError(t, err, "%s %v: %s", name, args, out) + return strings.TrimSpace(string(out)) +} + +// TestLockedGitSourceBuildsFromTheSuppliedMirror is the claim end to end. The +// project depends on a package in a subdirectory of a git repository whose URL +// the build cannot reach, standing in for one it has no credential for; the +// host's uv locks it through a host-side rewrite, as a host credential would +// let it. Built with the mirror context the caller supplies, the image carries +// the package; built without it, the build fails on the unreachable URL. +func TestLockedGitSourceBuildsFromTheSuppliedMirror(t *testing.T) { + for _, tool := range []string{"docker", "uv", "git"} { + if _, err := exec.LookPath(tool); err != nil { + t.Skipf("%s is not installed: the git source image cannot be built", tool) + } + } + const repository = "https://git.example.invalid/example/libraries" + work := t.TempDir() + library := filepath.Join(work, "libraries") + writeFile(t, filepath.Join(library, "lib", "pyproject.toml"), `[project] +name = "example-git-lib" +version = "0.3.0" +requires-python = ">=3.10" + +[build-system] +requires = ["uv_build>=0.8,<0.13"] +build-backend = "uv_build" +`) + writeFile(t, filepath.Join(library, "lib", "src", "example_git_lib", "__init__.py"), "") + identity := []string{"GIT_AUTHOR_NAME=test", "GIT_AUTHOR_EMAIL=test@example.com", "GIT_COMMITTER_NAME=test", "GIT_COMMITTER_EMAIL=test@example.com"} + run(t, library, nil, "git", "init", "-q") + run(t, library, nil, "git", "add", "-A") + run(t, library, identity, "git", "commit", "-qm", "library") + commit := run(t, library, nil, "git", "rev-parse", "HEAD") + served := filepath.Join(work, "served.git") + run(t, work, nil, "git", "clone", "-q", "--bare", library, served) + + builder, _ := createdBuilder(t) + writeFile(t, filepath.Join(builder.SourceLocation, "pyproject.toml"), fmt.Sprintf(`[project] +name = "example-service" +version = "0.1.0" +requires-python = ">=3.10" +dependencies = ["example-git-lib"] + +[tool.uv] +package = false + +[tool.uv.sources] +example-git-lib = { git = %q, subdirectory = "lib", rev = %q } +`, repository, commit)) + rewrite := []string{"GIT_CONFIG_COUNT=1", "GIT_CONFIG_KEY_0=url.file://" + served + ".insteadOf", "GIT_CONFIG_VALUE_0=" + repository, "UV_CACHE_DIR=" + filepath.Join(work, "uv-cache")} + run(t, builder.SourceLocation, rewrite, "uv", "lock", "-q") + lock, err := os.ReadFile(filepath.Join(builder.SourceLocation, "uv.lock")) + require.NoError(t, err) + require.Contains(t, string(lock), fmt.Sprintf("%s?subdirectory=lib&rev=%s#%s", repository, commit, commit)) + + outputDir := t.TempDir() + plan := emitRecipe(t, builder, outputDir) + recipe := plan.GetRecipes()[0] + requireDeclares(t, []*builderv0.GitSourceDownload{{Repository: repository, Commit: commit, MirrorContext: "uvsources"}}, recipe.GetGitSourceDownloads()) + + // The mirror context as the caller fills it. + mirrors := filepath.Join(work, "mirrors") + mirror := filepath.Join(mirrors, services.GitSourceMirrorPath(repository)) + run(t, work, nil, "git", "init", "-q", "--bare", mirror) + run(t, mirror, nil, "git", "fetch", "-q", served, "+"+commit+":"+services.GitSourceMirrorRef(commit)) + + tag := fmt.Sprintf("codefly-fastapi-uv-git-source-test:%d", time.Now().UnixMilli()) + build := func(args ...string) ([]byte, error) { + args = append([]string{"build", "-f", filepath.Join(outputDir, recipe.GetDockerfile())}, args...) + return exec.CommandContext(t.Context(), "docker", append(args, builder.Location)...).CombinedOutput() + } + out, err := build("-t", tag, "--build-context", "uvsources="+mirrors) + require.NoError(t, err, "the mirrored git source did not build:\n%s", out) + t.Cleanup(func() { _ = exec.Command("docker", "rmi", "-f", tag).Run() }) + installed, err := exec.CommandContext(t.Context(), "docker", "run", "--rm", tag, + "python", "-c", "import importlib.metadata as m; print(m.version('example-git-lib'))").CombinedOutput() + require.NoError(t, err, "%s", installed) + require.Contains(t, string(installed), "0.3.0", "the image does not carry the git source") + + out, err = build() + require.Error(t, err, "without the mirror the unreachable URL must fail the build:\n%s", out) + require.Contains(t, string(out), "git.example.invalid") +} diff --git a/uvgitsources.go b/uvgitsources.go new file mode 100644 index 0000000..003caed --- /dev/null +++ b/uvgitsources.go @@ -0,0 +1,122 @@ +package main + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "sort" + "strings" + + "github.com/codefly-dev/core/agents/services" + builderv0 "github.com/codefly-dev/core/generated/go/codefly/services/builder/v0" + "github.com/pelletier/go-toml/v2" +) + +// A git dependency that needs a credential (a private repository) resolves on +// the developer's machine, whose git has the credential, and fails in the +// builder stage, which has none. The recipe therefore declares every git +// commit the lock pins (DockerBuildRecipe.git_source_downloads): the caller +// fetches each on its own machine before the build and supplies them as the +// uvsourcesContext build context, one bare mirror per repository, and the +// builder stage points git at those mirrors. uv.lock is read, never rewritten. +// +// uv records a git source as `source = { git = "?#" }` and +// asks git for the commit from by the refspec `+:refs/commit/`, +// where is the recorded string up to its query, byte for byte: uv has +// already lowercased the scheme and host and dropped a `git+` prefix when it +// wrote the lock, and keeps a `.git` suffix, a trailing slash, a port and an +// ssh login as they are. git's `url..insteadOf=` matches that +// string by prefix, so the declaration carries it exactly; a normalized form +// would miss the mirror and silently go to the network. + +// uvSourcesContext is the build context the Dockerfile reads the caller's git +// mirrors from (the `uvsources` stage in the template). +const uvSourcesContext = "uvsources" + +// uvGitMirror is one repository the builder stage maps onto its mirror. +type uvGitMirror struct { + // Path is the mirror's directory inside the uvsources context. + Path string + // Repository is the URL as uv hands it to git. + Repository string +} + +// uvLockPath is the lock uv syncs the project from: the workspace root's for a +// member of a workspace in the service, the project's own otherwise. +func uvLockPath(serviceDir, sourceDir string, membership *uvWorkspaceMembership) string { + if membership != nil { + return filepath.Join(serviceDir, filepath.FromSlash(membership.Root), "uv.lock") + } + return filepath.Join(sourceDir, "uv.lock") +} + +// lockedGitSources returns the git commits the lock at lockPath pins, sorted +// by repository and commit, each (repository, commit) once: packages built +// from subdirectories of one repository at one commit share a declaration. A +// project with no lock declares nothing; its build fails on the missing lock as +// it always has. +func lockedGitSources(lockPath string) ([]*builderv0.GitSourceDownload, error) { + content, err := os.ReadFile(lockPath) + if errors.Is(err, os.ErrNotExist) { + return nil, nil + } + if err != nil { + return nil, err + } + var lock struct { + Package []struct { + Name string `toml:"name"` + Source struct { + Git string `toml:"git"` + } `toml:"source"` + } `toml:"package"` + } + if err := toml.Unmarshal(content, &lock); err != nil { + return nil, fmt.Errorf("parse %s: %w", lockPath, err) + } + seen := map[[2]string]struct{}{} + var downloads []*builderv0.GitSourceDownload + for _, pkg := range lock.Package { + if pkg.Source.Git == "" { + continue + } + // The URL is not quoted in errors: a lock written from a source that + // named a credential may still carry one. + locked, commit, found := strings.Cut(pkg.Source.Git, "#") + if !found || commit == "" { + return nil, fmt.Errorf("%s: package %q is locked to a git source without a commit: run `uv lock`", lockPath, pkg.Name) + } + repository, _, _ := strings.Cut(locked, "?") + if !strings.HasPrefix(repository, "https://") && !strings.HasPrefix(repository, "ssh://") { + return nil, fmt.Errorf("%s: package %q is locked to a git source that is not an https:// or ssh:// URL, which an image build cannot be supplied", lockPath, pkg.Name) + } + key := [2]string{repository, commit} + if _, ok := seen[key]; ok { + continue + } + seen[key] = struct{}{} + downloads = append(downloads, &builderv0.GitSourceDownload{Repository: repository, Commit: commit, MirrorContext: uvSourcesContext}) + } + sort.Slice(downloads, func(i, j int) bool { + if downloads[i].GetRepository() != downloads[j].GetRepository() { + return downloads[i].GetRepository() < downloads[j].GetRepository() + } + return downloads[i].GetCommit() < downloads[j].GetCommit() + }) + return downloads, nil +} + +// uvGitMirrors is each declared repository once, in declaration order, with +// the mirror the caller fetches it into. +func uvGitMirrors(downloads []*builderv0.GitSourceDownload) []uvGitMirror { + var mirrors []uvGitMirror + for _, download := range downloads { + repository := download.GetRepository() + if len(mirrors) > 0 && mirrors[len(mirrors)-1].Repository == repository { + continue + } + mirrors = append(mirrors, uvGitMirror{Path: services.GitSourceMirrorPath(repository), Repository: repository}) + } + return mirrors +} From 23d86f94c14eafd8faee6496f1845303d568e58d Mon Sep 17 00:00:00 2001 From: Saurabh Maheshwari <242053479+saurabhmaheshwari-obin@users.noreply.github.com> Date: Wed, 21 Oct 2026 17:32:50 -0400 Subject: [PATCH 03/11] fix(builder): leave http and file git sources to the build's own fetch Only https and ssh git sources can be mirrored. A package locked to any other git URL was refused, which would have broken a project that builds today. It is now left out of the declaration, with a warning naming the package (not the URL), and the build fetches it from the network as before. A locked git source without a commit is still refused: uv always records one, so that lock is malformed. Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 4 +++- builder.go | 6 ++++- uv_git_sources_recipe_test.go | 45 ++++++++++++++++++++--------------- uvgitsources.go | 20 +++++++++------- 4 files changed, 46 insertions(+), 29 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 35b5292..3d82ca2 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -162,7 +162,9 @@ into `templates/factory/code/pyproject.toml.tmpl` yourself. A git source is declared in the recipe (`git_source_downloads`) by the URL exactly as `uv.lock` records it up to its query, which is what uv hands git; the caller fetches it with its own credentials and the Dockerfile maps that - URL onto the mirror. A normalized URL would miss the mirror silently. + URL onto the mirror. A normalized URL would miss the mirror silently. An + http or file source is not declared (a warning names the package) and is + fetched by the build itself, as before. - **The runtime image is pinned.** `docker-image` rejects a bare name and `:latest`. The default `ghcr.io/codefly-dev/python` companion is built from core and pinned by the digest of its multi-platform index — prefer it. Its tag diff --git a/builder.go b/builder.go index ef56ee8..e1b765f 100644 --- a/builder.go +++ b/builder.go @@ -251,10 +251,14 @@ func (s *Builder) Build(ctx context.Context, req *builderv0.BuildRequest) (*buil if err != nil { return s.Base.Builder.BuildError(err) } - gitSources, err := lockedGitSources(uvLockPath(s.Location, s.SourceLocation, membership)) + gitSources, unmirrored, err := lockedGitSources(uvLockPath(s.Location, s.SourceLocation, membership)) if err != nil { return s.Base.Builder.BuildError(err) } + for _, name := range unmirrored { + // The URL is not logged: a lock may still carry a credential in it. + s.Wool.Warn("git source is neither https nor ssh, so it is not mirrored: the image build fetches it itself", wool.Field("package", name)) + } var options []services.RecipeOption if len(gitSources) > 0 { options = append(options, services.WithGitSourceDownloads(gitSources...)) diff --git a/uv_git_sources_recipe_test.go b/uv_git_sources_recipe_test.go index be0c97a..ecb1c34 100644 --- a/uv_git_sources_recipe_test.go +++ b/uv_git_sources_recipe_test.go @@ -21,7 +21,8 @@ const ( // gitSourcesLock is a lock in the form uv writes it: two packages built from // subdirectories of one repository at one commit, the same repository at a -// second commit, an ssh source with a port, and a registry package. +// second commit, an ssh source with a port, an http and a file source, which +// cannot be mirrored, and a registry package. var gitSourcesLock = fmt.Sprintf(`version = 1 requires-python = ">=3.12" @@ -45,6 +46,16 @@ name = "libd" version = "0.1.0" source = { git = "ssh://git@git.example.invalid:2222/acme/tools.git?rev=main#%[2]s" } +[[package]] +name = "libe" +version = "0.1.0" +source = { git = "http://token@git.example.invalid/acme/plain?rev=main#%[1]s" } + +[[package]] +name = "libf" +version = "0.1.0" +source = { git = "file:///srv/local?rev=main#%[1]s" } + [[package]] name = "six" version = "1.17.0" @@ -67,35 +78,31 @@ func requireDeclares(t *testing.T, want, got []*builderv0.GitSourceDownload) { // TestLockedGitSourcesDeclaresEachLockedCommitOnce reads the URL uv hands git // (the recorded one up to its query) and the locked commit, sorted, with a -// repository at one commit declared once however many packages it builds. +// repository at one commit declared once however many packages it builds. The +// http and file sources are left out, named by package for the warning, and +// left to the build to fetch as before. func TestLockedGitSourcesDeclaresEachLockedCommitOnce(t *testing.T) { lock := filepath.Join(t.TempDir(), "uv.lock") writeFile(t, lock, gitSourcesLock) - declared, err := lockedGitSources(lock) + declared, unmirrored, err := lockedGitSources(lock) require.NoError(t, err) requireDeclares(t, gitSourcesDeclared, declared) + require.Equal(t, []string{"libe", "libf"}, unmirrored) mirrors := uvGitMirrors(declared) require.Len(t, mirrors, 2, "one mirror per repository") require.Equal(t, services.GitSourceMirrorPath("https://git.example.invalid/acme/mono"), mirrors[0].Path) } -// TestLockedGitSourcesRefusesWhatTheBuildCannotBeSupplied names the package, -// never the URL, which may carry a credential. -func TestLockedGitSourcesRefusesWhatTheBuildCannotBeSupplied(t *testing.T) { - for name, source := range map[string]string{ - "insecure": "http://token@git.example.invalid/acme/mono?rev=main#" + commitA, - "local": "file:///srv/mono#" + commitA, - "unpinned": "https://token@git.example.invalid/acme/mono?rev=main", - } { - t.Run(name, func(t *testing.T) { - lock := filepath.Join(t.TempDir(), "uv.lock") - writeFile(t, lock, fmt.Sprintf("version = 1\n\n[[package]]\nname = \"liba\"\nversion = \"0.1.0\"\nsource = { git = %q }\n", source)) - _, err := lockedGitSources(lock) - require.ErrorContains(t, err, `package "liba"`) - require.NotContains(t, err.Error(), "token") - }) - } +// TestLockedGitSourcesRefusesAnUnpinnedSource: uv always records the commit, +// so a git source without one is a malformed lock. The error names the +// package, never the URL, which may carry a credential. +func TestLockedGitSourcesRefusesAnUnpinnedSource(t *testing.T) { + lock := filepath.Join(t.TempDir(), "uv.lock") + writeFile(t, lock, "version = 1\n\n[[package]]\nname = \"liba\"\nversion = \"0.1.0\"\nsource = { git = \"https://token@git.example.invalid/acme/mono?rev=main\" }\n") + _, _, err := lockedGitSources(lock) + require.ErrorContains(t, err, `package "liba"`) + require.NotContains(t, err.Error(), "token") } // TestBuildDeclaresTheLockedGitSources holds the recipe: exactly the lock's git diff --git a/uvgitsources.go b/uvgitsources.go index 003caed..74a704c 100644 --- a/uvgitsources.go +++ b/uvgitsources.go @@ -56,13 +56,17 @@ func uvLockPath(serviceDir, sourceDir string, membership *uvWorkspaceMembership) // from subdirectories of one repository at one commit share a declaration. A // project with no lock declares nothing; its build fails on the missing lock as // it always has. -func lockedGitSources(lockPath string) ([]*builderv0.GitSourceDownload, error) { +// +// Only https and ssh sources can be mirrored. The packages locked to any other +// git URL (http, file) are returned as unmirrored and left out of the +// declaration: the build fetches them itself, as it always has. +func lockedGitSources(lockPath string) (downloads []*builderv0.GitSourceDownload, unmirrored []string, err error) { content, err := os.ReadFile(lockPath) if errors.Is(err, os.ErrNotExist) { - return nil, nil + return nil, nil, nil } if err != nil { - return nil, err + return nil, nil, err } var lock struct { Package []struct { @@ -73,10 +77,9 @@ func lockedGitSources(lockPath string) ([]*builderv0.GitSourceDownload, error) { } `toml:"package"` } if err := toml.Unmarshal(content, &lock); err != nil { - return nil, fmt.Errorf("parse %s: %w", lockPath, err) + return nil, nil, fmt.Errorf("parse %s: %w", lockPath, err) } seen := map[[2]string]struct{}{} - var downloads []*builderv0.GitSourceDownload for _, pkg := range lock.Package { if pkg.Source.Git == "" { continue @@ -85,11 +88,12 @@ func lockedGitSources(lockPath string) ([]*builderv0.GitSourceDownload, error) { // named a credential may still carry one. locked, commit, found := strings.Cut(pkg.Source.Git, "#") if !found || commit == "" { - return nil, fmt.Errorf("%s: package %q is locked to a git source without a commit: run `uv lock`", lockPath, pkg.Name) + return nil, nil, fmt.Errorf("%s: package %q is locked to a git source without a commit: run `uv lock`", lockPath, pkg.Name) } repository, _, _ := strings.Cut(locked, "?") if !strings.HasPrefix(repository, "https://") && !strings.HasPrefix(repository, "ssh://") { - return nil, fmt.Errorf("%s: package %q is locked to a git source that is not an https:// or ssh:// URL, which an image build cannot be supplied", lockPath, pkg.Name) + unmirrored = append(unmirrored, pkg.Name) + continue } key := [2]string{repository, commit} if _, ok := seen[key]; ok { @@ -104,7 +108,7 @@ func lockedGitSources(lockPath string) ([]*builderv0.GitSourceDownload, error) { } return downloads[i].GetCommit() < downloads[j].GetCommit() }) - return downloads, nil + return downloads, unmirrored, nil } // uvGitMirrors is each declared repository once, in declaration order, with From 878291755334038eced7fd9c630c84590a4fef4f Mon Sep 17 00:00:00 2001 From: Saurabh Maheshwari <242053479+saurabhmaheshwari-obin@users.noreply.github.com> Date: Wed, 21 Oct 2026 17:40:19 -0400 Subject: [PATCH 04/11] build(deps): repin core to the exported git source checks Co-Authored-By: Claude Opus 5.5 --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 3d1707b..dfd83dd 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/codefly-dev/service-python-fastapi go 1.27.2 require ( - github.com/codefly-dev/core v0.19.6-0.20261021204332-519efae17f9e + github.com/codefly-dev/core v0.19.6-0.20261021213826-abdc7077fb98 github.com/codefly-dev/service-python v0.0.66 github.com/pelletier/go-toml/v2 v2.4.3 github.com/stretchr/testify v1.12.1 diff --git a/go.sum b/go.sum index bcf6da7..10f596e 100644 --- a/go.sum +++ b/go.sum @@ -43,8 +43,8 @@ github.com/clipperhouse/uax29/v2 v2.7.0 h1:+gs4oBZ2gPfVrKPthwbMzWZDaAFPGYK72F0NJ github.com/clipperhouse/uax29/v2 v2.7.0/go.mod h1:EFJ2TJMRUaplDxHKj1qAEhCtQPW2tJSwu5BF98AuoVM= github.com/cloudflare/circl v1.6.5 h1:O64F26HEqNhznd/hrC5KZXVKYuKM2rx4deZDTc4ihQA= github.com/cloudflare/circl v1.6.5/go.mod h1:h5LNyxAc5nTue9DS5jT+48en2PSDYt3zdGnz5OstK6c= -github.com/codefly-dev/core v0.19.6-0.20261021204332-519efae17f9e h1:fbASQDxSfXFKR8CL6IwF2YJCmawmsKNc80irNZSaho4= -github.com/codefly-dev/core v0.19.6-0.20261021204332-519efae17f9e/go.mod h1:0uX/7lTBVaGV0RI2Mb3mpEr/PPdx/jZqXHym/U8FVDI= +github.com/codefly-dev/core v0.19.6-0.20261021213826-abdc7077fb98 h1:/z9xBAdnw0vAfqZpZH58zmifFxSYZVqyQziKcMTX8Xo= +github.com/codefly-dev/core v0.19.6-0.20261021213826-abdc7077fb98/go.mod h1:0uX/7lTBVaGV0RI2Mb3mpEr/PPdx/jZqXHym/U8FVDI= github.com/codefly-dev/gortk v0.2.0 h1:7bOlS5valYz2zil+fZctQNcPCYBcPj86abcw9N8h1hQ= github.com/codefly-dev/gortk v0.2.0/go.mod h1:dDWUMFgAP063OCGhTEpV7FFUj9+zTcxxO/nV80VKEwg= github.com/codefly-dev/service-python v0.0.66 h1:Y74QvKAYMaOnZcsfRxuD9dpQo3//7Mjo12RYELyrp5o= From df359d5baf2634db617554b9874a2024d8ef922b Mon Sep 17 00:00:00 2001 From: Saurabh Maheshwari <242053479+saurabhmaheshwari-obin@users.noreply.github.com> Date: Wed, 21 Oct 2026 17:42:28 -0400 Subject: [PATCH 05/11] fix(builder): declare and render only the git sources core accepts A locked source core refuses (a credential over https, percent-encoding, "@" in the path, a scheme other than https or ssh) was declared and rendered into the Dockerfile before core refused the plan, failing a build that works today. Each source is now checked with core's own GitSourceRepositoryProblem first: a refused one is neither declared nor rendered, and a warning names the package and core's reason, never the URL; the build fetches it itself, as before. A commit that is not a full object id (IsGitSourceCommit) still fails, naming the package. Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 6 ++-- builder.go | 5 ++-- uv_git_sources_recipe_test.go | 52 +++++++++++++++++++++++++++-------- uvgitsources.go | 25 +++++++++++------ 4 files changed, 64 insertions(+), 24 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 3d82ca2..6f2d846 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -162,9 +162,9 @@ into `templates/factory/code/pyproject.toml.tmpl` yourself. A git source is declared in the recipe (`git_source_downloads`) by the URL exactly as `uv.lock` records it up to its query, which is what uv hands git; the caller fetches it with its own credentials and the Dockerfile maps that - URL onto the mirror. A normalized URL would miss the mirror silently. An - http or file source is not declared (a warning names the package) and is - fetched by the build itself, as before. + URL onto the mirror. A normalized URL would miss the mirror silently. A + source core refuses to declare is neither declared nor rendered (a warning + names the package and core's reason) and the build fetches it, as before. - **The runtime image is pinned.** `docker-image` rejects a bare name and `:latest`. The default `ghcr.io/codefly-dev/python` companion is built from core and pinned by the digest of its multi-platform index — prefer it. Its tag diff --git a/builder.go b/builder.go index e1b765f..4f91eef 100644 --- a/builder.go +++ b/builder.go @@ -255,9 +255,10 @@ func (s *Builder) Build(ctx context.Context, req *builderv0.BuildRequest) (*buil if err != nil { return s.Base.Builder.BuildError(err) } - for _, name := range unmirrored { + for _, source := range unmirrored { // The URL is not logged: a lock may still carry a credential in it. - s.Wool.Warn("git source is neither https nor ssh, so it is not mirrored: the image build fetches it itself", wool.Field("package", name)) + s.Wool.Warn("git source is not mirrored: the image build fetches it itself", + wool.Field("package", source.Package), wool.Field("reason", "repository "+source.Reason)) } var options []services.RecipeOption if len(gitSources) > 0 { diff --git a/uv_git_sources_recipe_test.go b/uv_git_sources_recipe_test.go index ecb1c34..235c5f6 100644 --- a/uv_git_sources_recipe_test.go +++ b/uv_git_sources_recipe_test.go @@ -21,8 +21,9 @@ const ( // gitSourcesLock is a lock in the form uv writes it: two packages built from // subdirectories of one repository at one commit, the same repository at a -// second commit, an ssh source with a port, an http and a file source, which -// cannot be mirrored, and a registry package. +// second commit, an ssh source with a port, sources a recipe cannot declare +// (http, file, a credential, percent-encoding, "@" in the path), and a +// registry package. var gitSourcesLock = fmt.Sprintf(`version = 1 requires-python = ">=3.12" @@ -56,6 +57,21 @@ name = "libf" version = "0.1.0" source = { git = "file:///srv/local?rev=main#%[1]s" } +[[package]] +name = "libg" +version = "0.1.0" +source = { git = "https://TOKEN@git.example.invalid/acme/secret?rev=main#%[1]s" } + +[[package]] +name = "libh" +version = "0.1.0" +source = { git = "https://git.example.invalid/Team%%20Project/repo?rev=main#%[1]s" } + +[[package]] +name = "libi" +version = "0.1.0" +source = { git = "https://git.example.invalid/acme/lib@v1?rev=main#%[1]s" } + [[package]] name = "six" version = "1.17.0" @@ -87,22 +103,30 @@ func TestLockedGitSourcesDeclaresEachLockedCommitOnce(t *testing.T) { declared, unmirrored, err := lockedGitSources(lock) require.NoError(t, err) requireDeclares(t, gitSourcesDeclared, declared) - require.Equal(t, []string{"libe", "libf"}, unmirrored) + require.Equal(t, []unmirroredSource{ + {Package: "libe", Reason: "must start with https:// or ssh://"}, + {Package: "libf", Reason: "must start with https:// or ssh://"}, + {Package: "libg", Reason: "must carry no user information over https"}, + {Package: "libh", Reason: "must carry no percent-encoding"}, + {Package: "libi", Reason: "must carry no @ in the path"}, + }, unmirrored) mirrors := uvGitMirrors(declared) require.Len(t, mirrors, 2, "one mirror per repository") require.Equal(t, services.GitSourceMirrorPath("https://git.example.invalid/acme/mono"), mirrors[0].Path) } -// TestLockedGitSourcesRefusesAnUnpinnedSource: uv always records the commit, -// so a git source without one is a malformed lock. The error names the -// package, never the URL, which may carry a credential. +// TestLockedGitSourcesRefusesAnUnpinnedSource: uv always records the full +// commit id, so a git source without one is a malformed lock. The error names +// the package, never the URL, which may carry a credential. func TestLockedGitSourcesRefusesAnUnpinnedSource(t *testing.T) { - lock := filepath.Join(t.TempDir(), "uv.lock") - writeFile(t, lock, "version = 1\n\n[[package]]\nname = \"liba\"\nversion = \"0.1.0\"\nsource = { git = \"https://token@git.example.invalid/acme/mono?rev=main\" }\n") - _, _, err := lockedGitSources(lock) - require.ErrorContains(t, err, `package "liba"`) - require.NotContains(t, err.Error(), "token") + for _, source := range []string{"https://token@git.example.invalid/acme/mono?rev=main", "https://token@git.example.invalid/acme/mono#8b58bb4"} { + lock := filepath.Join(t.TempDir(), "uv.lock") + writeFile(t, lock, fmt.Sprintf("version = 1\n\n[[package]]\nname = \"liba\"\nversion = \"0.1.0\"\nsource = { git = %q }\n", source)) + _, _, err := lockedGitSources(lock) + require.ErrorContains(t, err, `package "liba"`) + require.NotContains(t, err.Error(), "token") + } } // TestBuildDeclaresTheLockedGitSources holds the recipe: exactly the lock's git @@ -123,6 +147,12 @@ func TestBuildDeclaresTheLockedGitSources(t *testing.T) { requireDeclares(t, gitSourcesDeclared, plan.GetRecipes()[0].GetGitSourceDownloads()) require.Equal(t, services.DockerBuildRecipeGitSourcesContractVersion, plan.GetContractVersion()) requireGolden(t, "git-sources", dockerfileOf(t, outputDir, plan)) + // What a recipe cannot declare reaches neither the plan nor the Dockerfile. + for _, rendered := range []string{plan.String(), dockerfileOf(t, outputDir, plan)} { + for _, refused := range []string{"TOKEN", "Team", "lib@v1", "acme/plain", "/srv/local"} { + require.NotContains(t, rendered, refused) + } + } } // TestWorkspaceMemberDeclaresTheRootLocksGitSources reads the lock uv syncs a diff --git a/uvgitsources.go b/uvgitsources.go index 74a704c..33b896b 100644 --- a/uvgitsources.go +++ b/uvgitsources.go @@ -42,6 +42,13 @@ type uvGitMirror struct { Repository string } +// unmirroredSource is a locked git source the recipe does not declare. +type unmirroredSource struct { + Package string + // Reason is core's, which never quotes the URL. + Reason string +} + // uvLockPath is the lock uv syncs the project from: the workspace root's for a // member of a workspace in the service, the project's own otherwise. func uvLockPath(serviceDir, sourceDir string, membership *uvWorkspaceMembership) string { @@ -57,10 +64,12 @@ func uvLockPath(serviceDir, sourceDir string, membership *uvWorkspaceMembership) // project with no lock declares nothing; its build fails on the missing lock as // it always has. // -// Only https and ssh sources can be mirrored. The packages locked to any other -// git URL (http, file) are returned as unmirrored and left out of the -// declaration: the build fetches them itself, as it always has. -func lockedGitSources(lockPath string) (downloads []*builderv0.GitSourceDownload, unmirrored []string, err error) { +// A source core would refuse to declare (services.GitSourceRepositoryProblem: +// http, file, a credential, percent-encoding, "@" in the path, ...) is +// returned as unmirrored, with core's reason, and is neither declared nor +// rendered: the build fetches it itself, as it always has. A source whose +// commit is missing or not a full object id makes the lock malformed. +func lockedGitSources(lockPath string) (downloads []*builderv0.GitSourceDownload, unmirrored []unmirroredSource, err error) { content, err := os.ReadFile(lockPath) if errors.Is(err, os.ErrNotExist) { return nil, nil, nil @@ -87,12 +96,12 @@ func lockedGitSources(lockPath string) (downloads []*builderv0.GitSourceDownload // The URL is not quoted in errors: a lock written from a source that // named a credential may still carry one. locked, commit, found := strings.Cut(pkg.Source.Git, "#") - if !found || commit == "" { - return nil, nil, fmt.Errorf("%s: package %q is locked to a git source without a commit: run `uv lock`", lockPath, pkg.Name) + if !found || !services.IsGitSourceCommit(commit) { + return nil, nil, fmt.Errorf("%s: package %q is locked to a git source without a full commit id: run `uv lock`", lockPath, pkg.Name) } repository, _, _ := strings.Cut(locked, "?") - if !strings.HasPrefix(repository, "https://") && !strings.HasPrefix(repository, "ssh://") { - unmirrored = append(unmirrored, pkg.Name) + if reason := services.GitSourceRepositoryProblem(repository); reason != "" { + unmirrored = append(unmirrored, unmirroredSource{Package: pkg.Name, Reason: reason}) continue } key := [2]string{repository, commit} From bf8015b1e524f49959c707dc6802534b5f81e2fa Mon Sep 17 00:00:00 2001 From: Saurabh Maheshwari <242053479+saurabhmaheshwari-obin@users.noreply.github.com> Date: Wed, 21 Oct 2026 17:44:21 -0400 Subject: [PATCH 06/11] fix(builder): append the mirror mappings to inherited git config The sync step set GIT_CONFIG_COUNT and numbered its insteadOf mappings from 0, replacing any git configuration a base image or an earlier ENV had put in the environment. The mappings are now appended after ${GIT_CONFIG_COUNT:-0}, and the count is raised to cover them. Values stay single-quoted. Co-Authored-By: Claude Opus 5.5 --- templates/builder/Dockerfile.tmpl | 8 +++++--- testdata/dockerfile/git-sources.golden | 10 +++++---- uv_git_sources_recipe_test.go | 28 ++++++++++++++++++++++++++ 3 files changed, 39 insertions(+), 7 deletions(-) diff --git a/templates/builder/Dockerfile.tmpl b/templates/builder/Dockerfile.tmpl index b459001..5d39853 100644 --- a/templates/builder/Dockerfile.tmpl +++ b/templates/builder/Dockerfile.tmpl @@ -57,10 +57,12 @@ ENV UV_PROJECT_ENVIRONMENT=/app/.venv \ # With the caller's mirrors, git reads each locked repository from its mirror # instead of the URL the lock records (uv hands git that URL unchanged), so no # credential is needed, and a commit missing from the mirror fails here rather -# than prompting for one. +# than prompting for one. The mappings are appended after any git configuration +# the environment already carries (GIT_CONFIG_COUNT), never in its place. RUN --mount=type=bind,from=uvsources,target=/uvsources \ - if [ -n "$(ls -A /uvsources)" ]; then export GIT_TERMINAL_PROMPT=0 GIT_CONFIG_COUNT={{len .GitSources}}{{range $i, $source := .GitSources}} \ - GIT_CONFIG_KEY_{{$i}}='url.file:///uvsources/{{$source.Path}}.insteadOf' GIT_CONFIG_VALUE_{{$i}}='{{$source.Repository}}'{{end}}; fi && \ + if [ -n "$(ls -A /uvsources)" ]; then n=${GIT_CONFIG_COUNT:-0}{{range .GitSources}} && \ + export "GIT_CONFIG_KEY_$n=url.file:///uvsources/{{.Path}}.insteadOf" "GIT_CONFIG_VALUE_$n="'{{.Repository}}' && n=$((n+1)){{end}} && \ + export GIT_CONFIG_COUNT=$n GIT_TERMINAL_PROMPT=0; fi && \ mkdir -p /opt/uv/python && uv sync --frozen --no-dev --no-install-project --no-editable {{- else}} RUN mkdir -p /opt/uv/python && uv sync --frozen --no-dev --no-install-project --no-editable diff --git a/testdata/dockerfile/git-sources.golden b/testdata/dockerfile/git-sources.golden index 5b5308c..0927324 100644 --- a/testdata/dockerfile/git-sources.golden +++ b/testdata/dockerfile/git-sources.golden @@ -35,11 +35,13 @@ ENV UV_PROJECT_ENVIRONMENT=/app/.venv \ # With the caller's mirrors, git reads each locked repository from its mirror # instead of the URL the lock records (uv hands git that URL unchanged), so no # credential is needed, and a commit missing from the mirror fails here rather -# than prompting for one. +# than prompting for one. The mappings are appended after any git configuration +# the environment already carries (GIT_CONFIG_COUNT), never in its place. RUN --mount=type=bind,from=uvsources,target=/uvsources \ - if [ -n "$(ls -A /uvsources)" ]; then export GIT_TERMINAL_PROMPT=0 GIT_CONFIG_COUNT=2 \ - GIT_CONFIG_KEY_0='url.file:///uvsources/c9fa8363b851c2643d648edad1c8c7892904f717385511fe4771422d0817a55e.git.insteadOf' GIT_CONFIG_VALUE_0='https://git.example.invalid/acme/mono' \ - GIT_CONFIG_KEY_1='url.file:///uvsources/d2b987abe743521016c33e82dba28324380629b9d2544f9e5213ee4299a3af6c.git.insteadOf' GIT_CONFIG_VALUE_1='ssh://git@git.example.invalid:2222/acme/tools.git'; fi && \ + if [ -n "$(ls -A /uvsources)" ]; then n=${GIT_CONFIG_COUNT:-0} && \ + export "GIT_CONFIG_KEY_$n=url.file:///uvsources/c9fa8363b851c2643d648edad1c8c7892904f717385511fe4771422d0817a55e.git.insteadOf" "GIT_CONFIG_VALUE_$n="'https://git.example.invalid/acme/mono' && n=$((n+1)) && \ + export "GIT_CONFIG_KEY_$n=url.file:///uvsources/d2b987abe743521016c33e82dba28324380629b9d2544f9e5213ee4299a3af6c.git.insteadOf" "GIT_CONFIG_VALUE_$n="'ssh://git@git.example.invalid:2222/acme/tools.git' && n=$((n+1)) && \ + export GIT_CONFIG_COUNT=$n GIT_TERMINAL_PROMPT=0; fi && \ mkdir -p /opt/uv/python && uv sync --frozen --no-dev --no-install-project --no-editable diff --git a/uv_git_sources_recipe_test.go b/uv_git_sources_recipe_test.go index 235c5f6..24b81bf 100644 --- a/uv_git_sources_recipe_test.go +++ b/uv_git_sources_recipe_test.go @@ -273,3 +273,31 @@ example-git-lib = { git = %q, subdirectory = "lib", rev = %q } require.Error(t, err, "without the mirror the unreachable URL must fail the build:\n%s", out) require.Contains(t, string(out), "git.example.invalid") } + +// TestGitSourceMappingKeepsInheritedGitConfig runs the rendered mapping in the +// builder base image with git configuration already in the environment, as a +// base image or an earlier ENV may set it: the mappings are appended after it, +// and both are in effect. +func TestGitSourceMappingKeepsInheritedGitConfig(t *testing.T) { + if _, err := exec.LookPath("docker"); err != nil { + t.Skip("docker is not installed: the mapping cannot be run") + } + builder, _ := createdBuilder(t) + writeFile(t, filepath.Join(builder.SourceLocation, "uv.lock"), gitSourcesLock) + outputDir := t.TempDir() + dockerfile := dockerfileOf(t, outputDir, emitRecipe(t, builder, outputDir)) + start := strings.Index(dockerfile, "RUN --mount=type=bind,from=uvsources,target=/uvsources \\\n") + end := strings.Index(dockerfile, "mkdir -p /opt/uv/python && uv sync") + require.True(t, start >= 0 && end > start, "the rendered sync step has no mirror mapping") + mapping := strings.ReplaceAll(dockerfile[start+len("RUN --mount=type=bind,from=uvsources,target=/uvsources \\\n"):end], "\\\n", "") + + mirrors := t.TempDir() + writeFile(t, filepath.Join(mirrors, "present"), "") + mono := "url.file:///uvsources/" + services.GitSourceMirrorPath("https://git.example.invalid/acme/mono") + ".insteadOf" + out, err := exec.CommandContext(t.Context(), "docker", "run", "--rm", "--network=none", + "-e", "GIT_CONFIG_COUNT=1", "-e", "GIT_CONFIG_KEY_0=user.name", "-e", "GIT_CONFIG_VALUE_0=inherited", + "-v", mirrors+":/uvsources:ro", runtimeImage.FullName(), + "sh", "-c", mapping+"git config --get user.name && git config --get "+mono+" && echo count=$GIT_CONFIG_COUNT").CombinedOutput() + require.NoError(t, err, "%s", out) + require.Equal(t, "inherited\nhttps://git.example.invalid/acme/mono\ncount=3\n", string(out)) +} From c07d675374d5fe15f0133d6637386ca5aa9d1950 Mon Sep 17 00:00:00 2001 From: Saurabh Maheshwari <242053479+saurabhmaheshwari-obin@users.noreply.github.com> Date: Wed, 21 Oct 2026 17:44:24 -0400 Subject: [PATCH 07/11] build(builder): pin the Dockerfile frontend when git sources are declared The sync step's RUN --mount comes from the Dockerfile frontend, so a recipe that declares git sources pins it, with the same digest the Go agent's recipe pins. A recipe without git sources renders as before. Co-Authored-By: Claude Opus 5.5 --- templates/builder/Dockerfile.tmpl | 1 + testdata/dockerfile/git-sources.golden | 1 + 2 files changed, 2 insertions(+) diff --git a/templates/builder/Dockerfile.tmpl b/templates/builder/Dockerfile.tmpl index 5d39853..cf42205 100644 --- a/templates/builder/Dockerfile.tmpl +++ b/templates/builder/Dockerfile.tmpl @@ -1,4 +1,5 @@ {{if .GitSources -}} +# syntax=docker/dockerfile:1@sha256:ecfaec9ed6d810b56388c508f4121597bfbba70d41a6dfeee4d8cad5f295fc32 # The git commits uv.lock pins (git_source_downloads in the recipe), fetched by # the caller on its own machine, with its own credentials, before any image # build, and supplied here as the build context of this name: one bare mirror diff --git a/testdata/dockerfile/git-sources.golden b/testdata/dockerfile/git-sources.golden index 0927324..bd93dbf 100644 --- a/testdata/dockerfile/git-sources.golden +++ b/testdata/dockerfile/git-sources.golden @@ -1,3 +1,4 @@ +# syntax=docker/dockerfile:1@sha256:ecfaec9ed6d810b56388c508f4121597bfbba70d41a6dfeee4d8cad5f295fc32 # The git commits uv.lock pins (git_source_downloads in the recipe), fetched by # the caller on its own machine, with its own credentials, before any image # build, and supplied here as the build context of this name: one bare mirror From 3e33c731b68f606ba03940143c9c7240d53e45d8 Mon Sep 17 00:00:00 2001 From: Saurabh Maheshwari <242053479+saurabhmaheshwari-obin@users.noreply.github.com> Date: Wed, 21 Oct 2026 18:00:04 -0400 Subject: [PATCH 08/11] fix(builder): keep the mirror mappings from capturing or losing to other git config git's insteadOf rewrites every URL that begins the configured one, so a declared https://h/acme/mono also rewrote an undeclared locked https://h/acme/mono@v1 onto its mirror, failing a build that worked. A declarable source whose URL begins an undeclared locked source's URL is now left out as well (warned, fetched by the build), until no such capture remains. Two declared URLs, one beginning the other, stay declared: git applies the longest match. An inherited rewrite of the same URL came first and won over the appended mirror mapping. The mappings now take entries 0..N-1 and the inherited entries move after them, read by constructed name and never evaluated. The inherited count is read as git reads it (a leading "+" and leading zeros, so 01, +1 and 08 work); a count git would reject is left alone for git to reject. Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 7 +- builder.go | 2 +- templates/builder/Dockerfile.tmpl | 21 +++-- testdata/dockerfile/git-sources.golden | 23 +++-- uv_git_sources_recipe_test.go | 120 ++++++++++++++++++++----- uvgitsources.go | 58 ++++++++++-- 6 files changed, 186 insertions(+), 45 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 6f2d846..7600796 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -162,9 +162,10 @@ into `templates/factory/code/pyproject.toml.tmpl` yourself. A git source is declared in the recipe (`git_source_downloads`) by the URL exactly as `uv.lock` records it up to its query, which is what uv hands git; the caller fetches it with its own credentials and the Dockerfile maps that - URL onto the mirror. A normalized URL would miss the mirror silently. A - source core refuses to declare is neither declared nor rendered (a warning - names the package and core's reason) and the build fetches it, as before. + URL onto the mirror, ahead of inherited git config. A normalized URL would + miss the mirror silently. A source core refuses, or whose URL begins an + undeclared locked one (insteadOf matches by prefix), is not declared or + rendered; a warning names the package and why, and the build fetches it. - **The runtime image is pinned.** `docker-image` rejects a bare name and `:latest`. The default `ghcr.io/codefly-dev/python` companion is built from core and pinned by the digest of its multi-platform index — prefer it. Its tag diff --git a/builder.go b/builder.go index 4f91eef..a5e6575 100644 --- a/builder.go +++ b/builder.go @@ -258,7 +258,7 @@ func (s *Builder) Build(ctx context.Context, req *builderv0.BuildRequest) (*buil for _, source := range unmirrored { // The URL is not logged: a lock may still carry a credential in it. s.Wool.Warn("git source is not mirrored: the image build fetches it itself", - wool.Field("package", source.Package), wool.Field("reason", "repository "+source.Reason)) + wool.Field("package", source.Package), wool.Field("reason", source.Reason)) } var options []services.RecipeOption if len(gitSources) > 0 { diff --git a/templates/builder/Dockerfile.tmpl b/templates/builder/Dockerfile.tmpl index cf42205..3430eb0 100644 --- a/templates/builder/Dockerfile.tmpl +++ b/templates/builder/Dockerfile.tmpl @@ -58,12 +58,23 @@ ENV UV_PROJECT_ENVIRONMENT=/app/.venv \ # With the caller's mirrors, git reads each locked repository from its mirror # instead of the URL the lock records (uv hands git that URL unchanged), so no # credential is needed, and a commit missing from the mirror fails here rather -# than prompting for one. The mappings are appended after any git configuration -# the environment already carries (GIT_CONFIG_COUNT), never in its place. +# than prompting for one. The mirror mappings take the first entries of git's +# environment configuration, so they win over an inherited rewrite of the same +# URL, and the entries the environment already carries (GIT_CONFIG_COUNT, read +# as git reads it) move after them, read by name and never evaluated. A count +# git would reject is left as it is, for git to reject as before. RUN --mount=type=bind,from=uvsources,target=/uvsources \ - if [ -n "$(ls -A /uvsources)" ]; then n=${GIT_CONFIG_COUNT:-0}{{range .GitSources}} && \ - export "GIT_CONFIG_KEY_$n=url.file:///uvsources/{{.Path}}.insteadOf" "GIT_CONFIG_VALUE_$n="'{{.Repository}}' && n=$((n+1)){{end}} && \ - export GIT_CONFIG_COUNT=$n GIT_TERMINAL_PROMPT=0; fi && \ + if [ -n "$(ls -A /uvsources)" ]; then \ + n=${GIT_CONFIG_COUNT:-0}; n=${n#+}; while [ "${n#0}" != "$n" ] && [ -n "${n#0}" ]; do n=${n#0}; done; \ + case $n in ''|*[!0-9]*) ;; *) \ + i=$n; while [ "$i" -gt 0 ]; do i=$((i-1)); \ + eval "k=\${GIT_CONFIG_KEY_$i}"; eval "v=\${GIT_CONFIG_VALUE_$i}"; \ + export "GIT_CONFIG_KEY_$((i+{{len .GitSources}}))=$k" "GIT_CONFIG_VALUE_$((i+{{len .GitSources}}))=$v"; \ + done; \ + export GIT_CONFIG_COUNT=$((n+{{len .GitSources}})) GIT_TERMINAL_PROMPT=0{{range $i, $source := .GitSources}} \ + GIT_CONFIG_KEY_{{$i}}='url.file:///uvsources/{{$source.Path}}.insteadOf' GIT_CONFIG_VALUE_{{$i}}='{{$source.Repository}}'{{end}};; \ + esac; \ + fi && \ mkdir -p /opt/uv/python && uv sync --frozen --no-dev --no-install-project --no-editable {{- else}} RUN mkdir -p /opt/uv/python && uv sync --frozen --no-dev --no-install-project --no-editable diff --git a/testdata/dockerfile/git-sources.golden b/testdata/dockerfile/git-sources.golden index bd93dbf..25ab7e5 100644 --- a/testdata/dockerfile/git-sources.golden +++ b/testdata/dockerfile/git-sources.golden @@ -36,13 +36,24 @@ ENV UV_PROJECT_ENVIRONMENT=/app/.venv \ # With the caller's mirrors, git reads each locked repository from its mirror # instead of the URL the lock records (uv hands git that URL unchanged), so no # credential is needed, and a commit missing from the mirror fails here rather -# than prompting for one. The mappings are appended after any git configuration -# the environment already carries (GIT_CONFIG_COUNT), never in its place. +# than prompting for one. The mirror mappings take the first entries of git's +# environment configuration, so they win over an inherited rewrite of the same +# URL, and the entries the environment already carries (GIT_CONFIG_COUNT, read +# as git reads it) move after them, read by name and never evaluated. A count +# git would reject is left as it is, for git to reject as before. RUN --mount=type=bind,from=uvsources,target=/uvsources \ - if [ -n "$(ls -A /uvsources)" ]; then n=${GIT_CONFIG_COUNT:-0} && \ - export "GIT_CONFIG_KEY_$n=url.file:///uvsources/c9fa8363b851c2643d648edad1c8c7892904f717385511fe4771422d0817a55e.git.insteadOf" "GIT_CONFIG_VALUE_$n="'https://git.example.invalid/acme/mono' && n=$((n+1)) && \ - export "GIT_CONFIG_KEY_$n=url.file:///uvsources/d2b987abe743521016c33e82dba28324380629b9d2544f9e5213ee4299a3af6c.git.insteadOf" "GIT_CONFIG_VALUE_$n="'ssh://git@git.example.invalid:2222/acme/tools.git' && n=$((n+1)) && \ - export GIT_CONFIG_COUNT=$n GIT_TERMINAL_PROMPT=0; fi && \ + if [ -n "$(ls -A /uvsources)" ]; then \ + n=${GIT_CONFIG_COUNT:-0}; n=${n#+}; while [ "${n#0}" != "$n" ] && [ -n "${n#0}" ]; do n=${n#0}; done; \ + case $n in ''|*[!0-9]*) ;; *) \ + i=$n; while [ "$i" -gt 0 ]; do i=$((i-1)); \ + eval "k=\${GIT_CONFIG_KEY_$i}"; eval "v=\${GIT_CONFIG_VALUE_$i}"; \ + export "GIT_CONFIG_KEY_$((i+2))=$k" "GIT_CONFIG_VALUE_$((i+2))=$v"; \ + done; \ + export GIT_CONFIG_COUNT=$((n+2)) GIT_TERMINAL_PROMPT=0 \ + GIT_CONFIG_KEY_0='url.file:///uvsources/c9fa8363b851c2643d648edad1c8c7892904f717385511fe4771422d0817a55e.git.insteadOf' GIT_CONFIG_VALUE_0='https://git.example.invalid/acme/mono' \ + GIT_CONFIG_KEY_1='url.file:///uvsources/d2b987abe743521016c33e82dba28324380629b9d2544f9e5213ee4299a3af6c.git.insteadOf' GIT_CONFIG_VALUE_1='ssh://git@git.example.invalid:2222/acme/tools.git';; \ + esac; \ + fi && \ mkdir -p /opt/uv/python && uv sync --frozen --no-dev --no-install-project --no-editable diff --git a/uv_git_sources_recipe_test.go b/uv_git_sources_recipe_test.go index 24b81bf..d7adf5a 100644 --- a/uv_git_sources_recipe_test.go +++ b/uv_git_sources_recipe_test.go @@ -104,11 +104,11 @@ func TestLockedGitSourcesDeclaresEachLockedCommitOnce(t *testing.T) { require.NoError(t, err) requireDeclares(t, gitSourcesDeclared, declared) require.Equal(t, []unmirroredSource{ - {Package: "libe", Reason: "must start with https:// or ssh://"}, - {Package: "libf", Reason: "must start with https:// or ssh://"}, - {Package: "libg", Reason: "must carry no user information over https"}, - {Package: "libh", Reason: "must carry no percent-encoding"}, - {Package: "libi", Reason: "must carry no @ in the path"}, + {Package: "libe", Reason: "repository must start with https:// or ssh://"}, + {Package: "libf", Reason: "repository must start with https:// or ssh://"}, + {Package: "libg", Reason: "repository must carry no user information over https"}, + {Package: "libh", Reason: "repository must carry no percent-encoding"}, + {Package: "libi", Reason: "repository must carry no @ in the path"}, }, unmirrored) mirrors := uvGitMirrors(declared) @@ -274,30 +274,108 @@ example-git-lib = { git = %q, subdirectory = "lib", rev = %q } require.Contains(t, string(out), "git.example.invalid") } -// TestGitSourceMappingKeepsInheritedGitConfig runs the rendered mapping in the -// builder base image with git configuration already in the environment, as a -// base image or an earlier ENV may set it: the mappings are appended after it, -// and both are in effect. -func TestGitSourceMappingKeepsInheritedGitConfig(t *testing.T) { - if _, err := exec.LookPath("docker"); err != nil { - t.Skip("docker is not installed: the mapping cannot be run") +// lockWith is a uv-form lock with one package per git source, all at commitA. +func lockWith(sources ...string) string { + lock := "version = 1\n" + for i, source := range sources { + lock += fmt.Sprintf("\n[[package]]\nname = \"lib%d\"\nversion = \"0.1.0\"\nsource = { git = \"%s?rev=main#%s\" }\n", i, source, commitA) } + return lock +} + +// TestLockedGitSourcesLeavesOutAPrefixOfAnUndeclaredSource: git's insteadOf +// rewrites every URL that begins with the declared one, so a declared URL +// that begins another locked source's undeclared URL would send that source to +// the wrong mirror. It is left out too, until no such capture remains. +func TestLockedGitSourcesLeavesOutAPrefixOfAnUndeclaredSource(t *testing.T) { + lock := filepath.Join(t.TempDir(), "uv.lock") + writeFile(t, lock, lockWith("https://h.example/acme/mono", "https://h.example/acme/mono@v1", + "https://h.example/acme/x", "https://h.example/acme/xy", "https://h.example/acme/xyz@v1", "https://h.example/acme/kept")) + declared, unmirrored, err := lockedGitSources(lock) + require.NoError(t, err) + requireDeclares(t, []*builderv0.GitSourceDownload{{Repository: "https://h.example/acme/kept", Commit: commitA, MirrorContext: "uvsources"}}, declared) + var names []string + for _, source := range unmirrored { + names = append(names, source.Package) + } + require.ElementsMatch(t, []string{"lib0", "lib1", "lib2", "lib3", "lib4"}, names) + require.Contains(t, unmirrored, unmirroredSource{Package: "lib0", Reason: prefixCaptureReason}) + require.Contains(t, unmirrored, unmirroredSource{Package: "lib2", Reason: prefixCaptureReason}) +} + +// TestLockedGitSourcesKeepsALongerDeclaredSource: git applies the longest +// matching insteadOf, so two declared URLs, one beginning the other, each +// reach their own mirror. +func TestLockedGitSourcesKeepsALongerDeclaredSource(t *testing.T) { + lock := filepath.Join(t.TempDir(), "uv.lock") + writeFile(t, lock, lockWith("https://h.example/acme/mono", "https://h.example/acme/mono.git")) + declared, unmirrored, err := lockedGitSources(lock) + require.NoError(t, err) + require.Empty(t, unmirrored) + require.Len(t, declared, 2) +} + +// runMapping renders the recipe for lock and runs its mirror mapping, then +// script, in the builder base image with env set, as a base image or an +// earlier ENV may set it. +func runMapping(t *testing.T, lock string, env []string, script string) string { + t.Helper() builder, _ := createdBuilder(t) - writeFile(t, filepath.Join(builder.SourceLocation, "uv.lock"), gitSourcesLock) + writeFile(t, filepath.Join(builder.SourceLocation, "uv.lock"), lock) outputDir := t.TempDir() dockerfile := dockerfileOf(t, outputDir, emitRecipe(t, builder, outputDir)) - start := strings.Index(dockerfile, "RUN --mount=type=bind,from=uvsources,target=/uvsources \\\n") + run := "RUN --mount=type=bind,from=uvsources,target=/uvsources \\\n" + start := strings.Index(dockerfile, run) end := strings.Index(dockerfile, "mkdir -p /opt/uv/python && uv sync") require.True(t, start >= 0 && end > start, "the rendered sync step has no mirror mapping") - mapping := strings.ReplaceAll(dockerfile[start+len("RUN --mount=type=bind,from=uvsources,target=/uvsources \\\n"):end], "\\\n", "") + mapping := strings.ReplaceAll(dockerfile[start+len(run):end], "\\\n", "") mirrors := t.TempDir() writeFile(t, filepath.Join(mirrors, "present"), "") - mono := "url.file:///uvsources/" + services.GitSourceMirrorPath("https://git.example.invalid/acme/mono") + ".insteadOf" - out, err := exec.CommandContext(t.Context(), "docker", "run", "--rm", "--network=none", - "-e", "GIT_CONFIG_COUNT=1", "-e", "GIT_CONFIG_KEY_0=user.name", "-e", "GIT_CONFIG_VALUE_0=inherited", - "-v", mirrors+":/uvsources:ro", runtimeImage.FullName(), - "sh", "-c", mapping+"git config --get user.name && git config --get "+mono+" && echo count=$GIT_CONFIG_COUNT").CombinedOutput() + args := []string{"run", "--rm", "--network=none", "-w", "/tmp", "-v", mirrors + ":/uvsources:ro"} + for _, variable := range env { + args = append(args, "-e", variable) + } + out, err := exec.CommandContext(t.Context(), "docker", append(args, runtimeImage.FullName(), "sh", "-c", mapping+script)...).CombinedOutput() require.NoError(t, err, "%s", out) - require.Equal(t, "inherited\nhttps://git.example.invalid/acme/mono\ncount=3\n", string(out)) + return string(out) +} + +// TestGitSourceMappingInTheBuilderImage runs the rendered mapping where it +// runs: the mirror wins over an inherited rewrite of the same URL, inherited +// entries survive whatever form their count takes, and a source left out of +// the declaration is not rewritten by a mapping that begins it. +func TestGitSourceMappingInTheBuilderImage(t *testing.T) { + if _, err := exec.LookPath("docker"); err != nil { + t.Skip("docker is not installed: the mapping cannot be run") + } + const mono = "https://git.example.invalid/acme/mono" + mirror := "file:///uvsources/" + services.GitSourceMirrorPath(mono) + t.Run("inherited rewrite of a declared URL", func(t *testing.T) { + out := runMapping(t, gitSourcesLock, []string{"GIT_CONFIG_COUNT=2", + "GIT_CONFIG_KEY_0=url.https://proxy.invalid/.insteadOf", "GIT_CONFIG_VALUE_0=" + mono, + "GIT_CONFIG_KEY_1=user.name", "GIT_CONFIG_VALUE_1=inherited $HOME `x` \"q\""}, + "git ls-remote --get-url "+mono+" && git config --get user.name && git config --get url.https://proxy.invalid/.insteadOf && echo count=$GIT_CONFIG_COUNT") + require.Equal(t, mirror+"\ninherited $HOME `x` \"q\"\n"+mono+"\ncount=4\n", out) + }) + for _, count := range []string{"01", "+1"} { + t.Run("count "+count, func(t *testing.T) { + out := runMapping(t, gitSourcesLock, []string{"GIT_CONFIG_COUNT=" + count, "GIT_CONFIG_KEY_0=user.name", "GIT_CONFIG_VALUE_0=inherited"}, + "git ls-remote --get-url "+mono+" && git config --get user.name && echo count=$GIT_CONFIG_COUNT") + require.Equal(t, mirror+"\ninherited\ncount=3\n", out) + }) + } + t.Run("count 08", func(t *testing.T) { + env := []string{"GIT_CONFIG_COUNT=08"} + for i := 0; i < 8; i++ { + env = append(env, fmt.Sprintf("GIT_CONFIG_KEY_%d=test.k%d", i, i), fmt.Sprintf("GIT_CONFIG_VALUE_%d=v%d", i, i)) + } + out := runMapping(t, gitSourcesLock, env, "git ls-remote --get-url "+mono+" && git config --get test.k0 && git config --get test.k7 && echo count=$GIT_CONFIG_COUNT") + require.Equal(t, mirror+"\nv0\nv7\ncount=10\n", out) + }) + t.Run("an undeclared source the declared URL begins", func(t *testing.T) { + out := runMapping(t, lockWith(mono, mono+"@v1", "https://git.example.invalid/acme/tools"), nil, + "git ls-remote --get-url "+mono+"@v1 && git ls-remote --get-url "+mono) + require.Equal(t, mono+"@v1\n"+mono+"\n", out) + }) } diff --git a/uvgitsources.go b/uvgitsources.go index 33b896b..567fed4 100644 --- a/uvgitsources.go +++ b/uvgitsources.go @@ -45,7 +45,7 @@ type uvGitMirror struct { // unmirroredSource is a locked git source the recipe does not declare. type unmirroredSource struct { Package string - // Reason is core's, which never quotes the URL. + // Reason is core's, or the prefix capture; neither quotes the URL. Reason string } @@ -67,8 +67,11 @@ func uvLockPath(serviceDir, sourceDir string, membership *uvWorkspaceMembership) // A source core would refuse to declare (services.GitSourceRepositoryProblem: // http, file, a credential, percent-encoding, "@" in the path, ...) is // returned as unmirrored, with core's reason, and is neither declared nor -// rendered: the build fetches it itself, as it always has. A source whose -// commit is missing or not a full object id makes the lock malformed. +// rendered: the build fetches it itself, as it always has. So is a source +// whose URL begins another locked source's undeclared URL, because git's +// insteadOf rewrites by prefix and would send that source to this mirror. A +// source whose commit is missing or not a full object id makes the lock +// malformed. func lockedGitSources(lockPath string) (downloads []*builderv0.GitSourceDownload, unmirrored []unmirroredSource, err error) { content, err := os.ReadFile(lockPath) if errors.Is(err, os.ErrNotExist) { @@ -88,28 +91,61 @@ func lockedGitSources(lockPath string) (downloads []*builderv0.GitSourceDownload if err := toml.Unmarshal(content, &lock); err != nil { return nil, nil, fmt.Errorf("parse %s: %w", lockPath, err) } - seen := map[[2]string]struct{}{} + type accepted struct{ name, repository, commit string } + var candidates []accepted + var locked []string for _, pkg := range lock.Package { if pkg.Source.Git == "" { continue } // The URL is not quoted in errors: a lock written from a source that // named a credential may still carry one. - locked, commit, found := strings.Cut(pkg.Source.Git, "#") + source, commit, found := strings.Cut(pkg.Source.Git, "#") if !found || !services.IsGitSourceCommit(commit) { return nil, nil, fmt.Errorf("%s: package %q is locked to a git source without a full commit id: run `uv lock`", lockPath, pkg.Name) } - repository, _, _ := strings.Cut(locked, "?") + repository, _, _ := strings.Cut(source, "?") + locked = append(locked, repository) if reason := services.GitSourceRepositoryProblem(repository); reason != "" { - unmirrored = append(unmirrored, unmirroredSource{Package: pkg.Name, Reason: reason}) + unmirrored = append(unmirrored, unmirroredSource{Package: pkg.Name, Reason: "repository " + reason}) continue } - key := [2]string{repository, commit} + candidates = append(candidates, accepted{pkg.Name, repository, commit}) + } + // Leave out every declarable URL that begins an undeclared one, until none + // does: leaving one out can make it the undeclared URL a shorter one begins. + for changed := true; changed; { + changed = false + declared := map[string]bool{} + for _, candidate := range candidates { + declared[candidate.repository] = true + } + kept := candidates[:0] + for _, candidate := range candidates { + captures := false + for _, other := range locked { + if !declared[other] && strings.HasPrefix(other, candidate.repository) { + captures = true + break + } + } + if captures { + unmirrored = append(unmirrored, unmirroredSource{Package: candidate.name, Reason: prefixCaptureReason}) + changed = true + continue + } + kept = append(kept, candidate) + } + candidates = kept + } + seen := map[[2]string]struct{}{} + for _, candidate := range candidates { + key := [2]string{candidate.repository, candidate.commit} if _, ok := seen[key]; ok { continue } seen[key] = struct{}{} - downloads = append(downloads, &builderv0.GitSourceDownload{Repository: repository, Commit: commit, MirrorContext: uvSourcesContext}) + downloads = append(downloads, &builderv0.GitSourceDownload{Repository: candidate.repository, Commit: candidate.commit, MirrorContext: uvSourcesContext}) } sort.Slice(downloads, func(i, j int) bool { if downloads[i].GetRepository() != downloads[j].GetRepository() { @@ -133,3 +169,7 @@ func uvGitMirrors(downloads []*builderv0.GitSourceDownload) []uvGitMirror { } return mirrors } + +// prefixCaptureReason is why a declarable source is left out when git's +// prefix rewrite for it would also capture an undeclared locked source. +const prefixCaptureReason = "another locked source's URL begins with it, and git's rewrite would capture that source" From e611e22afde5587c81ecaaf3ab3f76bf14a28ce6 Mon Sep 17 00:00:00 2001 From: Saurabh Maheshwari <242053479+saurabhmaheshwari-obin@users.noreply.github.com> Date: Wed, 21 Oct 2026 18:11:47 -0400 Subject: [PATCH 09/11] fix(builder): append the mirror mappings only after a plain inherited count Moving inherited git config after the mirror mappings could not win precedence anyway (git breaks insteadOf ties by the order the destination base was registered, so a system or global rewrite still wins) and added edge cases of its own. The mappings are appended after the inherited entries again, without reading or moving them. They are used only when GIT_CONFIG_COUNT is unset, empty, or a plain count of at most four digits once one leading "+" and leading zeros are dropped; otherwise the git configuration is left alone, the build log says the sources are fetched without the mirror, and the build runs as it does today. An image whose own git configuration (system, global or environment) already rewrites a declared repository's URL keeps its rewrite: git prefers it, and the build fetches that source as it does without a mirror. Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 8 ++-- templates/builder/Dockerfile.tmpl | 26 ++++++------- testdata/dockerfile/git-sources.golden | 28 +++++++------- uv_git_sources_recipe_test.go | 51 +++++++++++++++----------- 4 files changed, 59 insertions(+), 54 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 7600796..0fdf4d9 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -162,10 +162,10 @@ into `templates/factory/code/pyproject.toml.tmpl` yourself. A git source is declared in the recipe (`git_source_downloads`) by the URL exactly as `uv.lock` records it up to its query, which is what uv hands git; the caller fetches it with its own credentials and the Dockerfile maps that - URL onto the mirror, ahead of inherited git config. A normalized URL would - miss the mirror silently. A source core refuses, or whose URL begins an - undeclared locked one (insteadOf matches by prefix), is not declared or - rendered; a warning names the package and why, and the build fetches it. + URL onto the mirror; a normalized URL would miss it silently. A source core + refuses, or whose URL begins an undeclared locked one (insteadOf is a + prefix match), is not declared; a warning says why. An image whose own git + config already rewrites a declared URL keeps that rewrite (git prefers it). - **The runtime image is pinned.** `docker-image` rejects a bare name and `:latest`. The default `ghcr.io/codefly-dev/python` companion is built from core and pinned by the digest of its multi-platform index — prefer it. Its tag diff --git a/templates/builder/Dockerfile.tmpl b/templates/builder/Dockerfile.tmpl index 3430eb0..fb84146 100644 --- a/templates/builder/Dockerfile.tmpl +++ b/templates/builder/Dockerfile.tmpl @@ -58,22 +58,20 @@ ENV UV_PROJECT_ENVIRONMENT=/app/.venv \ # With the caller's mirrors, git reads each locked repository from its mirror # instead of the URL the lock records (uv hands git that URL unchanged), so no # credential is needed, and a commit missing from the mirror fails here rather -# than prompting for one. The mirror mappings take the first entries of git's -# environment configuration, so they win over an inherited rewrite of the same -# URL, and the entries the environment already carries (GIT_CONFIG_COUNT, read -# as git reads it) move after them, read by name and never evaluated. A count -# git would reject is left as it is, for git to reject as before. +# than prompting for one. The mappings are appended after the git configuration +# the environment already carries, when its GIT_CONFIG_COUNT is a plain count; +# otherwise the configuration is left alone and the sources are fetched as +# without a mirror. An image whose own git configuration (system, global or +# environment) already rewrites a declared repository's URL keeps its rewrite: +# git prefers it, and the build fetches that source as it does without a mirror. RUN --mount=type=bind,from=uvsources,target=/uvsources \ if [ -n "$(ls -A /uvsources)" ]; then \ - n=${GIT_CONFIG_COUNT:-0}; n=${n#+}; while [ "${n#0}" != "$n" ] && [ -n "${n#0}" ]; do n=${n#0}; done; \ - case $n in ''|*[!0-9]*) ;; *) \ - i=$n; while [ "$i" -gt 0 ]; do i=$((i-1)); \ - eval "k=\${GIT_CONFIG_KEY_$i}"; eval "v=\${GIT_CONFIG_VALUE_$i}"; \ - export "GIT_CONFIG_KEY_$((i+{{len .GitSources}}))=$k" "GIT_CONFIG_VALUE_$((i+{{len .GitSources}}))=$v"; \ - done; \ - export GIT_CONFIG_COUNT=$((n+{{len .GitSources}})) GIT_TERMINAL_PROMPT=0{{range $i, $source := .GitSources}} \ - GIT_CONFIG_KEY_{{$i}}='url.file:///uvsources/{{$source.Path}}.insteadOf' GIT_CONFIG_VALUE_{{$i}}='{{$source.Repository}}'{{end}};; \ - esac; \ + n=${GIT_CONFIG_COUNT:-0}; n=${n#+}; \ + case $n in ''|*[!0-9]*) n= ;; *) while [ "${n#0}" != "$n" ] && [ -n "${n#0}" ]; do n=${n#0}; done; [ ${#n} -le 4 ] || n= ;; esac; \ + if [ -z "$n" ]; then echo "inherited GIT_CONFIG_COUNT is not a plain count; git sources are fetched without the mirror"; \ + else export GIT_TERMINAL_PROMPT=0{{range $i, $source := .GitSources}} \ + "GIT_CONFIG_KEY_$((n+{{$i}}))=url.file:///uvsources/{{$source.Path}}.insteadOf" "GIT_CONFIG_VALUE_$((n+{{$i}}))="'{{$source.Repository}}'{{end}} \ + GIT_CONFIG_COUNT=$((n+{{len .GitSources}})); fi; \ fi && \ mkdir -p /opt/uv/python && uv sync --frozen --no-dev --no-install-project --no-editable {{- else}} diff --git a/testdata/dockerfile/git-sources.golden b/testdata/dockerfile/git-sources.golden index 25ab7e5..3c49c69 100644 --- a/testdata/dockerfile/git-sources.golden +++ b/testdata/dockerfile/git-sources.golden @@ -36,23 +36,21 @@ ENV UV_PROJECT_ENVIRONMENT=/app/.venv \ # With the caller's mirrors, git reads each locked repository from its mirror # instead of the URL the lock records (uv hands git that URL unchanged), so no # credential is needed, and a commit missing from the mirror fails here rather -# than prompting for one. The mirror mappings take the first entries of git's -# environment configuration, so they win over an inherited rewrite of the same -# URL, and the entries the environment already carries (GIT_CONFIG_COUNT, read -# as git reads it) move after them, read by name and never evaluated. A count -# git would reject is left as it is, for git to reject as before. +# than prompting for one. The mappings are appended after the git configuration +# the environment already carries, when its GIT_CONFIG_COUNT is a plain count; +# otherwise the configuration is left alone and the sources are fetched as +# without a mirror. An image whose own git configuration (system, global or +# environment) already rewrites a declared repository's URL keeps its rewrite: +# git prefers it, and the build fetches that source as it does without a mirror. RUN --mount=type=bind,from=uvsources,target=/uvsources \ if [ -n "$(ls -A /uvsources)" ]; then \ - n=${GIT_CONFIG_COUNT:-0}; n=${n#+}; while [ "${n#0}" != "$n" ] && [ -n "${n#0}" ]; do n=${n#0}; done; \ - case $n in ''|*[!0-9]*) ;; *) \ - i=$n; while [ "$i" -gt 0 ]; do i=$((i-1)); \ - eval "k=\${GIT_CONFIG_KEY_$i}"; eval "v=\${GIT_CONFIG_VALUE_$i}"; \ - export "GIT_CONFIG_KEY_$((i+2))=$k" "GIT_CONFIG_VALUE_$((i+2))=$v"; \ - done; \ - export GIT_CONFIG_COUNT=$((n+2)) GIT_TERMINAL_PROMPT=0 \ - GIT_CONFIG_KEY_0='url.file:///uvsources/c9fa8363b851c2643d648edad1c8c7892904f717385511fe4771422d0817a55e.git.insteadOf' GIT_CONFIG_VALUE_0='https://git.example.invalid/acme/mono' \ - GIT_CONFIG_KEY_1='url.file:///uvsources/d2b987abe743521016c33e82dba28324380629b9d2544f9e5213ee4299a3af6c.git.insteadOf' GIT_CONFIG_VALUE_1='ssh://git@git.example.invalid:2222/acme/tools.git';; \ - esac; \ + n=${GIT_CONFIG_COUNT:-0}; n=${n#+}; \ + case $n in ''|*[!0-9]*) n= ;; *) while [ "${n#0}" != "$n" ] && [ -n "${n#0}" ]; do n=${n#0}; done; [ ${#n} -le 4 ] || n= ;; esac; \ + if [ -z "$n" ]; then echo "inherited GIT_CONFIG_COUNT is not a plain count; git sources are fetched without the mirror"; \ + else export GIT_TERMINAL_PROMPT=0 \ + "GIT_CONFIG_KEY_$((n+0))=url.file:///uvsources/c9fa8363b851c2643d648edad1c8c7892904f717385511fe4771422d0817a55e.git.insteadOf" "GIT_CONFIG_VALUE_$((n+0))="'https://git.example.invalid/acme/mono' \ + "GIT_CONFIG_KEY_$((n+1))=url.file:///uvsources/d2b987abe743521016c33e82dba28324380629b9d2544f9e5213ee4299a3af6c.git.insteadOf" "GIT_CONFIG_VALUE_$((n+1))="'ssh://git@git.example.invalid:2222/acme/tools.git' \ + GIT_CONFIG_COUNT=$((n+2)); fi; \ fi && \ mkdir -p /opt/uv/python && uv sync --frozen --no-dev --no-install-project --no-editable diff --git a/uv_git_sources_recipe_test.go b/uv_git_sources_recipe_test.go index d7adf5a..be23b68 100644 --- a/uv_git_sources_recipe_test.go +++ b/uv_git_sources_recipe_test.go @@ -342,36 +342,45 @@ func runMapping(t *testing.T, lock string, env []string, script string) string { } // TestGitSourceMappingInTheBuilderImage runs the rendered mapping where it -// runs: the mirror wins over an inherited rewrite of the same URL, inherited -// entries survive whatever form their count takes, and a source left out of -// the declaration is not rewritten by a mapping that begins it. +// runs: appended after the git configuration the environment carries when its +// count is plain, leaving that configuration alone otherwise, and never +// rewriting a source left out of the declaration. func TestGitSourceMappingInTheBuilderImage(t *testing.T) { if _, err := exec.LookPath("docker"); err != nil { t.Skip("docker is not installed: the mapping cannot be run") } const mono = "https://git.example.invalid/acme/mono" mirror := "file:///uvsources/" + services.GitSourceMirrorPath(mono) - t.Run("inherited rewrite of a declared URL", func(t *testing.T) { - out := runMapping(t, gitSourcesLock, []string{"GIT_CONFIG_COUNT=2", - "GIT_CONFIG_KEY_0=url.https://proxy.invalid/.insteadOf", "GIT_CONFIG_VALUE_0=" + mono, - "GIT_CONFIG_KEY_1=user.name", "GIT_CONFIG_VALUE_1=inherited $HOME `x` \"q\""}, - "git ls-remote --get-url "+mono+" && git config --get user.name && git config --get url.https://proxy.invalid/.insteadOf && echo count=$GIT_CONFIG_COUNT") - require.Equal(t, mirror+"\ninherited $HOME `x` \"q\"\n"+mono+"\ncount=4\n", out) - }) - for _, count := range []string{"01", "+1"} { + inherited := []string{"GIT_CONFIG_KEY_0=user.name", "GIT_CONFIG_VALUE_0=inherited"} + for _, count := range []string{"", "1", "01", "+1"} { t.Run("count "+count, func(t *testing.T) { - out := runMapping(t, gitSourcesLock, []string{"GIT_CONFIG_COUNT=" + count, "GIT_CONFIG_KEY_0=user.name", "GIT_CONFIG_VALUE_0=inherited"}, - "git ls-remote --get-url "+mono+" && git config --get user.name && echo count=$GIT_CONFIG_COUNT") - require.Equal(t, mirror+"\ninherited\ncount=3\n", out) + env := append([]string{"GIT_CONFIG_COUNT=" + count}, inherited...) + if count == "" { + env = nil + } + out := runMapping(t, gitSourcesLock, env, "git ls-remote --get-url "+mono+" && { git config --get user.name || true; } && echo count=$GIT_CONFIG_COUNT") + want := mirror + "\ninherited\ncount=3\n" + if count == "" { + want = mirror + "\ncount=2\n" + } + require.Equal(t, want, out) }) } - t.Run("count 08", func(t *testing.T) { - env := []string{"GIT_CONFIG_COUNT=08"} - for i := 0; i < 8; i++ { - env = append(env, fmt.Sprintf("GIT_CONFIG_KEY_%d=test.k%d", i, i), fmt.Sprintf("GIT_CONFIG_VALUE_%d=v%d", i, i)) - } - out := runMapping(t, gitSourcesLock, env, "git ls-remote --get-url "+mono+" && git config --get test.k0 && git config --get test.k7 && echo count=$GIT_CONFIG_COUNT") - require.Equal(t, mirror+"\nv0\nv7\ncount=10\n", out) + for _, count := range []string{" 1", "-0", "99999", "abc"} { + t.Run("count "+count, func(t *testing.T) { + env := append([]string{"GIT_CONFIG_COUNT=" + count}, inherited...) + out := runMapping(t, gitSourcesLock, env, `echo "count=[$GIT_CONFIG_COUNT] keys=$(env | grep -c '^GIT_CONFIG_KEY_') prompt=[$GIT_TERMINAL_PROMPT]"`) + require.Equal(t, "inherited GIT_CONFIG_COUNT is not a plain count; git sources are fetched without the mirror\ncount=["+count+"] keys=1 prompt=[]\n", out) + }) + } + // The limitation, as it is today without a mirror: an image whose own git + // configuration rewrites a declared URL keeps its rewrite, which git + // prefers, so the source is fetched where the image says, not broken. + t.Run("inherited rewrite of a declared URL", func(t *testing.T) { + out := runMapping(t, gitSourcesLock, []string{"GIT_CONFIG_COUNT=1", + "GIT_CONFIG_KEY_0=url.https://proxy.invalid/.insteadOf", "GIT_CONFIG_VALUE_0=" + mono}, + "git ls-remote --get-url "+mono+" && echo count=$GIT_CONFIG_COUNT") + require.Equal(t, "https://proxy.invalid/\ncount=3\n", out) }) t.Run("an undeclared source the declared URL begins", func(t *testing.T) { out := runMapping(t, lockWith(mono, mono+"@v1", "https://git.example.invalid/acme/tools"), nil, From f4d219ac4703a7d5218b18032ea05729341dea87 Mon Sep 17 00:00:00 2001 From: Saurabh Maheshwari <242053479+saurabhmaheshwari-obin@users.noreply.github.com> Date: Fri, 23 Oct 2026 14:00:06 -0400 Subject: [PATCH 10/11] build(deps): pin core to v0.19.6, which carries the git source declaration (#832) Co-Authored-By: Claude Opus 5.5 --- go.mod | 4 ++-- go.sum | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index dfd83dd..a90dcd9 100644 --- a/go.mod +++ b/go.mod @@ -1,9 +1,9 @@ module github.com/codefly-dev/service-python-fastapi -go 1.27.2 +go 1.27.3 require ( - github.com/codefly-dev/core v0.19.6-0.20261021213826-abdc7077fb98 + github.com/codefly-dev/core v0.19.6 github.com/codefly-dev/service-python v0.0.66 github.com/pelletier/go-toml/v2 v2.4.3 github.com/stretchr/testify v1.12.1 diff --git a/go.sum b/go.sum index 10f596e..4630295 100644 --- a/go.sum +++ b/go.sum @@ -43,8 +43,8 @@ github.com/clipperhouse/uax29/v2 v2.7.0 h1:+gs4oBZ2gPfVrKPthwbMzWZDaAFPGYK72F0NJ github.com/clipperhouse/uax29/v2 v2.7.0/go.mod h1:EFJ2TJMRUaplDxHKj1qAEhCtQPW2tJSwu5BF98AuoVM= github.com/cloudflare/circl v1.6.5 h1:O64F26HEqNhznd/hrC5KZXVKYuKM2rx4deZDTc4ihQA= github.com/cloudflare/circl v1.6.5/go.mod h1:h5LNyxAc5nTue9DS5jT+48en2PSDYt3zdGnz5OstK6c= -github.com/codefly-dev/core v0.19.6-0.20261021213826-abdc7077fb98 h1:/z9xBAdnw0vAfqZpZH58zmifFxSYZVqyQziKcMTX8Xo= -github.com/codefly-dev/core v0.19.6-0.20261021213826-abdc7077fb98/go.mod h1:0uX/7lTBVaGV0RI2Mb3mpEr/PPdx/jZqXHym/U8FVDI= +github.com/codefly-dev/core v0.19.6 h1:ZkCChK/BioK+B7zZ0fp+/rXDCy0k+lsdFqHjyc5xFIg= +github.com/codefly-dev/core v0.19.6/go.mod h1:1ina3kP+YSAL4tlZvnEYd4P8bFGnJECfkP86ojeeF1A= github.com/codefly-dev/gortk v0.2.0 h1:7bOlS5valYz2zil+fZctQNcPCYBcPj86abcw9N8h1hQ= github.com/codefly-dev/gortk v0.2.0/go.mod h1:dDWUMFgAP063OCGhTEpV7FFUj9+zTcxxO/nV80VKEwg= github.com/codefly-dev/service-python v0.0.66 h1:Y74QvKAYMaOnZcsfRxuD9dpQo3//7Mjo12RYELyrp5o= From 19a8b2242ef26b67feffb0cd1fb6a836ee47d1c8 Mon Sep 17 00:00:00 2001 From: Saurabh Maheshwari <242053479+saurabhmaheshwari-obin@users.noreply.github.com> Date: Fri, 23 Oct 2026 16:21:29 -0400 Subject: [PATCH 11/11] ci: run the reusable service CI from core v0.19.6, the linked core's tag Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 2 +- AGENTS.md | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f1236f2..89746f2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,7 +12,7 @@ concurrency: jobs: ci: - uses: codefly-dev/core/.github/workflows/go-service-ci.yml@v0.19.4 + uses: codefly-dev/core/.github/workflows/go-service-ci.yml@v0.19.6 with: # The Python runtime tests drive `uv` on the host (dependency sync, # pytest); it isn't on the runner by default. diff --git a/AGENTS.md b/AGENTS.md index 0fdf4d9..0259e4a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -67,9 +67,9 @@ within reach. ## Build and test Derived from `.github/workflows/ci.yml`, which delegates to core's shared -`go-service-ci.yml`, pinned to the linked core's tag (`@v0.19.4`; move it with +`go-service-ci.yml`, pinned to the linked core's tag (`@v0.19.6`; move it with the core bump), and adds a `race` job of its own. Go comes from `go.mod` -(1.27.2). All of it runs locally: +(1.27.3). All of it runs locally: ```bash go build ./... # CI: build job