feat(build): fetch declared git sources on the host and hand the build a mirror context

.github/workflows/go.yml

@@ -190,6 +190,14 @@ jobs:
         if: matrix.gate == 'control-integration'
         run: go test -tags=integration -run '^TestTwoServicesPinningOneRepositoryFetchWithoutGitMaintainingTheCache$' ./pkg/orchestration -v
 
+      # A locked git commit only the host's git configuration reaches is
+      # mirrored on the host, by id and through a host that refuses unadvertised
+      # commits, and an image build with no network fetches it from the mirror
+      # context and fails without it. Needs git and Docker; no provider toolchain.
+      - name: Prove a git source build needs no credential with its prefetched mirror
+        if: matrix.gate == 'control-integration'
+        run: go test -tags=integration -run '^(TestALockedGitSourceIsMirroredWithTheHostsGitConfiguration|TestAGitSourceBuildNeedsNoCredentialWithThePrefetchedMirror)$' ./pkg/orchestration -v
+
       # Keep the tag in step with qualificationImage in
       # cmd/generate/container_recovery_docker_test.go.
       - name: Pull the interrupted-generate proof's image

docs/supported-matrix.md

@@ -27,7 +27,10 @@ When `go.mod` moves to a new core release line, update the `core` field in
 `go test ./pkg/conformance/...` before the full coverage and race gates.
 `TestMatrixCoreMatchesGoMod` refuses a stale declared version.
 
-The current pin is a `v0.19.6` pseudo-version of core, ahead of any tag:
+The current pin is a `v0.19.7` pseudo-version of core: `257e6b23`, branch
+`chore/envelope-on-0.19.6`, which merges `v0.19.6` (with the git source
+declaration, codefly-dev/core#832) into `5522c4da`, the commit carrying the
+module envelope that `pkg/delivery/signing` uses. `v0.19.6` carries
 `b81d92e6`, the merge of codefly-dev/core#812 — the module manifest
 (`solutionhost.ModuleManifest`, one signed document per module, whose
 workload is `(service, container)` and which names no build) and

go.mod

@@ -11,7 +11,7 @@ require (
 	github.com/asottile/dockerfile v3.1.0+incompatible
 	github.com/blang/semver v3.5.1+incompatible
 	github.com/briandowns/spinner v1.23.2
-	github.com/codefly-dev/core v0.19.6-0.20261022093222-5522c4dac804
+	github.com/codefly-dev/core v0.19.7-0.20261023201433-257e6b234bd3
 	github.com/codefly-dev/golor v0.1.3
 	github.com/codefly-dev/llm v0.1.7
 	github.com/codefly-dev/sdk-go/workcontext v0.1.1-0.20261018052022-70dc7a546458

go.sum

@@ -103,8 +103,8 @@ github.com/clipperhouse/uax29/v2 v2.7.0 h1:+gs4oBZ2gPfVrKPthwbMzWZDaAFPGYK72F0NJ
 github.com/clipperhouse/uax29/v2 v2.7.0/go.mod h1:EFJ2TJMRUaplDxHKj1qAEhCtQPW2tJSwu5BF98AuoVM=
 github.com/cloudflare/circl v1.6.5 h1:O64F26HEqNhznd/hrC5KZXVKYuKM2rx4deZDTc4ihQA=
 github.com/cloudflare/circl v1.6.5/go.mod h1:h5LNyxAc5nTue9DS5jT+48en2PSDYt3zdGnz5OstK6c=
-github.com/codefly-dev/core v0.19.6-0.20261022093222-5522c4dac804 h1:JyW6eDXENq/BqNJbPy0De30MpFHoEDBW9Uwc0eR2Prs=
-github.com/codefly-dev/core v0.19.6-0.20261022093222-5522c4dac804/go.mod h1:1ina3kP+YSAL4tlZvnEYd4P8bFGnJECfkP86ojeeF1A=
+github.com/codefly-dev/core v0.19.7-0.20261023201433-257e6b234bd3 h1:Qi+DwoxKK7UU6OVmiwj+bLIqlvrLXk5ttnz96+2f97A=
+github.com/codefly-dev/core v0.19.7-0.20261023201433-257e6b234bd3/go.mod h1:1ina3kP+YSAL4tlZvnEYd4P8bFGnJECfkP86ojeeF1A=
 github.com/codefly-dev/golor v0.1.3 h1:xmo+ceyJFRYZdvpWE2fNd0jeaadp/Ibm1BnganiGKOc=
 github.com/codefly-dev/golor v0.1.3/go.mod h1:sl/u/K1l7J0Pr3xyVZp8fOJYQItKKst1No9JqgzLLoY=
 github.com/codefly-dev/gortk v0.2.0 h1:7bOlS5valYz2zil+fZctQNcPCYBcPj86abcw9N8h1hQ=

pkg/conformance/matrix.json

@@ -1,7 +1,7 @@
 {
   "schema_version": 1,
   "cli": "source",
-  "core": "v0.19.6",
+  "core": "v0.19.7",
   "rows": [
     {
       "id": "linux-amd64-source",

pkg/orchestration/build_git_sources.go

@@ -0,0 +1,289 @@
+package orchestration
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"sync"
+	"syscall"
+	"time"
+
+	coreservices "github.com/codefly-dev/core/agents/services"
+	builderv0 "github.com/codefly-dev/core/generated/go/codefly/services/builder/v0"
+)
+
+// gitSourcePrefetch fetches the locked git commits a flow's recipes declare
+// (DockerBuildRecipe.git_source_downloads) once, on the host, before any image
+// is built, and hands each build the result as bare mirrors in a named build
+// context. It is goModulePrefetch for a dependency a lockfile pins to a git
+// commit rather than to a module version.
+//
+// The fetch runs with the host's own git and environment, so it authenticates
+// exactly as the developer's or the CI job's git does (insteadOf rewrites, a
+// credential helper, an ssh agent); nothing secret reaches the build. Like the
+// go tool's fetches, which run in repositories under the module cache, it runs
+// in a repository of its own, so the configuration of a checkout the CLI runs
+// from is not consulted; global and environment configuration is.
+//
+// Each repository has one bare mirror at GitSourceMirrorPath(repository),
+// holding each declared commit under GitSourceMirrorRef(commit). The mirror is
+// shallow by design: it carries the locked commits and nothing else, which
+// serves a consumer that fetches by commit id, as a package manager installing
+// a locked git dependency does (`git fetch <url> +<commit>:refs/commit/<commit>`).
+// Every recipe of the flow is handed the one directory holding all the mirrors,
+// as every Go recipe is handed one module proxy: a build fetches only the
+// commits its own lockfile names, so another service's mirrors are inert.
+type gitSourcePrefetch struct {
+	mu      sync.Mutex
+	root    string
+	fetched map[[2]string]bool
+	// formats is the object format each repository's mirror was created with.
+	formats map[string]string
+	// run executes git and returns its standard output; a test substitutes it.
+	run func(ctx context.Context, env []string, args ...string) (string, error)
+}
+
+func newGitSourcePrefetch() *gitSourcePrefetch {
+	return &gitSourcePrefetch{fetched: map[[2]string]bool{}, formats: map[string]string{}, run: runGitCommand}
+}
+
+// fetch places commit of repository in its mirror and returns the directory
+// holding every mirror. A commit fetched earlier in the flow is not fetched
+// again.
+func (p *gitSourcePrefetch) fetch(ctx context.Context, repository, commit string) (string, error) {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	if p.fetched[[2]string{repository, commit}] {
+		return p.root, nil
+	}
+	// A 40-digit id is SHA-1 and a 64-digit one SHA-256; a repository has one.
+	format := "sha1"
+	if len(commit) == 64 {
+		format = "sha256"
+	}
+	if known, ok := p.formats[repository]; ok && known != format {
+		return "", fmt.Errorf("a git source is declared with both %s and %s commit ids", known, format)
+	}
+	if p.root == "" {
+		root, err := os.MkdirTemp("", "codefly-git-sources-*")
+		if err != nil {
+			return "", fmt.Errorf("cannot stage the git source prefetch: %w", err)
+		}
+		p.root = root
+	}
+	mirror := filepath.Join(p.root, coreservices.GitSourceMirrorPath(repository))
+	env := gitSourcePrefetchEnv(os.Environ())
+	if _, known := p.formats[repository]; !known {
+		// An empty template: a host template's hooks or configuration would
+		// otherwise be copied into the mirror, and so into the build context.
+		if _, err := p.run(ctx, env, "init", "--quiet", "--bare", "--template=", "--object-format="+format, mirror); err != nil {
+			return "", fmt.Errorf("cannot create the git source mirror: %w", err)
+		}
+		p.formats[repository] = format
+	}
+	git := func(args ...string) (string, error) {
+		return p.run(ctx, env, append([]string{"--git-dir=" + mirror}, args...)...)
+	}
+	ref := coreservices.GitSourceMirrorRef(commit)
+	// A host that serves a commit by id gives just that commit. One that refuses
+	// an unadvertised commit (protocol v0 without allowAnySHA1InWant) gives it
+	// with its branches and tags, which is how any git client then reaches it.
+	if _, err := git("fetch", "--depth=1", "--end-of-options", repository, "+"+commit+":"+ref); err != nil {
+		full := []string{"fetch"}
+		if _, shallow := os.Stat(filepath.Join(mirror, "shallow")); shallow == nil {
+			full = append(full, "--unshallow")
+		}
+		full = append(full, "--end-of-options", repository, "+refs/heads/*:refs/heads/*", "+refs/tags/*:refs/tags/*")
+		if _, fullErr := git(full...); fullErr != nil {
+			return "", fmt.Errorf("cannot fetch commit %s of a declared git source: %w", commit, errors.Join(err, fullErr))
+		}
+		// The branches and tags arrived but none reaches the commit: the
+		// source no longer holds it, whatever git's own wording was.
+		if _, err := git("update-ref", ref, commit); err != nil {
+			return "", fmt.Errorf("%s: the declared git source does not hold commit %s", gitCommitUnavailable, commit)
+		}
+	}
+	// The mirror ref must name exactly the locked commit: a branch spelled like
+	// a commit id is never taken for it.
+	resolved, err := git("rev-parse", "--verify", "--end-of-options", ref+"^{commit}")
+	if err != nil {
+		return "", fmt.Errorf("%s: the declared git source does not hold commit %s", gitCommitUnavailable, commit)
+	}
+	if resolved != commit {
+		return "", fmt.Errorf("%s: the mirror ref of commit %s resolves to %s", gitCommitUnavailable, commit, resolved)
+	}
+	p.fetched[[2]string{repository, commit}] = true
+	return p.root, nil
+}
+
+// mirrorsFor fetches every git source a recipe declares and returns the
+// directory to supply under each declared mirror context.
+func (p *gitSourcePrefetch) mirrorsFor(ctx context.Context, recipe *builderv0.DockerBuildRecipe) (map[string]string, error) {
+	downloads := recipe.GetGitSourceDownloads()
+	if len(downloads) == 0 {
+		return nil, nil
+	}
+	mirrors := make(map[string]string, len(downloads))
+	for index, download := range downloads {
+		root, err := p.fetch(ctx, download.GetRepository(), download.GetCommit())
+		if err != nil {
+			return nil, fmt.Errorf("recipe %s git source %d: %w", recipe.GetName(), index, err)
+		}
+		mirrors[download.GetMirrorContext()] = root
+	}
+	return mirrors, nil
+}
+
+// recipeMissesGitSourceMirrors reports whether a recipe declares a git source
+// that has no fetched mirror under its build-context name.
+func recipeMissesGitSourceMirrors(recipe *builderv0.DockerBuildRecipe, contexts map[string]string) bool {
+	for _, download := range recipe.GetGitSourceDownloads() {
+		if _, ok := contexts[download.GetMirrorContext()]; !ok {
+			return true
+		}
+	}
+	return false
+}
+
+// Close removes every mirror the flow fetched.
+func (p *gitSourcePrefetch) Close() error {
+	if p == nil {
+		return nil
+	}
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	if p.root == "" {
+		return nil
+	}
+	err := os.RemoveAll(p.root)
+	p.root = ""
+	p.fetched = map[[2]string]bool{}
+	p.formats = map[string]string{}
+	return err
+}
+
+// gitSourcePrefetchEnv is the host environment, so the host's git configuration
+// decides how a repository is reached, as goPrefetchEnv leaves it to decide how
+// a module is. What it imposes:
+//   - no prompt for a credential the host lacks (GIT_TERMINAL_PROMPT=0, and
+//     GCM_INTERACTIVE=never unless the host chose, as the go tool does), so the
+//     fetch fails instead of hanging;
+//   - gitPrefetchConfig;
+//   - no variable pointing git at another repository or template;
+//   - no tracing, whether the host enables it in the environment or in its
+//     git configuration: a trace prints a credential helper's command line and
+//     protocol traffic into the output this reports or a file.
+//
+// It sets no GIT_ALLOW_PROTOCOL: git checks that after the host's insteadOf
+// rewrites, so it would refuse a host rewrite to a local path; the declared URL
+// itself is https or ssh, which the recipe's validation enforces.
+func gitSourcePrefetchEnv(base []string) []string {
+	env := make([]string, 0, len(base)+2)
+	interactive := false
+	for _, entry := range base {
+		key, value, _ := strings.Cut(entry, "=")
+		switch key {
+		case "GIT_TERMINAL_PROMPT", "GIT_DIR", "GIT_WORK_TREE", "GIT_COMMON_DIR", "GIT_INDEX_FILE",
+			"GIT_OBJECT_DIRECTORY", "GIT_ALTERNATE_OBJECT_DIRECTORIES", "GIT_NAMESPACE",
+			"GIT_TEMPLATE_DIR", "GIT_CURL_VERBOSE":
+			continue
+		case "GCM_INTERACTIVE":
+			interactive = value != ""
+		}
+		if strings.HasPrefix(key, "GIT_TRACE") {
+			continue
+		}
+		env = append(env, entry)
+	}
+	// Trace2 is also enabled from configuration (trace2.*Target), which the
+	// environment overrides; turning each target off here covers both.
+	env = append(env, "GIT_TERMINAL_PROMPT=0", "GIT_TRACE2=0", "GIT_TRACE2_EVENT=0", "GIT_TRACE2_PERF=0")
+	if !interactive {
+		env = append(env, "GCM_INTERACTIVE=never")
+	}
+	return withGitPrefetchConfig(env)
+}
+
+var errGitMissing = errors.New("a recipe declares git sources, which the CLI fetches with the host's git, and `git` is not on PATH")
+
+// gitWaitDelay bounds how long a cancelled git may hold its output open: a
+// credential helper or askpass it started can outlive it.
+const gitWaitDelay = 5 * time.Second
+
+// runGitCommand runs git in its own process group, so cancelling the context
+// stops git and every helper it started, and returns its trimmed standard
+// output.
+//
+// A failure never carries git's own output. git prints a credential helper's
+// command line, a missing helper's path, and whatever a helper or askpass
+// writes to stderr, and none of that is a URL a redaction can recognise. The
+// error names the git subcommand, its exit status and a reason classified from
+// the output, which is matched here and then dropped.
+func runGitCommand(ctx context.Context, env []string, args ...string) (string, error) {
+	gitBinary, err := exec.LookPath("git")
+	if err != nil {
+		return "", errGitMissing
+	}
+	command := exec.CommandContext(ctx, gitBinary, args...)
+	command.Env = env
+	command.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
+	command.Cancel = func() error { return syscall.Kill(-command.Process.Pid, syscall.SIGKILL) }
+	command.WaitDelay = gitWaitDelay
+	var stdout, output bytes.Buffer
+	command.Stdout = &stdout
+	command.Stderr = &output
+	if err := command.Run(); err != nil {
+		subcommand := gitSubcommand(args)
+		failure := fmt.Errorf("git %s exited with status %d: %s; rerun the same git %s with the host's git to see its own output",
+			subcommand, command.ProcessState.ExitCode(), classifyGitFailure(output.String()+stdout.String()), subcommand)
+		if ctx.Err() != nil {
+			return "", errors.Join(failure, ctx.Err())
+		}
+		return "", failure
+	}
+	return strings.TrimSpace(stdout.String()), nil
+}
+
+// gitSubcommand is the first argument that is not an option.
+func gitSubcommand(args []string) string {
+	for _, argument := range args {
+		if !strings.HasPrefix(argument, "-") {
+			return argument
+		}
+	}
+	return "git"
+}
+
+// gitFailureReasons is the fixed set a git failure is reported as, each with
+// the output that identifies it, checked in order.
+var gitFailureReasons = []struct {
+	reason  string
+	markers []string
+}{
+	{gitCommitUnavailable, []string{"not our ref", "couldn't find remote ref", "unadvertised object"}},
+	{"authentication failed", []string{"authentication failed", "could not read username", "could not read password",
+		"terminal prompts disabled", "permission denied (publickey)", "told us to quit"}},
+	{"host unreachable", []string{"could not resolve host", "connection refused", "connection timed out", "failed to connect"}},
+	{"repository not found", []string{"repository not found", "does not appear to be a git repository", "not found"}},
+}
+
+const gitCommitUnavailable = "commit not available"
+
+// classifyGitFailure reduces git's output to one of gitFailureReasons, or
+// "git failed".
+func classifyGitFailure(output string) string {
+	output = strings.ToLower(output)
+	for _, candidate := range gitFailureReasons {
+		for _, marker := range candidate.markers {
+			if strings.Contains(output, marker) {
+				return candidate.reason
+			}
+		}
+	}
+	return "git failed"
+}