diff --git a/.github/workflows/companions-build.yml b/.github/workflows/companions-build.yml index 25675a46..c62f60fa 100644 --- a/.github/workflows/companions-build.yml +++ b/.github/workflows/companions-build.yml @@ -48,8 +48,8 @@ jobs: build: needs: plan runs-on: ubuntu-latest - # The proto companion compiles ~10 Go plugins from source and its arm64 - # half runs under QEMU emulation. + # Go tools cross-compile natively. Target runtime package installation + # and executable smoke checks still use QEMU for the arm64 half. timeout-minutes: 180 strategy: # One companion failing to build must not hide whether the others do. diff --git a/companions/proto/Dockerfile b/companions/proto/Dockerfile index a66bccdc..baf8f559 100644 --- a/companions/proto/Dockerfile +++ b/companions/proto/Dockerfile @@ -1,16 +1,9 @@ # Proto compilation companion for `codefly generate proto`. # -# Size strategy: -# 1. Multi-stage: build all the Go plugins in a golang:1.27-alpine -# builder stage, copy only the static binaries into the runtime. -# 2. Strip Go binaries with -ldflags="-s -w" during install — drops -# DWARF + symbol tables, typically 20-30% smaller per binary. -# 3. Clean $GOMODCACHE + $GOCACHE after the last install — those -# caches are scaffolding the runtime doesn't need. -# 4. Runtime FROM the codefly companion base instead of golang:1.26- -# alpine — saves ~170 MB of Go stdlib we don't need since the -# binaries are statically compiled. Still need apk-installed -# nodejs+npm for TypeScript generation. +# Compile pinned Go tools on the build host and cross-compile static binaries +# for the target. Target-platform emulation is only needed by runtime package +# installation, never by the Go compiler. Upstream prebuilt plugins often carry +# older Go runtimes and dependencies, so downloading them is not equivalent. # The codefly base image is a build argument: this repository declares which # companion image it builds on, and the builder resolves that to a registry. @@ -19,16 +12,18 @@ ARG CODEFLY_BASE_IMAGE=ghcr.io/codefly-dev/codefly:0.0.6 FROM ${CODEFLY_BASE_IMAGE} AS codeflybase -# --- Builder stage: compile every Go plugin from source -------------- +# The runtime needs the TARGET architecture's Go distribution for goimports +# and Swagger. This stage only supplies files; it executes no target compiler. +FROM golang:1.27-alpine AS toolchain -FROM golang:1.27-alpine AS builder +# --- Native builder: cross-compile the pinned plugins ---------------- +FROM --platform=$BUILDPLATFORM golang:1.27-alpine AS builder +ARG TARGETARCH +ENV GOOS=linux GOARCH=${TARGETARCH} CGO_ENABLED=0 +RUN apk add --no-cache git -RUN apk add --no-cache git python3 py3-pip nodejs npm protobuf - -# Install the Go-based plugins, stripped, in one RUN so the cache -# cleanup at the end actually takes effect in the final image layer. -# Each binary is copied out individually in the runtime stage; the -# module cache is not. +# Install stripped plugins in one layer. Only their binaries are copied +# into the runtime; the source and build caches remain in this stage. RUN go install -ldflags="-s -w" github.com/bufbuild/buf/cmd/buf@v1.73.0 && \ go install -ldflags="-s -w" google.golang.org/protobuf/cmd/protoc-gen-go@v1.36.11 && \ go install -ldflags="-s -w" google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.6.1 && \ @@ -39,17 +34,17 @@ RUN go install -ldflags="-s -w" github.com/bufbuild/buf/cmd/buf@v1.73.0 && \ go install -ldflags="-s -w" github.com/envoyproxy/protoc-gen-validate@v1.2.1 && \ go install -ldflags="-s -w" golang.org/x/tools/cmd/goimports@v0.50.0 +# go install puts cross-compiled commands in $GOPATH/bin/linux_. +# Normalize the output location for the runtime COPY instructions below. +RUN if [ "$TARGETARCH" != "$(go env GOHOSTARCH)" ]; then \ + mv /go/bin/linux_"$TARGETARCH"/* /go/bin/; \ + fi + # Go facade plugin (protoc-gen-codefly-facade-go), built from in-tree source. COPY companions/proto/facades/go /src/facade-go RUN cd /src/facade-go && \ go build -ldflags="-s -w" -o /go/bin/protoc-gen-codefly-facade-go . -# Python grpcio-tools for the builder's own grpc_* helper binaries, copied -# out of /usr/bin below. This venv is NOT copied into the runtime stage — the -# runtime builds its own /venv, which is where grpcio-tools has to be -# installed for `python -m grpc_tools.protoc` to exist in the image. -RUN python3 -m venv /venv && /venv/bin/pip install --no-cache-dir grpcio-tools - # --- Runtime stage: minimal alpine + the codefly base --------------- # Keep the interpreter compatible with the pinned generator's wheels and @@ -75,11 +70,11 @@ COPY --from=codeflybase /bin/codefly /bin/codefly # committed shape, so the companion's output drifts from every # consumer by exactly those lines. The apk go is whatever the base # alpine ships (1.23 on 3.21); the toolchain is COPYed from the -# builder below instead, so the runtime Go is the ecosystem's Go +# target toolchain below instead, so the runtime Go is the ecosystem's Go # by construction. The Go-built plugins (buf, protoc-gen-*) are # static and never need it. Adds ~270 MB; unavoidable. # - Go-built plugins (buf, protoc-gen-*) ARE static (CGO_ENABLED=0 -# by default for `go install`), so we COPY just those binaries +# explicitly in the builder), so we COPY just those binaries # without bringing the Go runtime. # HOME=/tmp so the companion works when run as an arbitrary non-root UID # (codefly runs it as the host user on Linux CI). buf and npm/npx write @@ -99,10 +94,10 @@ RUN apk add --no-cache bash ca-certificates curl git jq procps tzdata nodejs npm @bufbuild/protoc-gen-es@2.11.0 && \ npm cache clean --force -# The builder's toolchain, not apk's: goimports and swagger both shell out +# The target toolchain, not apk's: goimports and swagger both shell out # to `go`, and a Go older than what consumers' go.mod files require makes # goimports degrade silently (see the runtime-dependencies note above). -COPY --from=builder /usr/local/go /usr/local/go +COPY --from=toolchain /usr/local/go /usr/local/go ENV PATH=/usr/local/go/bin:$PATH COPY --from=builder /go/bin/buf /usr/local/bin/ COPY --from=builder /go/bin/protoc-gen-go /usr/local/bin/ @@ -118,7 +113,6 @@ COPY --from=builder /go/bin/swagger /usr/local/bin/ # companion formats what buf wrote before it hands the tree back. Pinned like # the plugins — a formatter that moves with @latest moves the committed shape. COPY --from=builder /go/bin/goimports /usr/local/bin/ -COPY --from=builder /usr/bin/grpc_* /usr/local/bin/ # Facade plugin sources (Python scripts + the TS npm package). COPY companions/proto/facades/python /opt/codefly-facades/python @@ -150,4 +144,13 @@ RUN printf '#!/bin/sh\nexec /venv/bin/python /opt/codefly-facades/python/protoc_ chmod +x /usr/local/bin/protoc-gen-codefly-python /usr/local/bin/protoc-gen-codefly-facade-python /usr/local/bin/codefly-proto-strip-options && \ ln -sf protoc-gen-codefly-facade-python /usr/local/bin/protoc-gen-solution_facade +# Exercise target executables in both architecture builds. A successful native +# compile alone would not catch copying the host Go distribution into runtime. +ARG TARGETARCH +RUN test "$(go env GOARCH)" = "$TARGETARCH" && \ + buf --version && \ + protoc-gen-go --version && \ + protoc-gen-go-grpc --version && \ + swagger version + WORKDIR /workspace diff --git a/companions/proto/README.md b/companions/proto/README.md index c50ec029..09c83154 100644 --- a/companions/proto/README.md +++ b/companions/proto/README.md @@ -57,8 +57,20 @@ the existing `tag.sh` version-bump flow continues to work. codefly companion build proto ``` -The Dockerfile assembles the same set via apk + `go install`. Less -reproducible than Nix (apk packages vary across Alpine releases) but +The Dockerfile assembles the same set via apk + `go install`. The Go builder +runs on `BUILDPLATFORM` and cross-compiles static plugins for `TARGETARCH`; +it never compiles under ARM emulation. A separate target-platform Go image +supplies the runtime `go`/`gofmt`, so cross-building cannot accidentally ship +the build host's compiler. Each image build executes Buf, the Go generators, +Swagger and the runtime Go architecture check on its target platform. +Upstream downloads were checked: Buf 1.73.0's +prebuilt binary carries Go 1.26.8 and x/net 0.58.0, including fixed findings, +and several other plugin downloads carry still older Go toolchains. Retaining +source builds preserves the current Go compiler instead of importing those +additional findings. Plugin dependency versions are unchanged by this build +speed fix; it does not claim a clean vulnerability scan of every plugin. + +This path is less reproducible than Nix (apk packages vary across Alpine releases) but needs no Linux builder VM, which is why the publish workflow uses it. ## What's in the image @@ -98,9 +110,9 @@ may also contain handwritten Go, so only files carrying Go's standard ## Versioning `info.codefly.yaml` carries the image version. Version 0.0.21 removes Rust -generation and its build dependencies. Publish through `codefly companion -publish` using the [publication procedure](../../docs/runbooks/publish-companions.md); -changing the manifest alone does not publish an image. Core 0.19.13 selects +generation and its build dependencies; 0.0.22 removes emulated Go compilation. +Publish through `codefly companion publish` using the [publication procedure](../../docs/runbooks/publish-companions.md); +changing the manifest alone does not publish an image. Core 0.19.14 selects this companion. ## Python generation diff --git a/companions/proto/companion_plugins_test.go b/companions/proto/companion_plugins_test.go index 02170157..2576d5e3 100644 --- a/companions/proto/companion_plugins_test.go +++ b/companions/proto/companion_plugins_test.go @@ -79,9 +79,9 @@ func TestDockerfileShipsGoimportsInRuntime(t *testing.T) { // go.mod requirement makes it refuse — after which goimports quietly formats // syntax only and emits no alias where a package's name is not its path's // last element. That is the committed shape, so every consumer drifts by -// exactly those lines, and nothing errors. The builder stage's Go is the -// ecosystem's; the runtime must carry that one, not whatever apk ships. -func TestDockerfileRuntimeGoIsTheBuilderToolchain(t *testing.T) { +// exactly those lines, and nothing errors. The target toolchain stage supplies +// the ecosystem's Go for the runtime architecture, not the native builder's. +func TestDockerfileRuntimeGoIsTheTargetToolchain(t *testing.T) { dockerfile, err := os.ReadFile("Dockerfile") if err != nil { t.Fatalf("read Dockerfile: %v", err) @@ -94,8 +94,8 @@ func TestDockerfileRuntimeGoIsTheBuilderToolchain(t *testing.T) { if regexp.MustCompile(`(?m)^RUN apk add[^\n]*\bgo\b`).MatchString(runtimeStage) { t.Error("Dockerfile installs go from apk in the runtime stage; that is the base alpine's Go, older than consumers' go.mod require, and goimports then degrades silently (see the runtime-dependencies note)") } - if !regexp.MustCompile(`(?m)^COPY --from=builder /usr/local/go /usr/local/go`).MatchString(runtimeStage) { - t.Error("Dockerfile does not COPY the builder's Go toolchain into the runtime stage") + if !regexp.MustCompile(`(?m)^COPY --from=toolchain /usr/local/go /usr/local/go`).MatchString(runtimeStage) { + t.Error("Dockerfile does not COPY the target platform's Go toolchain into the runtime stage") } if !regexp.MustCompile(`(?m)^FROM golang:1\.27`).MatchString(string(dockerfile)) { t.Error("the builder stage is not golang:1.27; the runtime Go it hands the formatter would not satisfy modules that require go 1.27") @@ -313,7 +313,7 @@ func TestTsFacadeLockfileMatchesTheRuntime(t *testing.T) { // neighbour — the reader then checks the wrong line, finds something plausible, // and moves on. var runbookCitations = map[string]string{ - "Dockerfile:99": "@bufbuild/protoc-gen-es@", + "Dockerfile:94": "@bufbuild/protoc-gen-es@", "flake.nix:51": "protocGenEsVersion =", "flake.nix:59": "hash =", "flake.nix:61": "npmDepsHash =", diff --git a/companions/proto/info.codefly.yaml b/companions/proto/info.codefly.yaml index fb15e716..0a9d1e31 100644 --- a/companions/proto/info.codefly.yaml +++ b/companions/proto/info.codefly.yaml @@ -1 +1 @@ -version: 0.0.21 +version: 0.0.22 diff --git a/docs/runbooks/bump-protobuf-es.md b/docs/runbooks/bump-protobuf-es.md index 64ee3e3d..7273b091 100644 --- a/docs/runbooks/bump-protobuf-es.md +++ b/docs/runbooks/bump-protobuf-es.md @@ -103,7 +103,7 @@ Six places, and they must agree: |---|---|---| | `companions/proto/facades/ts/package.json` | `@bufbuild/protoplugin` | yes | | `companions/proto/facades/ts/package-lock.json` | the resolved `protoplugin` + `protobuf` closure | yes | -| `companions/proto/Dockerfile:99` | `@bufbuild/protoc-gen-es@` installed via npm | yes | +| `companions/proto/Dockerfile:94` | `@bufbuild/protoc-gen-es@` installed via npm | yes | | `companions/proto/flake.nix:51` | `protocGenEsVersion`, built from source | yes | | `companions/proto/flake.nix:59` | `hash` — the protobuf-es source tarball | **no** | | `companions/proto/flake.nix:61` | `npmDepsHash` — its npm dependency closure | **no** | diff --git a/version/info.codefly.yaml b/version/info.codefly.yaml index ea8113a3..309a17ff 100644 --- a/version/info.codefly.yaml +++ b/version/info.codefly.yaml @@ -1 +1 @@ -version: 0.19.13 +version: 0.19.14