Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
14 commits
Select commit Hold shift + click to select a range
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions .github/workflows/companions-build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -48,8 +48,8 @@ jobs:
build:
needs: plan
runs-on: ubuntu-latest
# The proto companion compiles ~10 Go plugins from source and its arm64
# half runs under QEMU emulation.
# Go tools cross-compile natively. Target runtime package installation
# and executable smoke checks still use QEMU for the arm64 half.
timeout-minutes: 180
strategy:
# One companion failing to build must not hide whether the others do.
Expand Down
63 changes: 33 additions & 30 deletions companions/proto/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,16 +1,9 @@
# Proto compilation companion for `codefly generate proto`.
#
# Size strategy:
# 1. Multi-stage: build all the Go plugins in a golang:1.27-alpine
# builder stage, copy only the static binaries into the runtime.
# 2. Strip Go binaries with -ldflags="-s -w" during install — drops
# DWARF + symbol tables, typically 20-30% smaller per binary.
# 3. Clean $GOMODCACHE + $GOCACHE after the last install — those
# caches are scaffolding the runtime doesn't need.
# 4. Runtime FROM the codefly companion base instead of golang:1.26-
# alpine — saves ~170 MB of Go stdlib we don't need since the
# binaries are statically compiled. Still need apk-installed
# nodejs+npm for TypeScript generation.
# Compile pinned Go tools on the build host and cross-compile static binaries
# for the target. Target-platform emulation is only needed by runtime package
# installation, never by the Go compiler. Upstream prebuilt plugins often carry
# older Go runtimes and dependencies, so downloading them is not equivalent.

# The codefly base image is a build argument: this repository declares which
# companion image it builds on, and the builder resolves that to a registry.
Expand All @@ -19,16 +12,18 @@
ARG CODEFLY_BASE_IMAGE=ghcr.io/codefly-dev/codefly:0.0.6
FROM ${CODEFLY_BASE_IMAGE} AS codeflybase

# --- Builder stage: compile every Go plugin from source --------------
# The runtime needs the TARGET architecture's Go distribution for goimports
# and Swagger. This stage only supplies files; it executes no target compiler.
FROM golang:1.27-alpine AS toolchain

FROM golang:1.27-alpine AS builder
# --- Native builder: cross-compile the pinned plugins ----------------
FROM --platform=$BUILDPLATFORM golang:1.27-alpine AS builder
ARG TARGETARCH
ENV GOOS=linux GOARCH=${TARGETARCH} CGO_ENABLED=0
RUN apk add --no-cache git

RUN apk add --no-cache git python3 py3-pip nodejs npm protobuf

# Install the Go-based plugins, stripped, in one RUN so the cache
# cleanup at the end actually takes effect in the final image layer.
# Each binary is copied out individually in the runtime stage; the
# module cache is not.
# Install stripped plugins in one layer. Only their binaries are copied
# into the runtime; the source and build caches remain in this stage.
RUN go install -ldflags="-s -w" github.com/bufbuild/buf/cmd/buf@v1.73.0 && \
go install -ldflags="-s -w" google.golang.org/protobuf/cmd/protoc-gen-go@v1.36.11 && \
go install -ldflags="-s -w" google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.6.1 && \
Expand All @@ -39,17 +34,17 @@ RUN go install -ldflags="-s -w" github.com/bufbuild/buf/cmd/buf@v1.73.0 && \
go install -ldflags="-s -w" github.com/envoyproxy/protoc-gen-validate@v1.2.1 && \
go install -ldflags="-s -w" golang.org/x/tools/cmd/goimports@v0.50.0

# go install puts cross-compiled commands in $GOPATH/bin/linux_<arch>.
# Normalize the output location for the runtime COPY instructions below.
RUN if [ "$TARGETARCH" != "$(go env GOHOSTARCH)" ]; then \
mv /go/bin/linux_"$TARGETARCH"/* /go/bin/; \
fi

# Go facade plugin (protoc-gen-codefly-facade-go), built from in-tree source.
COPY companions/proto/facades/go /src/facade-go
RUN cd /src/facade-go && \
go build -ldflags="-s -w" -o /go/bin/protoc-gen-codefly-facade-go .

# Python grpcio-tools for the builder's own grpc_* helper binaries, copied
# out of /usr/bin below. This venv is NOT copied into the runtime stage — the
# runtime builds its own /venv, which is where grpcio-tools has to be
# installed for `python -m grpc_tools.protoc` to exist in the image.
RUN python3 -m venv /venv && /venv/bin/pip install --no-cache-dir grpcio-tools

# --- Runtime stage: minimal alpine + the codefly base ---------------

# Keep the interpreter compatible with the pinned generator's wheels and
Expand All @@ -75,11 +70,11 @@ COPY --from=codeflybase /bin/codefly /bin/codefly
# committed shape, so the companion's output drifts from every
# consumer by exactly those lines. The apk go is whatever the base
# alpine ships (1.23 on 3.21); the toolchain is COPYed from the
# builder below instead, so the runtime Go is the ecosystem's Go
# target toolchain below instead, so the runtime Go is the ecosystem's Go
# by construction. The Go-built plugins (buf, protoc-gen-*) are
# static and never need it. Adds ~270 MB; unavoidable.
# - Go-built plugins (buf, protoc-gen-*) ARE static (CGO_ENABLED=0
# by default for `go install`), so we COPY just those binaries
# explicitly in the builder), so we COPY just those binaries
# without bringing the Go runtime.
# HOME=/tmp so the companion works when run as an arbitrary non-root UID
# (codefly runs it as the host user on Linux CI). buf and npm/npx write
Expand All @@ -99,10 +94,10 @@ RUN apk add --no-cache bash ca-certificates curl git jq procps tzdata nodejs npm
@bufbuild/protoc-gen-es@2.11.0 && \
npm cache clean --force

# The builder's toolchain, not apk's: goimports and swagger both shell out
# The target toolchain, not apk's: goimports and swagger both shell out
# to `go`, and a Go older than what consumers' go.mod files require makes
# goimports degrade silently (see the runtime-dependencies note above).
COPY --from=builder /usr/local/go /usr/local/go
COPY --from=toolchain /usr/local/go /usr/local/go
ENV PATH=/usr/local/go/bin:$PATH
COPY --from=builder /go/bin/buf /usr/local/bin/
COPY --from=builder /go/bin/protoc-gen-go /usr/local/bin/
Expand All @@ -118,7 +113,6 @@ COPY --from=builder /go/bin/swagger /usr/local/bin/
# companion formats what buf wrote before it hands the tree back. Pinned like
# the plugins — a formatter that moves with @latest moves the committed shape.
COPY --from=builder /go/bin/goimports /usr/local/bin/
COPY --from=builder /usr/bin/grpc_* /usr/local/bin/

# Facade plugin sources (Python scripts + the TS npm package).
COPY companions/proto/facades/python /opt/codefly-facades/python
Expand Down Expand Up @@ -150,4 +144,13 @@ RUN printf '#!/bin/sh\nexec /venv/bin/python /opt/codefly-facades/python/protoc_
chmod +x /usr/local/bin/protoc-gen-codefly-python /usr/local/bin/protoc-gen-codefly-facade-python /usr/local/bin/codefly-proto-strip-options && \
ln -sf protoc-gen-codefly-facade-python /usr/local/bin/protoc-gen-solution_facade

# Exercise target executables in both architecture builds. A successful native
# compile alone would not catch copying the host Go distribution into runtime.
ARG TARGETARCH
RUN test "$(go env GOARCH)" = "$TARGETARCH" && \
buf --version && \
protoc-gen-go --version && \
protoc-gen-go-grpc --version && \
swagger version

WORKDIR /workspace
22 changes: 17 additions & 5 deletions companions/proto/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -57,8 +57,20 @@ the existing `tag.sh` version-bump flow continues to work.
codefly companion build proto
```

The Dockerfile assembles the same set via apk + `go install`. Less
reproducible than Nix (apk packages vary across Alpine releases) but
The Dockerfile assembles the same set via apk + `go install`. The Go builder
runs on `BUILDPLATFORM` and cross-compiles static plugins for `TARGETARCH`;
it never compiles under ARM emulation. A separate target-platform Go image
supplies the runtime `go`/`gofmt`, so cross-building cannot accidentally ship
the build host's compiler. Each image build executes Buf, the Go generators,
Swagger and the runtime Go architecture check on its target platform.
Upstream downloads were checked: Buf 1.73.0's
prebuilt binary carries Go 1.26.8 and x/net 0.58.0, including fixed findings,
and several other plugin downloads carry still older Go toolchains. Retaining
source builds preserves the current Go compiler instead of importing those
additional findings. Plugin dependency versions are unchanged by this build
speed fix; it does not claim a clean vulnerability scan of every plugin.

This path is less reproducible than Nix (apk packages vary across Alpine releases) but
needs no Linux builder VM, which is why the publish workflow uses it.

## What's in the image
Expand Down Expand Up @@ -98,9 +110,9 @@ may also contain handwritten Go, so only files carrying Go's standard
## Versioning

`info.codefly.yaml` carries the image version. Version 0.0.21 removes Rust
generation and its build dependencies. Publish through `codefly companion
publish` using the [publication procedure](../../docs/runbooks/publish-companions.md);
changing the manifest alone does not publish an image. Core 0.19.13 selects
generation and its build dependencies; 0.0.22 removes emulated Go compilation.
Publish through `codefly companion publish` using the [publication procedure](../../docs/runbooks/publish-companions.md);
changing the manifest alone does not publish an image. Core 0.19.14 selects
this companion.

## Python generation
Expand Down
12 changes: 6 additions & 6 deletions companions/proto/companion_plugins_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -79,9 +79,9 @@ func TestDockerfileShipsGoimportsInRuntime(t *testing.T) {
// go.mod requirement makes it refuse — after which goimports quietly formats
// syntax only and emits no alias where a package's name is not its path's
// last element. That is the committed shape, so every consumer drifts by
// exactly those lines, and nothing errors. The builder stage's Go is the
// ecosystem's; the runtime must carry that one, not whatever apk ships.
func TestDockerfileRuntimeGoIsTheBuilderToolchain(t *testing.T) {
// exactly those lines, and nothing errors. The target toolchain stage supplies
// the ecosystem's Go for the runtime architecture, not the native builder's.
func TestDockerfileRuntimeGoIsTheTargetToolchain(t *testing.T) {
dockerfile, err := os.ReadFile("Dockerfile")
if err != nil {
t.Fatalf("read Dockerfile: %v", err)
Expand All @@ -94,8 +94,8 @@ func TestDockerfileRuntimeGoIsTheBuilderToolchain(t *testing.T) {
if regexp.MustCompile(`(?m)^RUN apk add[^\n]*\bgo\b`).MatchString(runtimeStage) {
t.Error("Dockerfile installs go from apk in the runtime stage; that is the base alpine's Go, older than consumers' go.mod require, and goimports then degrades silently (see the runtime-dependencies note)")
}
if !regexp.MustCompile(`(?m)^COPY --from=builder /usr/local/go /usr/local/go`).MatchString(runtimeStage) {
t.Error("Dockerfile does not COPY the builder's Go toolchain into the runtime stage")
if !regexp.MustCompile(`(?m)^COPY --from=toolchain /usr/local/go /usr/local/go`).MatchString(runtimeStage) {
t.Error("Dockerfile does not COPY the target platform's Go toolchain into the runtime stage")
}
if !regexp.MustCompile(`(?m)^FROM golang:1\.27`).MatchString(string(dockerfile)) {
t.Error("the builder stage is not golang:1.27; the runtime Go it hands the formatter would not satisfy modules that require go 1.27")
Expand Down Expand Up @@ -313,7 +313,7 @@ func TestTsFacadeLockfileMatchesTheRuntime(t *testing.T) {
// neighbour — the reader then checks the wrong line, finds something plausible,
// and moves on.
var runbookCitations = map[string]string{
"Dockerfile:99": "@bufbuild/protoc-gen-es@",
"Dockerfile:94": "@bufbuild/protoc-gen-es@",
"flake.nix:51": "protocGenEsVersion =",
"flake.nix:59": "hash =",
"flake.nix:61": "npmDepsHash =",
Expand Down
2 changes: 1 addition & 1 deletion companions/proto/info.codefly.yaml
Original file line number Diff line number Diff line change
@@ -1 +1 @@
version: 0.0.21
version: 0.0.22
2 changes: 1 addition & 1 deletion docs/runbooks/bump-protobuf-es.md
Original file line number Diff line number Diff line change
Expand Up @@ -103,7 +103,7 @@ Six places, and they must agree:
|---|---|---|
| `companions/proto/facades/ts/package.json` | `@bufbuild/protoplugin` | yes |
| `companions/proto/facades/ts/package-lock.json` | the resolved `protoplugin` + `protobuf` closure | yes |
| `companions/proto/Dockerfile:99` | `@bufbuild/protoc-gen-es@` installed via npm | yes |
| `companions/proto/Dockerfile:94` | `@bufbuild/protoc-gen-es@` installed via npm | yes |
| `companions/proto/flake.nix:51` | `protocGenEsVersion`, built from source | yes |
| `companions/proto/flake.nix:59` | `hash` — the protobuf-es source tarball | **no** |
| `companions/proto/flake.nix:61` | `npmDepsHash` — its npm dependency closure | **no** |
Expand Down
2 changes: 1 addition & 1 deletion version/info.codefly.yaml
Original file line number Diff line number Diff line change
@@ -1 +1 @@
version: 0.19.13
version: 0.19.14
Loading