diff --git a/companions/imagepolicy_test.go b/companions/imagepolicy_test.go index 8b26d8517..d1b7b0cd4 100644 --- a/companions/imagepolicy_test.go +++ b/companions/imagepolicy_test.go @@ -34,7 +34,6 @@ var unpinnedUpstreamBases = []string{ "docker:29-dind", "golang:1.27-alpine", "node:26.3.0-alpine3.22", - "rust:1.99-alpine", } // corePolicy is what core requires of its own images. It is an ordinary diff --git a/companions/proto/Dockerfile b/companions/proto/Dockerfile index 3ab37c516..a66bccdce 100644 --- a/companions/proto/Dockerfile +++ b/companions/proto/Dockerfile @@ -52,11 +52,6 @@ RUN python3 -m venv /venv && /venv/bin/pip install --no-cache-dir grpcio-tools # --- Runtime stage: minimal alpine + the codefly base --------------- -FROM rust:1.99-alpine AS rust-builder -RUN apk add --no-cache musl-dev && \ - cargo install --locked protoc-gen-prost --version 0.5.0 && \ - cargo install --locked protoc-gen-tonic --version 0.5.0 - # Keep the interpreter compatible with the pinned generator's wheels and # gencode contract. Alpine's python3 moves independently; grpcio-tools 1.71 # has Python 3.13 wheels, not 3.14 wheels. As in the Python companion, copy @@ -124,8 +119,6 @@ COPY --from=builder /go/bin/swagger /usr/local/bin/ # the plugins — a formatter that moves with @latest moves the committed shape. COPY --from=builder /go/bin/goimports /usr/local/bin/ COPY --from=builder /usr/bin/grpc_* /usr/local/bin/ -COPY --from=rust-builder /usr/local/cargo/bin/protoc-gen-prost /usr/local/bin/ -COPY --from=rust-builder /usr/local/cargo/bin/protoc-gen-tonic /usr/local/bin/ # Facade plugin sources (Python scripts + the TS npm package). COPY companions/proto/facades/python /opt/codefly-facades/python diff --git a/companions/proto/README.md b/companions/proto/README.md index 94858a07f..c50ec029e 100644 --- a/companions/proto/README.md +++ b/companions/proto/README.md @@ -2,8 +2,13 @@ The codefly proto companion bundles `buf`, `protoc`, `protoc-gen-*`, `swagger`, `grpcio-tools`, and the JS/TS generators into one OCI -image. Used by `codefly generate proto` and the -`codefly generate openAPI` pipeline. +image. All proto generation runs through Codefly (`codefly generate proto` +and `codefly generate client`), which owns the pinned companion invocation. +Supported client languages are Go, Python, and TypeScript. The image and its +build contain no Rust toolchain or Rust generation plugins; unsupported +language requests fail before writing output. +The generation runner keeps the companion alive while executing its tools, +including Swagger client generation; it does not rely on the base image's default command. ## Build @@ -92,32 +97,16 @@ may also contain handwritten Go, so only files carrying Go's standard ## Versioning -`info.codefly.yaml` carries the image version (`0.0.10` at time of -writing). Bump via `scripts/tag.sh` — same convention as every other -companion. Both the Dockerfile build and the Nix build read this -value so the tag stays consistent. -## Rust client imports +`info.codefly.yaml` carries the image version. Version 0.0.21 removes Rust +generation and its build dependencies. Publish through `codefly companion +publish` using the [publication procedure](../../docs/runbooks/publish-companions.md); +changing the manifest alone does not publish an image. Core 0.19.13 selects +this companion. -The proto companion supplies local `protoc-gen-prost` and `protoc-gen-tonic` -0.5.0, targeting prost/tonic 0.14. Rust clients retain googleapis and -protovalidate message bindings beside the application's bindings. Standard -`google.protobuf` types continue to resolve through `prost-types`. - -Client generation validates descriptor-set type references before invoking the -plugins. Both source and descriptor-set generation are covered by a Cargo -compile and encode/decode regression containing `google.rpc.Status` and -`buf.validate.Violations`: - -```sh -go test ./companions/proto -tags=proto_companion_required -run TestRustClientRetainsImportedMessageFields -timeout=180s -``` - -This requires the built proto companion and Cargo. The updated plugins ship in -companion version 0.0.16; bumping the manifest alone does not publish that image. -Use the [companion publication procedure](../../docs/runbooks/publish-companions.md). +## Python generation Python bindings come from the companion's own plugin, `protoc-gen-codefly-python` -(`facades/python/protoc_gen_codefly_python.py`), as Go's and Rust's come from +(`facades/python/protoc_gen_codefly_python.py`), as Go's come from local plugins: it compiles the `*_pb2` messages and `*_pb2_grpc` stubs with the `grpc_tools` pinned in the image's venv (`grpcio-tools==1.71.0`, `protobuf==5.29.3`), so the gencode is versioned with the image. The Docker @@ -139,11 +128,3 @@ A change to any of a companion's image inputs (its Dockerfile and every build-context `COPY` source, `BuildSpec.Inputs`) must bump its `info.codefly.yaml`: `internal/companionbump`, the required `Companion version` check on every pull request, refuses one that does not. - -Rust generation stages output before publishing it. `.codefly-rust-output.json` -records the generated files and their hashes; only unchanged, tracked files may -be replaced or removed. Unrelated files are preserved. A collision with an -untracked file or an edited binding is an error, including when adopting output -created before ownership tracking. Generate into an empty output directory in -that case; do not discard local edits. Failed generation leaves existing output -untouched, and publication errors roll back changed files. diff --git a/companions/proto/client.go b/companions/proto/client.go index 829e64bb9..fa92f86f0 100644 --- a/companions/proto/client.go +++ b/companions/proto/client.go @@ -113,6 +113,12 @@ type clientSpec struct { func generateClient(ctx context.Context, spec clientSpec) error { w := wool.Get(ctx).In("generateClient", wool.DirField(spec.destination)) + switch spec.language { + case languages.GO, languages.PYTHON, languages.TYPESCRIPT: + default: + return w.NewError("unsupported proto generation language %q; supported languages: go, python, typescript", spec.language) + } + image, err := CompanionImage(ctx) if err != nil { return w.Wrapf(err, "cannot get companion image") @@ -152,9 +158,7 @@ func generateClient(ctx context.Context, spec clientSpec) error { // Sources only: a descriptor set carries its foreign files already marked as // buf image imports, and re-targeting them would put a second Timestamp next // to the consumer's runtime one. - // Rust also needs imported bindings; prost keeps google.protobuf external - // through its default prost-types mapping. - includeImports := len(spec.sources) > 0 && (spec.language == languages.TYPESCRIPT || spec.language == languages.RUST) + includeImports := len(spec.sources) > 0 && spec.language == languages.TYPESCRIPT if err = CreateBufConfiguration(ctx, tmpDir, spec.service, spec.language, spec.facade, WithGoPackageOverrides(goPackageOverrides), WithIncludeImports(includeImports)); err != nil { @@ -210,12 +214,6 @@ func generateClient(ctx context.Context, spec clientSpec) error { if _, err = shared.CheckDirectoryOrCreate(ctx, spec.destination); err != nil { return w.Wrapf(err, "cannot create destination") } - if spec.language == languages.RUST { - return generateRustOutput(ctx, spec.destination, func(output string) error { - name := fmt.Sprintf("proto-%s-%d-%s", spec.service, time.Now().UnixMilli(), spec.language) - return runBuf(ctx, name, image, tmpDir, output, false, depUpdate, generateArgs, before) - }) - } if includeImports { // The run writes a tree for every namespace the sources import, which is diff --git a/companions/proto/client_internal_test.go b/companions/proto/client_internal_test.go index 37a66c16a..2e9f08c85 100644 --- a/companions/proto/client_internal_test.go +++ b/companions/proto/client_internal_test.go @@ -2,154 +2,14 @@ package proto import ( "context" - "fmt" "os" "path/filepath" "testing" - "time" "github.com/codefly-dev/core/languages" - "github.com/gofrs/flock" "github.com/stretchr/testify/require" ) -func writeRustOutput(t *testing.T, dir, name, content string) { - t.Helper() - file := filepath.Join(dir, name) - require.NoError(t, os.MkdirAll(filepath.Dir(file), 0o750)) - require.NoError(t, os.WriteFile(file, []byte(content), 0o600)) -} - -func TestCanceledRustGenerationPreservesDestination(t *testing.T) { - dest := t.TempDir() - for _, name := range []string{"api/api.rs", "src/user.rs", "google/rpc/google.rpc.rs"} { - writeRustOutput(t, dest, name, "existing content") - } - ctx, cancel := context.WithCancel(t.Context()) - cancel() - err := GenerateClient(ctx, ClientRequest{Language: languages.RUST, Destination: dest, - Sources: []Source{{Path: "api.proto", Content: []byte(`syntax = "proto3"; package api; message Response { string value = 1; }`)}}}) - require.Error(t, err) - for _, name := range []string{"api/api.rs", "src/user.rs", "google/rpc/google.rpc.rs"} { - data, err := os.ReadFile(filepath.Join(dest, name)) - require.NoError(t, err) - require.Equal(t, "existing content", string(data)) - } -} - -func TestRustPublicationOwnsFilesNotDirectories(t *testing.T) { - dest := t.TempDir() - writeRustOutput(t, dest, "src/user.rs", "user content") - first := t.TempDir() - writeRustOutput(t, first, "api/api.rs", "old generated") - writeRustOutput(t, first, "google/rpc/google.rpc.rs", "old import") - require.NoError(t, publishRustOutput(dest, first)) - writeRustOutput(t, dest, "google/rpc/user.rs", "user content") - second := t.TempDir() - writeRustOutput(t, second, "api/api.rs", "new generated") - require.NoError(t, publishRustOutput(dest, second)) - require.NoFileExists(t, filepath.Join(dest, "google/rpc/google.rpc.rs")) - for _, name := range []string{"src/user.rs", "google/rpc/user.rs"} { - data, err := os.ReadFile(filepath.Join(dest, name)) - require.NoError(t, err) - require.Equal(t, "user content", string(data)) - } - data, err := os.ReadFile(filepath.Join(dest, "api/api.rs")) - require.NoError(t, err) - require.Equal(t, "new generated", string(data)) -} - -func TestRustPublicationRejectsEditsAndUnownedCollisions(t *testing.T) { - for _, owned := range []bool{false, true} { - t.Run(fmt.Sprint(owned), func(t *testing.T) { - dest := t.TempDir() - if owned { - first := t.TempDir() - writeRustOutput(t, first, "api/api.rs", "generated") - require.NoError(t, publishRustOutput(dest, first)) - } - writeRustOutput(t, dest, "api/api.rs", "user edit") - stage := t.TempDir() - writeRustOutput(t, stage, "api/api.rs", "replacement") - writeRustOutput(t, stage, "aaa/new.rs", "new output") - require.ErrorContains(t, publishRustOutput(dest, stage), "unowned or edited") - data, err := os.ReadFile(filepath.Join(dest, "api/api.rs")) - require.NoError(t, err) - require.Equal(t, "user edit", string(data)) - require.NoFileExists(t, filepath.Join(dest, "aaa/new.rs")) - }) - } -} - -func TestRustPublicationDoesNotPruneEditedImports(t *testing.T) { - dest, first := t.TempDir(), t.TempDir() - writeRustOutput(t, first, "google/rpc/google.rpc.rs", "generated import") - require.NoError(t, publishRustOutput(dest, first)) - writeRustOutput(t, dest, "google/rpc/google.rpc.rs", "edited import") - require.ErrorContains(t, publishRustOutput(dest, t.TempDir()), "unowned or edited") - data, err := os.ReadFile(filepath.Join(dest, "google/rpc/google.rpc.rs")) - require.NoError(t, err) - require.Equal(t, "edited import", string(data)) -} - -func TestRustPublicationRollsBackPartialFailure(t *testing.T) { - dest := t.TempDir() - first := t.TempDir() - writeRustOutput(t, first, "aaa/old.rs", "old output") - require.NoError(t, publishRustOutput(dest, first)) - manifest, err := os.ReadFile(filepath.Join(dest, rustOutputManifest)) - require.NoError(t, err) - require.NoError(t, os.Symlink("missing-target", filepath.Join(dest, "zzz"))) - stage := t.TempDir() - writeRustOutput(t, stage, "aaa/old.rs", "replacement") - writeRustOutput(t, stage, "zzz/new.rs", "cannot publish") - require.Error(t, publishRustOutput(dest, stage)) - data, err := os.ReadFile(filepath.Join(dest, "aaa/old.rs")) - require.NoError(t, err) - require.Equal(t, "old output", string(data)) - data, err = os.ReadFile(filepath.Join(dest, rustOutputManifest)) - require.NoError(t, err) - require.Equal(t, manifest, data) - require.NoFileExists(t, filepath.Join(dest, "zzz/new.rs")) -} - -func TestRustPublicationRejectsEscapingSymlink(t *testing.T) { - dest, stage, outside := t.TempDir(), t.TempDir(), t.TempDir() - require.NoError(t, os.Symlink(outside, filepath.Join(dest, "api"))) - writeRustOutput(t, stage, "api/api.rs", "generated") - require.Error(t, publishRustOutput(dest, stage)) - require.NoFileExists(t, filepath.Join(outside, "api.rs")) -} - -func TestRustGenerationHonorsDestinationLockThroughSymlink(t *testing.T) { - dest, err := filepath.EvalSymlinks(t.TempDir()) - require.NoError(t, err) - writeRustOutput(t, dest, "api/api.rs", "existing output") - alias := filepath.Join(t.TempDir(), "alias") - require.NoError(t, os.Symlink(dest, alias)) - lock := flock.New(filepath.Join(dest, rustOutputLock)) - require.NoError(t, lock.Lock()) - t.Cleanup(func() { require.NoError(t, lock.Close()) }) - ctx, cancel := context.WithTimeout(t.Context(), 100*time.Millisecond) - defer cancel() - err = GenerateClient(ctx, ClientRequest{Language: languages.RUST, Destination: alias, - Sources: []Source{{Path: "api.proto", Content: []byte(`syntax = "proto3"; package api; message Response {}`)}}}) - require.ErrorIs(t, err, context.DeadlineExceeded) - data, err := os.ReadFile(filepath.Join(dest, "api/api.rs")) - require.NoError(t, err) - require.Equal(t, "existing output", string(data)) -} - -func TestRustPublicationRejectsInvalidOwnership(t *testing.T) { - for _, body := range []string{`{`, `{"../outside":"hash"}`, `{"/outside":"hash"}`, `{".codefly-rust-output.json":"hash"}`, `{".codefly-rust-output.lock":"hash"}`} { - dest, stage := t.TempDir(), t.TempDir() - writeRustOutput(t, dest, rustOutputManifest, body) - writeRustOutput(t, stage, "api/api.rs", "generated") - require.Error(t, publishRustOutput(dest, stage)) - require.NoFileExists(t, filepath.Join(dest, "api/api.rs")) - } -} - // TestRemoveUnownedOutput pins what a TypeScript run is allowed to keep in its // destination. Asking buf for the imports means the run writes a tree per // imported namespace, which is not a set known ahead of time — so the @@ -213,3 +73,19 @@ func TestRemoveUnownedOutputKeepsRootLevelOutput(t *testing.T) { _, err := os.Stat(filepath.Join(dest, "google")) require.True(t, os.IsNotExist(err)) } + +func TestUnsupportedClientLanguagePreservesDestination(t *testing.T) { + dest := t.TempDir() + file := filepath.Join(dest, "existing.txt") + require.NoError(t, os.WriteFile(file, []byte("keep"), 0600)) + err := GenerateClient(t.Context(), ClientRequest{Language: languages.RUST, Destination: dest, + Sources: []Source{{Path: "api.proto", Content: []byte(`syntax = "proto3"; package api; message Request {}`)}}}) + require.ErrorContains(t, err, "unsupported proto generation language") + entries, err := os.ReadDir(dest) + require.NoError(t, err) + require.Len(t, entries, 1) + data, err := os.ReadFile(file) + require.NoError(t, err) + require.Equal(t, "keep", string(data)) + require.ErrorContains(t, CreateBufConfiguration(t.Context(), dest, "api", languages.RUST, FacadeOptions{}), "unsupported proto generation language") +} diff --git a/companions/proto/companion_plugins_test.go b/companions/proto/companion_plugins_test.go index c39abc045..02170157d 100644 --- a/companions/proto/companion_plugins_test.go +++ b/companions/proto/companion_plugins_test.go @@ -21,19 +21,6 @@ import ( // content hashes — fail the Nix build rather than any test here. const protocGenEsRuntimeVersion = "2.11.0" -func TestCompanionInstallsLocalRustPlugins(t *testing.T) { - for _, file := range []string{"Dockerfile", "flake.nix", "templates/rust/buf.gen.yaml.tmpl"} { - assertMentionsAll(t, file, []string{"protoc-gen-prost", "protoc-gen-tonic"}) - } - content, err := os.ReadFile("templates/rust/buf.gen.yaml.tmpl") - if err != nil { - t.Fatal(err) - } - if strings.Contains(string(content), "buf.build/") { - t.Fatal("Rust generation must use the companion's local plugins") - } -} - // bumpRunbook records why the pin sits where it does and what moving it costs. // The guards below keep it honest, because a stale runbook is worse than none: // it still reads as current. @@ -326,10 +313,10 @@ func TestTsFacadeLockfileMatchesTheRuntime(t *testing.T) { // neighbour — the reader then checks the wrong line, finds something plausible, // and moves on. var runbookCitations = map[string]string{ - "Dockerfile:104": "@bufbuild/protoc-gen-es@", - "flake.nix:51": "protocGenEsVersion =", - "flake.nix:59": "hash =", - "flake.nix:61": "npmDepsHash =", + "Dockerfile:99": "@bufbuild/protoc-gen-es@", + "flake.nix:51": "protocGenEsVersion =", + "flake.nix:59": "hash =", + "flake.nix:61": "npmDepsHash =", } func TestRunbookCitationsPointAtWhatTheyName(t *testing.T) { @@ -402,7 +389,7 @@ func TestTypeScriptTemplateUsesTheBakedProtocGenEs(t *testing.T) { } } -// Python's bindings come from the companion's own plugin, as Go's and Rust's do: +// Python's bindings come from the companion's own plugin, as Go's do: // buf's remote python plugins moved on their own, and their gencode outran the // protobuf runtime a generated library declares. Both build definitions ship // the plugin, and the image pins the grpc_tools it runs. @@ -419,3 +406,17 @@ func TestCompanionGeneratesPythonWithItsLocalPlugin(t *testing.T) { } assertMentionsAll(t, "Dockerfile", []string{"protobuf==5.29.3", "grpcio-tools==1.71.0"}) } + +func TestCompanionBuildsWithoutRust(t *testing.T) { + for _, file := range []string{"Dockerfile", "flake.nix"} { + data, err := os.ReadFile(file) + if err != nil { + t.Fatal(err) + } + for _, tool := range []string{"rust:", "rust-builder", "cargo", "protoc-gen-prost", "protoc-gen-tonic"} { + if strings.Contains(string(data), tool) { + t.Errorf("%s includes removed Rust tooling %q", file, tool) + } + } + } +} diff --git a/companions/proto/descriptorset_test.go b/companions/proto/descriptorset_test.go index 5df7a83c4..37f27a24b 100644 --- a/companions/proto/descriptorset_test.go +++ b/companions/proto/descriptorset_test.go @@ -78,87 +78,6 @@ func plainDescriptorSetFor(t *testing.T, ctx context.Context, source string) []b return image } -func TestRustClientRetainsImportedMessageFields(t *testing.T) { - ctx := t.Context() - testutil.RequireProtoImage(t, ctx) - source := strings.Replace(restProto, `import "google/protobuf/timestamp.proto";`, `import "google/protobuf/timestamp.proto"; -import "google/rpc/status.proto";`, 1) - source = strings.Replace(source, "google.protobuf.Timestamp created_at = 1;", `google.protobuf.Timestamp created_at = 1; - google.rpc.Status status = 2; - buf.validate.Violations violations = 3;`, 1) - for _, descriptors := range []bool{false, true} { - name := "sources" - if descriptors { - name = "descriptor-set" - } - t.Run(name, func(t *testing.T) { - parent := t.TempDir() - dest := filepath.Join(parent, "output") - require.NoError(t, os.Mkdir(dest, 0o750)) - require.NoError(t, os.Chmod(parent, 0o550)) - t.Cleanup(func() { require.NoError(t, os.Chmod(parent, 0o750)) }) - request := proto.ClientRequest{Language: languages.RUST, Destination: dest, Module: "rest"} - if descriptors { - request.DescriptorSet = plainDescriptorSetFor(t, ctx, source) - } else { - request.Sources = []proto.Source{{Path: restProtoPath, Content: []byte(source)}} - } - require.NoError(t, proto.GenerateClient(ctx, request)) - bindings, err := os.ReadFile(filepath.Join(dest, "saas/rest/v1/saas.rest.v1.rs")) - require.NoError(t, err) - require.Contains(t, string(bindings), "pub status:") - require.Contains(t, string(bindings), "pub violations:") - require.Contains(t, string(bindings), "::prost_types::Timestamp") - require.NoDirExists(t, filepath.Join(dest, "google/protobuf")) - unrelated := filepath.Join(dest, "src", "user.rs") - require.NoError(t, os.MkdirAll(filepath.Dir(unrelated), 0o750)) - require.NoError(t, os.WriteFile(unrelated, []byte("user source"), 0o600)) - broken := proto.ClientRequest{Language: languages.RUST, Destination: dest, - Sources: []proto.Source{{Path: "api.proto", Content: []byte("invalid proto")}}} - require.Error(t, proto.GenerateClient(ctx, broken)) - unchanged, err := os.ReadFile(filepath.Join(dest, "saas/rest/v1/saas.rest.v1.rs")) - require.NoError(t, err) - require.Equal(t, bindings, unchanged) - require.NoError(t, proto.GenerateClient(ctx, request)) - unchanged, err = os.ReadFile(unrelated) - require.NoError(t, err) - require.Equal(t, "user source", string(unchanged)) - require.NoError(t, os.WriteFile(filepath.Join(dest, "Cargo.toml"), []byte(`[package] -name = "generated-imports-test" -version = "0.0.0" -edition = "2021" -[dependencies] -prost = "=0.14.1" -prost-types = "=0.14.1" -tonic = "=0.14.2" -tonic-prost = "=0.14.2" -[lib] -path = "lib.rs" -`), 0o600)) - require.NoError(t, os.WriteFile(filepath.Join(dest, "lib.rs"), []byte(`pub mod google { pub mod rpc { include!("google/rpc/google.rpc.rs"); } } -pub mod buf { pub mod validate { include!("buf/validate/buf.validate.rs"); } } -pub mod saas { pub mod rest { pub mod v1 { include!("saas/rest/v1/saas.rest.v1.rs"); } } } -#[test] -fn imported_messages_round_trip() { - use prost::Message; - let value = saas::rest::v1::GetResponse { - created_at: Some(prost_types::Timestamp { seconds: 42, nanos: 7 }), - status: Some(google::rpc::Status { code: 9, message: "retained".into(), details: vec![] }), - violations: Some(buf::validate::Violations { violations: vec![] }), - }; - let decoded = saas::rest::v1::GetResponse::decode(value.encode_to_vec().as_slice()).unwrap(); - assert_eq!(decoded, value); - assert_eq!(decoded.status.unwrap().message, "retained"); -} -`), 0o600)) - command := exec.CommandContext(ctx, "cargo", "test", "--quiet") - command.Dir = dest - out, err := command.CombinedOutput() - require.NoError(t, err, "%s", out) - }) - } -} - // TestGoClientFromPlainDescriptorSetOwnsOnlyItsOwnProtos is the test that fails // without the marking, in both of the ways the marking has to fix. // diff --git a/companions/proto/flake.nix b/companions/proto/flake.nix index 6a47fc956..247340b73 100644 --- a/companions/proto/flake.nix +++ b/companions/proto/flake.nix @@ -221,8 +221,6 @@ # `go install`). protoc-gen-go protoc-gen-go-grpc - protoc-gen-prost - protoc-gen-tonic # grpc-gateway ships both protoc-gen-grpc-gateway and # protoc-gen-openapiv2 binaries; there are no separate # top-level attrs for them (referencing them directly is an diff --git a/companions/proto/grpc.go b/companions/proto/grpc.go index 31dce0c91..c80a0fd76 100644 --- a/companions/proto/grpc.go +++ b/companions/proto/grpc.go @@ -89,11 +89,6 @@ type TypeScriptConfiguration struct { IncludeImports bool } -type RustConfiguration struct { - Destination string - IncludeImports bool -} - // BufConfigurationOption tunes the rendered buf configuration. It is variadic so // existing callers keep compiling unchanged. type BufConfigurationOption func(*bufConfigurationOptions) @@ -143,16 +138,10 @@ func CreateBufConfiguration(ctx context.Context, bufDir string, service string, return w.Wrapf(err, "cannot templatize") } return nil - case languages.RUST: - err := templateRustConfiguration(ctx, bufDir, options.includeImports) - if err != nil { - return w.Wrapf(err, "cannot templatize") - } - return nil case languages.NotSupported: return w.NewError("language not supported") } - return w.NewError("unknown language") + return w.NewError("unsupported proto generation language %q; supported languages: go, python, typescript", language) } func templateGoConfiguration(ctx context.Context, bufDir string, goPackagePrefix string, facade FacadeOptions, goPackageOverrides map[string]string) error { @@ -206,20 +195,6 @@ func templateTypeScriptConfiguration(ctx context.Context, bufDir string, facade return nil } -func templateRustConfiguration(ctx context.Context, bufDir string, includeImports bool) error { - w := wool.Get(ctx).In("templateRustConfiguration", wool.Field("bufDir", bufDir)) - templator := &templates.Templator{NameReplacer: templates.CutTemplateSuffix{}} - conf := RustConfiguration{ - Destination: outputDir, - IncludeImports: includeImports, - } - err := templator.CopyAndApply(ctx, rustFS, "templates/rust", bufDir, conf) - if err != nil { - return w.Wrapf(err, "cannot copy and apply template") - } - return nil -} - // runBuf sets up the proto companion over a prepared temp dir and runs buf. // before, when set, runs after the runner is initialized but before buf // generate (used by the Python facade path to strip the descriptor image). @@ -294,6 +269,3 @@ var pythonFS embed.FS //go:embed templates/typescript var typescriptFS embed.FS - -//go:embed templates/rust -var rustFS embed.FS diff --git a/companions/proto/grpc_configuration_test.go b/companions/proto/grpc_configuration_test.go index 12e82fb97..945051534 100644 --- a/companions/proto/grpc_configuration_test.go +++ b/companions/proto/grpc_configuration_test.go @@ -283,7 +283,7 @@ func TestEveryLanguageResolvesTheSameSharedProtos(t *testing.T) { t.Fatalf("Go resolves no %s", module) } } - for _, language := range []languages.Language{languages.PYTHON, languages.TYPESCRIPT, languages.RUST} { + for _, language := range []languages.Language{languages.PYTHON, languages.TYPESCRIPT} { if got := deps(language); got != want { t.Errorf("%s resolves\n%s\nwhere Go resolves\n%s", language, got, want) } diff --git a/companions/proto/grpc_test.go b/companions/proto/grpc_test.go index c2d79c181..e6246347c 100644 --- a/companions/proto/grpc_test.go +++ b/companions/proto/grpc_test.go @@ -142,30 +142,6 @@ func TestGeneratePythonGRPC(t *testing.T) { } } -func TestGenerateRustGRPC(t *testing.T) { - wool.SetGlobalLogLevel(wool.DEBUG) - ctx := context.Background() - - testutil.RequireProtoImage(t, ctx) - - apiProto := filepath.Join(testdataDir(t), "api.proto") - ep := &resources.Endpoint{Module: "app", Service: "svc", Name: "api", Visibility: "private"} - api, err := resources.LoadGrpcAPI(ctx, shared.Pointer(apiProto)) - require.NoError(t, err) - grpc, err := resources.NewAPI(ctx, ep, resources.ToGrpcAPI(api)) - require.NoError(t, err) - destination := t.TempDir() - - err = proto.GenerateGRPC(ctx, languages.RUST, destination, "app/svc", grpc) - require.NoError(t, err, "proto companion image not built: %s", testutil.BuildCompanionsHint) - - // prost + tonic emit per proto package ("api/"): messages in api.rs, - // the tonic client in api.tonic.rs. - for _, name := range []string{"api/api.rs", "api/api.tonic.rs"} { - require.FileExists(t, filepath.Join(destination, name)) - } -} - // TestGenerateTypeScriptGRPC covers the namespaces protoc-gen-es imports by a // path relative to the bindings rather than from @bufbuild/protobuf/wkt. buf // generates nothing for a dependency module, so without being told otherwise it diff --git a/companions/proto/image.go b/companions/proto/image.go index dea2573b9..4feb11c73 100644 --- a/companions/proto/image.go +++ b/companions/proto/image.go @@ -1,14 +1,12 @@ package proto import ( - "fmt" "path" "strings" "github.com/codefly-dev/core/languages" "google.golang.org/protobuf/encoding/protowire" googleproto "google.golang.org/protobuf/proto" - "google.golang.org/protobuf/reflect/protodesc" "google.golang.org/protobuf/reflect/protoreflect" "google.golang.org/protobuf/types/descriptorpb" ) @@ -165,13 +163,10 @@ func isForeign(file *descriptorpb.FileDescriptorProto, language languages.Langua // Go and Python name a dropped file's package absolutely — a rewritten // go_package, an untouched `from buf.validate import validate_pb2` — so // every foreign namespace resolves to whatever the consumer installed. - // TypeScript and Rust do so only for their runtime-supplied protobuf types. + // TypeScript does so only for its runtime-supplied protobuf types. if language == languages.TYPESCRIPT { return protobufESRuntimeFiles[file.GetName()] } - if language == languages.RUST { - return file.GetPackage() == "google.protobuf" || strings.HasPrefix(file.GetPackage(), "google.protobuf.") - } return true } @@ -182,7 +177,7 @@ func isForeign(file *descriptorpb.FileDescriptorProto, language languages.Langua // // Which namespaces those are depends on language, because dropping one only // works if the generator refers to it by package name: Go and Python do that for -// all of them, TypeScript and Rust only for runtime-supplied protobuf types. +// all of them, TypeScript only for runtime-supplied protobuf types. // See foreignNamespace. // // A descriptor set handed to GenerateClient is a *plain* FileDescriptorSet — @@ -198,7 +193,7 @@ func isForeign(file *descriptorpb.FileDescriptorProto, language languages.Langua // bindings importing a package nothing generated. The returned map is fed back // into buf.gen.yaml as managed.override.GO_PACKAGE so each marked file keeps // the go_package it declares, and the module's bindings reference the canonical -// upstream package (google.golang.org/genproto/..., prost-types, ...) exactly +// upstream package (google.golang.org/genproto/..., ...) exactly // as they do on the Sources path. // // `buf generate --exclude-path google/protobuf` would drop the well-known types @@ -225,11 +220,6 @@ func MarkForeignImports(descriptorSet []byte, language languages.Language, goReq if err := googleproto.Unmarshal(descriptorSet, &set); err != nil { return nil, nil, err } - if language == languages.RUST { - if _, err := protodesc.NewFiles(&set); err != nil { - return nil, nil, fmt.Errorf("resolve Rust client descriptor types: %w", err) - } - } isImport := protowire.AppendVarint(protowire.AppendTag(nil, bufImageFileIsImportField, protowire.VarintType), 1) extension := protowire.AppendBytes(protowire.AppendTag(nil, bufImageFileExtensionField, protowire.BytesType), isImport) goPackages := make(map[string]string) diff --git a/companions/proto/image_test.go b/companions/proto/image_test.go index 5ddd01646..02f470d16 100644 --- a/companions/proto/image_test.go +++ b/companions/proto/image_test.go @@ -83,31 +83,6 @@ func marshal(t *testing.T, set *descriptorpb.FileDescriptorSet) []byte { return data } -func TestRustForeignBoundary(t *testing.T) { - for _, file := range foreignSet().File { - want := strings.HasPrefix(file.GetName(), "google/") && (file.GetPackage() == "google.protobuf" || strings.HasPrefix(file.GetPackage(), "google.protobuf.")) - require.Equal(t, want, isForeign(file, languages.RUST, nil), file.GetName()) - } - file := &descriptorpb.FileDescriptorProto{Name: googleproto.String("google/rpc/status.proto"), Package: googleproto.String("google.rpc")} - require.False(t, isForeign(file, languages.RUST, nil)) -} - -func TestRustRejectsUnresolvedMessageTypes(t *testing.T) { - set := &descriptorpb.FileDescriptorSet{File: []*descriptorpb.FileDescriptorProto{{ - Name: googleproto.String("response.proto"), Package: googleproto.String("example"), Syntax: googleproto.String("proto3"), - MessageType: []*descriptorpb.DescriptorProto{{ - Name: googleproto.String("Response"), - Field: []*descriptorpb.FieldDescriptorProto{{ - Name: googleproto.String("status"), Number: googleproto.Int32(1), - Type: descriptorpb.FieldDescriptorProto_TYPE_MESSAGE.Enum(), - TypeName: googleproto.String(".google.rpc.Status"), - }}, - }}, - }}} - _, _, err := MarkForeignImports(marshal(t, set), languages.RUST) - require.ErrorContains(t, err, "google.rpc.Status") -} - // TestMarkForeignImports pins which files of a descriptor set buf is told not to // generate for: every namespace whose bindings someone else publishes. Vendoring // any of them into the generated library puts a second registration of the same diff --git a/companions/proto/info.codefly.yaml b/companions/proto/info.codefly.yaml index 89acbc43c..fb15e7160 100644 --- a/companions/proto/info.codefly.yaml +++ b/companions/proto/info.codefly.yaml @@ -1 +1 @@ -version: 0.0.20 +version: 0.0.21 diff --git a/companions/proto/openapi.go b/companions/proto/openapi.go index a768ec2bb..d8b4ef9bb 100644 --- a/companions/proto/openapi.go +++ b/companions/proto/openapi.go @@ -101,6 +101,7 @@ func generateOpenAPIGo(ctx context.Context, unique string, image *resources.Dock runner.WithMount(root, "/workspace") runner.WithWorkDir("/workspace") runner.WithUser(hostUserSpec()) + runner.WithPause() defer func() { if shutErr := runner.Shutdown(ctx); shutErr != nil { w.Warn("cannot shutdown runner", wool.ErrField(shutErr)) diff --git a/companions/proto/rust_output.go b/companions/proto/rust_output.go deleted file mode 100644 index 35429314e..000000000 --- a/companions/proto/rust_output.go +++ /dev/null @@ -1,173 +0,0 @@ -package proto - -import ( - "context" - "crypto/sha256" - "encoding/hex" - "encoding/json" - "errors" - "fmt" - "io/fs" - "os" - "path/filepath" - "sort" - "time" - - "github.com/gofrs/flock" -) - -const rustOutputManifest = ".codefly-rust-output.json" -const rustOutputLock = ".codefly-rust-output.lock" - -func generateRustOutput(ctx context.Context, destination string, generate func(string) error) error { - destination, err := filepath.Abs(destination) - if err != nil { - return err - } - destination, err = filepath.EvalSymlinks(destination) - if err != nil { - return err - } - lock := flock.New(filepath.Join(destination, rustOutputLock), flock.SetPermissions(0o600)) - locked, err := lock.TryLockContext(ctx, 50*time.Millisecond) - if err != nil { - return err - } - if !locked { - return ctx.Err() - } - defer lock.Close() - stage, err := os.MkdirTemp(destination, ".rust-output-*") - if err != nil { - return err - } - defer os.RemoveAll(stage) - if err := generate(stage); err != nil { - return err - } - if err := ctx.Err(); err != nil { - return err - } - return publishRustOutput(destination, stage) -} - -func publishRustOutput(destination, stage string) error { - root, err := os.OpenRoot(destination) - if err != nil { - return err - } - defer root.Close() - previous := map[string]string{} - data, err := root.ReadFile(rustOutputManifest) - if err == nil { - if err := json.Unmarshal(data, &previous); err != nil { - return fmt.Errorf("read Rust output ownership: %w", err) - } - } else if !errors.Is(err, fs.ErrNotExist) { - return err - } - next := map[string]string{} - if err := filepath.WalkDir(stage, func(name string, entry fs.DirEntry, err error) error { - if err != nil || entry.IsDir() { - return err - } - if !entry.Type().IsRegular() { - return fmt.Errorf("non-regular Rust output: %s", name) - } - rel, err := filepath.Rel(stage, name) - if err != nil { - return err - } - data, err := os.ReadFile(name) - if err != nil { - return err - } - next[filepath.ToSlash(rel)] = rustOutputDigest(data) - return nil - }); err != nil { - return err - } - paths := map[string]bool{} - for name := range previous { - paths[name] = true - } - for name := range next { - paths[name] = true - } - names := make([]string, 0, len(paths)) - for name := range paths { - if !fs.ValidPath(name) || name == "." || name == rustOutputManifest || name == rustOutputLock { - return fmt.Errorf("invalid Rust output path: %q", name) - } - info, err := root.Lstat(name) - if errors.Is(err, fs.ErrNotExist) { - names = append(names, name) - continue - } - if err != nil { - return err - } - if !info.Mode().IsRegular() { - return fmt.Errorf("Rust output conflicts with non-regular file: %s", name) - } - data, err := root.ReadFile(name) - if err != nil { - return err - } - if previous[name] == "" || rustOutputDigest(data) != previous[name] { - return fmt.Errorf("Rust output conflicts with unowned or edited file: %s", name) - } - names = append(names, name) - } - sort.Strings(names) - data, err = json.Marshal(next) - if err != nil { - return err - } - if err := os.WriteFile(filepath.Join(stage, rustOutputManifest), data, 0o600); err != nil { - return err - } - names = append(names, rustOutputManifest) - backup, err := os.MkdirTemp(destination, ".rust-backup-*") - if err != nil { - return err - } - defer os.Remove(backup) - var restore []func() error - rollback := func(cause error) error { - for i := len(restore) - 1; i >= 0; i-- { - cause = errors.Join(cause, restore[i]()) - } - return cause - } - for index, name := range names { - old := filepath.Join(filepath.Base(backup), fmt.Sprint(index)) - if err := root.Rename(name, old); err == nil { - restore = append(restore, func() error { return root.Rename(old, name) }) - } else if !errors.Is(err, fs.ErrNotExist) { - return rollback(err) - } - if _, exists := next[name]; !exists && name != rustOutputManifest { - continue - } - if err := root.MkdirAll(filepath.Dir(name), 0o750); err != nil { - return rollback(err) - } - // Stage and destination share a filesystem; root confines publication even - // when a destination ancestor is a symlink. - staged := filepath.Join(filepath.Base(backup), "next") - if err := os.Rename(filepath.Join(stage, filepath.FromSlash(name)), filepath.Join(destination, staged)); err != nil { - return rollback(err) - } - if err := root.Rename(staged, name); err != nil { - return rollback(err) - } - restore = append(restore, func() error { return root.Remove(name) }) - } - return os.RemoveAll(backup) -} - -func rustOutputDigest(data []byte) string { - digest := sha256.Sum256(data) - return hex.EncodeToString(digest[:]) -} diff --git a/companions/proto/templates/rust/buf.gen.yaml.tmpl b/companions/proto/templates/rust/buf.gen.yaml.tmpl deleted file mode 100644 index feb7d3f89..000000000 --- a/companions/proto/templates/rust/buf.gen.yaml.tmpl +++ /dev/null @@ -1,21 +0,0 @@ -version: v2 -plugins: - # The companion pins both local plugins to 0.5.0, targeting tonic/prost 0.14 - # (codec via the separate `tonic-prost` crate). Local execution preserves - # descriptor-set target selection for imported message types. - # - # NOT compile_well_known_types: that makes prost emit RELATIVE paths for - # google.protobuf.* (`super::super::google::…`) which assume one exact module - # nesting and break under any include layout. The default (extern paths to the - # `prost-types` crate — `::prost_types::Timestamp`, and `Empty` → `()`) is - # layout-independent, so the consuming crate just depends on `prost-types`. - - local: protoc-gen-prost - out: {{.Destination}} - strategy: all - include_imports: {{.IncludeImports}} - - local: protoc-gen-tonic - out: {{.Destination}} - strategy: all - include_imports: {{.IncludeImports}} - opt: - - no_server diff --git a/companions/proto/templates/rust/buf.yaml.tmpl b/companions/proto/templates/rust/buf.yaml.tmpl deleted file mode 100644 index 9403b2c6e..000000000 --- a/companions/proto/templates/rust/buf.yaml.tmpl +++ /dev/null @@ -1,11 +0,0 @@ -version: v1 -deps: - - buf.build/googleapis/googleapis - - buf.build/grpc-ecosystem/grpc-gateway - - buf.build/grpc/grpc - - buf.build/bufbuild/protovalidate -lint: - use: - - DEFAULT - ignore: - - PACKAGE_UNUSED diff --git a/docs/runbooks/bump-protobuf-es.md b/docs/runbooks/bump-protobuf-es.md index 3bdc03dfe..64ee3e3da 100644 --- a/docs/runbooks/bump-protobuf-es.md +++ b/docs/runbooks/bump-protobuf-es.md @@ -103,7 +103,7 @@ Six places, and they must agree: |---|---|---| | `companions/proto/facades/ts/package.json` | `@bufbuild/protoplugin` | yes | | `companions/proto/facades/ts/package-lock.json` | the resolved `protoplugin` + `protobuf` closure | yes | -| `companions/proto/Dockerfile:104` | `@bufbuild/protoc-gen-es@` installed via npm | yes | +| `companions/proto/Dockerfile:99` | `@bufbuild/protoc-gen-es@` installed via npm | yes | | `companions/proto/flake.nix:51` | `protocGenEsVersion`, built from source | yes | | `companions/proto/flake.nix:59` | `hash` — the protobuf-es source tarball | **no** | | `companions/proto/flake.nix:61` | `npmDepsHash` — its npm dependency closure | **no** | diff --git a/version/info.codefly.yaml b/version/info.codefly.yaml index 9469d9d18..ea8113a3e 100644 --- a/version/info.codefly.yaml +++ b/version/info.codefly.yaml @@ -1 +1 @@ -version: 0.19.12 +version: 0.19.13