From 7a3553d7e79d273325f848cc8adc4a30db2e9039 Mon Sep 17 00:00:00 2001 From: crisfiorini-oss <301773486+crisfiorini-oss@users.noreply.github.com> Date: Thu, 8 Oct 2026 16:03:12 -0400 Subject: [PATCH 01/10] feat(auth): add the shared x-codefly-token caller authentication (#12) A gateway a Codefly workspace composes is reachable by whatever shares its network, so it has to ask each caller for the deployment's token. This adds the mechanism service-object-storage already uses (internal/auth) and the Codefly host uses for agent plugins: a shared secret in the x-codefly-token metadata header, compared in constant time, on unary and streaming calls, with the standard gRPC health service exempt because a readiness probe carries no token. It is a copy rather than an import: codefly-dev/core exports only the header name, from the package that links the whole agent runtime, and keeps its interceptors unexported, and codefly-dev/sdk-go has none. The package says so, so a fix to one is made in the other. This is authentication only. Which caller may run which statement stays settled before the request arrives; nothing here knows a caller. Not wired into the server yet; the next commit does that. Co-Authored-By: Claude Sonnet 5.5 --- internal/auth/auth.go | 107 ++++++++++++++++++++++++++++++ internal/auth/auth_test.go | 132 +++++++++++++++++++++++++++++++++++++ 2 files changed, 239 insertions(+) create mode 100644 internal/auth/auth.go create mode 100644 internal/auth/auth_test.go diff --git a/internal/auth/auth.go b/internal/auth/auth.go new file mode 100644 index 0000000..193fcdd --- /dev/null +++ b/internal/auth/auth.go @@ -0,0 +1,107 @@ +// Package auth authenticates callers of the gateway's gRPC surface: does the +// caller present the one token this deployment was started with. It decides +// nothing else — which caller may run which statement is settled before the +// request arrives (see AGENTS.md), so there is no identity here, no +// per-method rule, and no statement inspection. +// +// It is the same mechanism codefly-dev/service-object-storage uses +// (internal/auth) and the one the Codefly host already uses to talk to agent +// plugins: a shared secret carried as the "x-codefly-token" metadata header, +// compared in constant time. No exported equivalent exists to import: +// codefly-dev/core exports only the key (agents.AuthMetadataKey, in the +// package that links the whole agent runtime) and keeps its interceptors +// unexported, and codefly-dev/sdk-go has none. The key is therefore redeclared +// here, and this package is a copy of service-object-storage's; a fix to one +// belongs in the other. +package auth + +import ( + "context" + "crypto/subtle" + "strings" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" +) + +// MetadataKey is the gRPC metadata key carrying the token. Lowercase per gRPC +// convention — metadata keys are case-insensitive but the wire form is +// lowercase. +const MetadataKey = "x-codefly-token" + +// HealthServicePrefix is the standard gRPC health service. Its Check and Watch +// carry no data and no authority: a readiness probe — the Codefly CLI's, a +// load balancer's, a mesh's — presents no token, so a guarded health service +// makes the gateway look permanently unready to everything that would route to +// it. Every other method stays behind the token. +const HealthServicePrefix = "/grpc.health.v1.Health/" + +// IsHealthMethod reports whether fullMethod belongs to the gRPC health service. +func IsHealthMethod(fullMethod string) bool { + return strings.HasPrefix(fullMethod, HealthServicePrefix) +} + +// UnaryInterceptor rejects every unary call that does not present token. +func UnaryInterceptor(token string) grpc.UnaryServerInterceptor { + return func(ctx context.Context, req any, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (any, error) { + if !IsHealthMethod(info.FullMethod) { + if err := verify(ctx, token); err != nil { + return nil, err + } + } + return handler(ctx, req) + } +} + +// StreamInterceptor rejects every streaming call that does not present token. +// gRPC runs it before the handler observes a single frame, so an unauthorized +// InsertRows is refused at stream open rather than after its first batch is +// accepted. +func StreamInterceptor(token string) grpc.StreamServerInterceptor { + return func(srv any, ss grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) error { + if !IsHealthMethod(info.FullMethod) { + if err := verify(ss.Context(), token); err != nil { + return err + } + } + return handler(srv, ss) + } +} + +// DialOption is how a client presents token on every call it makes to the +// gateway. +func DialOption(token string) grpc.DialOption { + return grpc.WithPerRPCCredentials(bearer(token)) +} + +func verify(ctx context.Context, expected string) error { + md, ok := metadata.FromIncomingContext(ctx) + if !ok { + return status.Error(codes.Unauthenticated, "missing "+MetadataKey) + } + presented := md.Get(MetadataKey) + // An empty presented value would compare equal to an empty expected one, so + // it is refused as absent rather than reaching the comparison. + if len(presented) == 0 || presented[0] == "" { + return status.Error(codes.Unauthenticated, "missing "+MetadataKey) + } + if subtle.ConstantTimeCompare([]byte(presented[0]), []byte(expected)) != 1 { + return status.Error(codes.Unauthenticated, "invalid "+MetadataKey) + } + return nil +} + +// bearer carries the token as per-RPC credentials. +type bearer string + +func (b bearer) GetRequestMetadata(context.Context, ...string) (map[string]string, error) { + return map[string]string{MetadataKey: string(b)}, nil +} + +// RequireTransportSecurity is false: the local profile reaches the gateway over +// the Docker host bridge without TLS, and the deployed profile terminates +// transport security at the mesh. Demanding it here would make the credential +// unusable on both. +func (b bearer) RequireTransportSecurity() bool { return false } diff --git a/internal/auth/auth_test.go b/internal/auth/auth_test.go new file mode 100644 index 0000000..dab3f3a --- /dev/null +++ b/internal/auth/auth_test.go @@ -0,0 +1,132 @@ +package auth_test + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" + + "github.com/codefly-dev/service-warehouse/internal/auth" +) + +const token = "89d0d1f6d0e2f5a4c3b2a1908f7e6d5c" + +// peerContext is the context a handler sees for a call whose metadata carries +// the given header values; none means the call carried no metadata at all. +func peerContext(values ...string) context.Context { + if len(values) == 0 { + return context.Background() + } + return metadata.NewIncomingContext(context.Background(), metadata.Pairs(auth.MetadataKey, values[0])) +} + +// serverStream is the least a stream interceptor needs of a ServerStream: the +// context the call arrived with. +type serverStream struct { + grpc.ServerStream + ctx context.Context +} + +func (s serverStream) Context() context.Context { return s.ctx } + +// callUnary runs one unary call through the interceptor and reports whether +// the handler was reached and what the caller was told. +func callUnary(ctx context.Context, method string) (reached bool, err error) { + _, err = auth.UnaryInterceptor(token)(ctx, nil, &grpc.UnaryServerInfo{FullMethod: method}, + func(context.Context, any) (any, error) { + reached = true + return nil, nil + }) + return reached, err +} + +func callStream(ctx context.Context, method string) (reached bool, err error) { + err = auth.StreamInterceptor(token)(nil, serverStream{ctx: ctx}, &grpc.StreamServerInfo{FullMethod: method}, + func(any, grpc.ServerStream) error { + reached = true + return nil + }) + return reached, err +} + +const dataMethod = "/codefly.warehouse.v0.Warehouse/Query" + +// Every way of not presenting the token is refused with Unauthenticated before +// the handler runs, on both interceptors. The near misses matter most: a +// comparison that stopped at the shorter length, or that accepted an empty +// value against an empty expectation, would pass an obvious wrong-token test. +func TestOnlyTheExactTokenReachesTheHandler(t *testing.T) { + refused := map[string][]string{ + "no metadata": nil, + "empty value": {""}, + "wrong token": {"00000000000000000000000000000000"}, + "prefix of token": {token[:len(token)-1]}, + "token plus extra": {token + "x"}, + "different case": {"89D0D1F6D0E2F5A4C3B2A1908F7E6D5C"}, + } + for name, values := range refused { + t.Run(name, func(t *testing.T) { + reached, err := callUnary(peerContext(values...), dataMethod) + require.False(t, reached, "the handler must not run for a refused unary call") + require.Equal(t, codes.Unauthenticated, status.Code(err), "got %v", err) + + reached, err = callStream(peerContext(values...), dataMethod) + require.False(t, reached, "the handler must not run for a refused stream") + require.Equal(t, codes.Unauthenticated, status.Code(err), "got %v", err) + }) + } + + reached, err := callUnary(peerContext(token), dataMethod) + require.NoError(t, err) + require.True(t, reached) + + reached, err = callStream(peerContext(token), dataMethod) + require.NoError(t, err) + require.True(t, reached) +} + +// The refusal says which header was wrong, not what the right one is. +func TestRefusalNeverEchoesTheExpectedToken(t *testing.T) { + _, err := callUnary(peerContext("wrong"), dataMethod) + require.Error(t, err) + require.NotContains(t, err.Error(), token) +} + +// The health service is the one exemption: a readiness probe carries no token. +// Anything that merely looks like it does not inherit it. +func TestOnlyTheHealthServiceIsExempt(t *testing.T) { + for _, method := range []string{"/grpc.health.v1.Health/Check", "/grpc.health.v1.Health/Watch"} { + t.Run(method, func(t *testing.T) { + require.True(t, auth.IsHealthMethod(method)) + + reached, err := callUnary(peerContext(), method) + require.NoError(t, err) + require.True(t, reached) + + reached, err = callStream(peerContext(), method) + require.NoError(t, err) + require.True(t, reached) + }) + } + + for _, method := range []string{ + "/grpc.health.v1.HealthX/Check", + "/grpc.health.v1.Healthy/Check", + "/other.Service/grpc.health.v1.Health/Check", + "/codefly.warehouse.v0.Warehouse/Health", + "grpc.health.v1.Health/Check", + "", + } { + t.Run("not "+method, func(t *testing.T) { + require.False(t, auth.IsHealthMethod(method)) + + reached, err := callUnary(peerContext(), method) + require.False(t, reached) + require.Equal(t, codes.Unauthenticated, status.Code(err), "got %v", err) + }) + } +} From 899f139b671da5e38d50c7cd3837a0d386ebed8c Mon Sep 17 00:00:00 2001 From: crisfiorini-oss <301773486+crisfiorini-oss@users.noreply.github.com> Date: Thu, 8 Oct 2026 16:03:12 -0400 Subject: [PATCH 02/10] feat(config): refuse an unauthenticated listener and require the caller's token (#12) The server runs SQL against the database it is bound to and its listen address says nothing about who can reach it, so config resolution now refuses to start without SWH_AUTH_TOKEN unless the operator states SWH_ALLOW_ANONYMOUS=true for a listener confined elsewhere (network policy, mesh mTLS). Both together is refused as a contradiction, a malformed opt-out is an error rather than a silent false, and the token is trimmed because a secret delivered through a file or block scalar carries a trailing newline no client would send. newGRPCServer, which run() now uses, installs the auth interceptors from that token and registers the gRPC health service they exempt. The health service reports the process as serving, not that a warehouse is reachable: this server does not probe its backend. The tests enumerate the methods the server registers instead of a list kept beside it, so an RPC added to the proto is covered by the refusal test the day it exists. Mutating the wiring (no interceptors, no stream interceptor, no health exemption) turns the matching tests red. Behaviour change: a bare SWH_BACKEND=mem no longer starts. README and AGENTS.md now give the command with SWH_ALLOW_ANONYMOUS=true. Not verified: no codefly host was involved, so token delivery through the configuration the host provides is untested here; this repository has no agent yet to receive it. Co-Authored-By: Claude Sonnet 5.5 --- AGENTS.md | 6 +- README.md | 25 +++ cmd/service-warehouse/auth_test.go | 242 +++++++++++++++++++++++++++++ cmd/service-warehouse/main.go | 40 ++++- internal/config/config.go | 61 ++++++++ internal/config/config_test.go | 93 ++++++++++- 6 files changed, 460 insertions(+), 7 deletions(-) create mode 100644 cmd/service-warehouse/auth_test.go diff --git a/AGENTS.md b/AGENTS.md index 39a1449..81883c0 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -91,10 +91,12 @@ go test -race ./... ``` Run the server locally against the zero-dependency backend — it needs no -database, no credential, and no network: +database, no warehouse credential, and no network. It still refuses an +unauthenticated listener, so a local run states that choice (or sets +`SWH_AUTH_TOKEN`): ```sh -SWH_BACKEND=mem go run ./cmd/service-warehouse +SWH_BACKEND=mem SWH_ALLOW_ANONYMOUS=true go run ./cmd/service-warehouse ``` The default `SWH_BACKEND=duckdb` refuses to start, by design: the DuckDB diff --git a/README.md b/README.md index 211d342..465f01a 100644 --- a/README.md +++ b/README.md @@ -93,6 +93,30 @@ next milestone is `duckdb` (SQL execution + Arrow encoding), which makes the | `SWH_CREDENTIALS_FILE` | — | BigQuery service-account JSON (else ADC) | | `SWH_MAX_QUERY_BYTES` | `0` | per-query scan cap (0 = backend default) | | `SWH_QUERY_TIMEOUT` | `0` | per-query timeout (Go duration; 0 = backend default) | +| `SWH_AUTH_TOKEN` | — | shared secret every caller must present; required unless `SWH_ALLOW_ANONYMOUS=true` | +| `SWH_ALLOW_ANONYMOUS` | `false` | `true` accepts callers with no token; refused together with `SWH_AUTH_TOKEN` | + +## Authentication + +The server holds the warehouse credentials and runs SQL against the database it +is bound to, and the listen address says nothing about who can reach it. So it +**refuses to start on an unauthenticated listener** unless the operator says +otherwise: `SWH_AUTH_TOKEN` must be set, or `SWH_ALLOW_ANONYMOUS` must be `true` +(for a listener confined to a private boundary enforced elsewhere, such as a +cluster NetworkPolicy or service-mesh mTLS). + +With `SWH_AUTH_TOKEN` set, every RPC — unary and streaming, `Capabilities` +included — must carry the secret as the `x-codefly-token` gRPC metadata header, +the key the Codefly host already uses to authenticate agent plugins. Anything +else is rejected with `UNAUTHENTICATED` before it reaches a backend. The token +is compared in constant time and never logged. The standard gRPC health service +(`grpc.health.v1.Health`) is the one exemption: a readiness probe carries no +token. It reports that the process is serving, not that a warehouse is +reachable. + +This is authentication only — whether the caller holds the deployment's token. +The server still authorizes nothing: which caller may run which statement is +settled before the request arrives. ## Distribution and SBOM evidence @@ -175,6 +199,7 @@ proto/buf.{yaml,gen.yaml} buf config, read by the proto companion gen/ generated gRPC stubs internal/backend/ Backend interface + mem, duckdb (+ cloud backends) internal/server/ gRPC Warehouse implementation + proto↔backend mapping +internal/auth/ caller authentication (the x-codefly-token interceptors) internal/config/ env configuration internal/serr/ normalized error model cmd/service-warehouse/ the server binary diff --git a/cmd/service-warehouse/auth_test.go b/cmd/service-warehouse/auth_test.go new file mode 100644 index 0000000..78355e1 --- /dev/null +++ b/cmd/service-warehouse/auth_test.go @@ -0,0 +1,242 @@ +package main + +import ( + "context" + "errors" + "io" + "net" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/require" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials/insecure" + healthv1 "google.golang.org/grpc/health/grpc_health_v1" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" + "google.golang.org/grpc/test/bufconn" + "google.golang.org/protobuf/types/known/emptypb" + + whv0 "github.com/codefly-dev/service-warehouse/gen/codefly/warehouse/v0" + "github.com/codefly-dev/service-warehouse/internal/auth" + "github.com/codefly-dev/service-warehouse/internal/backend" + "github.com/codefly-dev/service-warehouse/internal/backend/mem" +) + +// serverToken is the secret the test gateway enforces. +const serverToken = "89d0d1f6d0e2f5a4c3b2a1908f7e6d5c" + +// startGateway serves the server run() serves — built by the same newGRPCServer +// — over an in-memory listener and returns it with a way to dial it. +func startGateway(t *testing.T, token string) (*grpc.Server, func(...grpc.DialOption) *grpc.ClientConn) { + t.Helper() + lis := bufconn.Listen(1 << 20) + srv := newGRPCServer(mem.New(backend.Config{}), token) + go func() { _ = srv.Serve(lis) }() + t.Cleanup(srv.Stop) + + return srv, func(opts ...grpc.DialOption) *grpc.ClientConn { + t.Helper() + conn, err := grpc.NewClient("passthrough:///bufnet", append(opts, + grpc.WithContextDialer(func(context.Context, string) (net.Conn, error) { return lis.Dial() }), + grpc.WithTransportCredentials(insecure.NewCredentials()), + )...) + require.NoError(t, err) + t.Cleanup(func() { _ = conn.Close() }) + return conn + } +} + +// rpc is one method a peer could call, as the server itself registered it. +type rpc struct { + name string + call func(context.Context, *grpc.ClientConn) error +} + +// registeredRPCs lists every method of every service the server registers +// except the health service, read from the server rather than from a list kept +// beside it: an RPC added to the proto, or a service added to the server, is +// covered by the refusal tests the moment it exists, without anyone remembering +// to extend them. +// +// Each call sends an empty message, which every request type decodes, so the +// outcome is decided by the interceptor alone. A call that gets past it ends in +// whatever status its handler gives an empty request; that is not asserted here. +func registeredRPCs(t *testing.T, srv *grpc.Server) []rpc { + t.Helper() + var rpcs []rpc + for service, info := range srv.GetServiceInfo() { + for _, m := range info.Methods { + full := "/" + service + "/" + m.Name + if auth.IsHealthMethod(full) { + continue + } + rpcs = append(rpcs, rpc{name: full, call: caller(full, m)}) + } + } + require.NotEmpty(t, rpcs) + return rpcs +} + +func caller(full string, m grpc.MethodInfo) func(context.Context, *grpc.ClientConn) error { + if !m.IsClientStream && !m.IsServerStream { + return func(ctx context.Context, conn *grpc.ClientConn) error { + return conn.Invoke(ctx, full, &emptypb.Empty{}, &emptypb.Empty{}) + } + } + return func(ctx context.Context, conn *grpc.ClientConn) error { + st, err := conn.NewStream(ctx, &grpc.StreamDesc{ + ServerStreams: m.IsServerStream, + ClientStreams: m.IsClientStream, + }, full) + if err != nil { + return err + } + // A refused stream makes SendMsg fail with io.EOF; RecvMsg then carries + // the real status. + if err := st.SendMsg(&emptypb.Empty{}); err != nil && !errors.Is(err, io.EOF) { + return err + } + if err := st.CloseSend(); err != nil { + return err + } + for { + if err := st.RecvMsg(&emptypb.Empty{}); err != nil { + if errors.Is(err, io.EOF) { + return nil + } + return err + } + } + } +} + +// TestGatewayCoversTheWholeContract pins what the two tests below enumerate: the +// Warehouse service as the proto declares it, the streaming methods included. +// Without it, those tests would still pass if the enumeration quietly saw less +// than the server serves. +func TestGatewayCoversTheWholeContract(t *testing.T) { + srv, _ := startGateway(t, serverToken) + + prefix := "/" + whv0.Warehouse_ServiceDesc.ServiceName + "/" + declared := len(whv0.Warehouse_ServiceDesc.Methods) + len(whv0.Warehouse_ServiceDesc.Streams) + var covered int + for _, r := range registeredRPCs(t, srv) { + if strings.HasPrefix(r.name, prefix) { + covered++ + } + } + require.Equal(t, declared, covered) + require.NotEmpty(t, whv0.Warehouse_ServiceDesc.Streams, "the contract has streaming methods, which take the stream interceptor") +} + +// TestUnauthenticatedPeerIsRefused is the core guarantee: a peer that reaches +// the port without the deployment's token can run nothing — over unary and +// streaming RPCs alike. +func TestUnauthenticatedPeerIsRefused(t *testing.T) { + srv, dial := startGateway(t, serverToken) + + peers := map[string]*grpc.ClientConn{ + "no credentials": dial(), + "wrong token": dial(auth.DialOption("00000000000000000000000000000000")), + "prefix of token": dial(auth.DialOption(serverToken[:len(serverToken)-1])), + "token plus extra": dial(auth.DialOption(serverToken + "x")), + "empty token": dial(auth.DialOption("")), + } + + for peerName, conn := range peers { + for _, r := range registeredRPCs(t, srv) { + t.Run(peerName+"/"+r.name, func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + err := r.call(ctx, conn) + require.Error(t, err) + require.Equal(t, codes.Unauthenticated, status.Code(err), "got %v", err) + }) + } + } +} + +// TestAuthorizedPeerReachesTheHandler pairs with the refusal test: the same +// calls must get through once the token is presented, or the gateway would be +// locked shut rather than protected. A handler-level status (InvalidArgument, +// Unimplemented) still proves the interceptor let the call through. +func TestAuthorizedPeerReachesTheHandler(t *testing.T) { + srv, dial := startGateway(t, serverToken) + conn := dial(auth.DialOption(serverToken)) + + for _, r := range registeredRPCs(t, srv) { + t.Run(r.name, func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + err := r.call(ctx, conn) + require.NotEqual(t, codes.Unauthenticated, status.Code(err), "got %v", err) + }) + } +} + +// TestTokenTravelsUnderTheEcosystemKey pins the wire contract: the gateway reads +// the same metadata key the codefly host uses for agent plugins, so a caller +// built against that convention interoperates. +func TestTokenTravelsUnderTheEcosystemKey(t *testing.T) { + require.Equal(t, "x-codefly-token", auth.MetadataKey) + + _, dial := startGateway(t, serverToken) + conn := dial() + ctx := metadata.AppendToOutgoingContext(context.Background(), "x-codefly-token", serverToken) + ctx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + _, err := whv0.NewWarehouseClient(conn).Capabilities(ctx, &whv0.CapabilitiesRequest{}) + require.NoError(t, err) +} + +// TestTokenUnderAnotherHeaderIsRefused guards against a caller that presents the +// right secret the wrong way: an HTTP-style Authorization bearer is not the +// gateway's credential and must not be accepted. +func TestTokenUnderAnotherHeaderIsRefused(t *testing.T) { + _, dial := startGateway(t, serverToken) + ctx := metadata.AppendToOutgoingContext(context.Background(), "authorization", "Bearer "+serverToken) + ctx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + _, err := whv0.NewWarehouseClient(dial()).Capabilities(ctx, &whv0.CapabilitiesRequest{}) + require.Equal(t, codes.Unauthenticated, status.Code(err), "got %v", err) +} + +// TestHealthNeedsNoToken pins the one exemption: the standard gRPC health +// service answers a peer that presents no token. A readiness probe — the Codefly +// CLI's, a load balancer's, a mesh's — carries none, and a guarded health service +// makes the gateway look permanently unready to all of them. Every data RPC stays +// refused; TestUnauthenticatedPeerIsRefused holds that side. +func TestHealthNeedsNoToken(t *testing.T) { + _, dial := startGateway(t, serverToken) + health := healthv1.NewHealthClient(dial()) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + resp, err := health.Check(ctx, &healthv1.HealthCheckRequest{}) + require.NoError(t, err) + require.Equal(t, healthv1.HealthCheckResponse_SERVING, resp.GetStatus()) + + stream, err := health.Watch(ctx, &healthv1.HealthCheckRequest{}) + require.NoError(t, err) + first, err := stream.Recv() + require.NoError(t, err) + require.Equal(t, healthv1.HealthCheckResponse_SERVING, first.GetStatus()) +} + +// TestWithoutATokenTheListenerIsAnonymous is the other side of the config guard: +// config.FromEnv resolves an empty token only when the operator chose +// SWH_ALLOW_ANONYMOUS, and then the server must take callers with no credential +// rather than refuse everyone. +func TestWithoutATokenTheListenerIsAnonymous(t *testing.T) { + _, dial := startGateway(t, "") + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + _, err := whv0.NewWarehouseClient(dial()).Capabilities(ctx, &whv0.CapabilitiesRequest{}) + require.NoError(t, err) +} diff --git a/cmd/service-warehouse/main.go b/cmd/service-warehouse/main.go index 4743980..abbb011 100644 --- a/cmd/service-warehouse/main.go +++ b/cmd/service-warehouse/main.go @@ -15,8 +15,11 @@ import ( "time" "google.golang.org/grpc" + grpchealth "google.golang.org/grpc/health" + healthv1 "google.golang.org/grpc/health/grpc_health_v1" whv0 "github.com/codefly-dev/service-warehouse/gen/codefly/warehouse/v0" + "github.com/codefly-dev/service-warehouse/internal/auth" "github.com/codefly-dev/service-warehouse/internal/backend" "github.com/codefly-dev/service-warehouse/internal/config" "github.com/codefly-dev/service-warehouse/internal/server" @@ -55,8 +58,7 @@ func run() error { return err } - grpcServer := grpc.NewServer() - whv0.RegisterWarehouseServer(grpcServer, server.New(be)) + grpcServer := newGRPCServer(be, cfg.AuthToken) sig := make(chan os.Signal, 1) signal.Notify(sig, os.Interrupt, syscall.SIGTERM) @@ -66,11 +68,41 @@ func run() error { gracefulStop(grpcServer, shutdownGrace) }() - log.Printf("warehouse gateway listening on %s (backend=%s database=%s)", - cfg.ListenAddr, cfg.Backend.Kind, cfg.Backend.Database) + // Config resolution refuses an empty token unless the operator accepted an + // unauthenticated listener, so the empty case here is that choice. The token + // itself is never logged. + authMode := "none" + if cfg.AuthToken != "" { + authMode = "token" + } + log.Printf("warehouse gateway listening on %s (backend=%s database=%s auth=%s)", + cfg.ListenAddr, cfg.Backend.Kind, cfg.Backend.Database, authMode) return grpcServer.Serve(lis) } +// newGRPCServer assembles the gateway's gRPC server: the Warehouse service over +// be, the standard health service, and — when token is set — the interceptors +// that refuse any call that does not present it. The health service is exempt +// from them (see auth.HealthServicePrefix) so a readiness probe, which carries +// no token, can still reach it. +// +// The health service reports the process as serving and nothing more. It is +// liveness, not a verdict on the backend: this server does not probe its +// warehouse, so it claims nothing about whether one is reachable. +func newGRPCServer(be backend.Backend, token string) *grpc.Server { + var options []grpc.ServerOption + if token != "" { + options = append(options, + grpc.ChainUnaryInterceptor(auth.UnaryInterceptor(token)), + grpc.ChainStreamInterceptor(auth.StreamInterceptor(token)), + ) + } + srv := grpc.NewServer(options...) + healthv1.RegisterHealthServer(srv, grpchealth.NewServer()) + whv0.RegisterWarehouseServer(srv, server.New(be)) + return srv +} + // gracefulStop drains in-flight RPCs, but escalates to a hard Stop if they do // not finish within grace, so a stuck stream can never make shutdown hang. func gracefulStop(srv *grpc.Server, grace time.Duration) { diff --git a/internal/config/config.go b/internal/config/config.go index dafe75e..1a9c8a2 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -9,8 +9,10 @@ import ( "fmt" "os" "strconv" + "strings" "time" + "github.com/codefly-dev/service-warehouse/internal/auth" "github.com/codefly-dev/service-warehouse/internal/backend" ) @@ -18,6 +20,12 @@ import ( type Config struct { ListenAddr string Backend backend.Config + // AuthToken is the shared secret every caller must present as the + // auth.MetadataKey metadata header, whitespace-trimmed: a secret delivered + // through a file or a block scalar arrives with a trailing newline that no + // client would send. Empty means the listener is anonymous, which FromEnv + // resolves only when the operator asked for it with SWH_ALLOW_ANONYMOUS. + AuthToken string } // FromEnv resolves configuration from SWH_* environment variables. @@ -61,6 +69,7 @@ func FromEnv() (Config, error) { MaxQueryBytes: maxQueryBytes, QueryTimeout: queryTimeout, }, + AuthToken: strings.TrimSpace(os.Getenv("SWH_AUTH_TOKEN")), } // Resolve the kind before anything derived from it, so an unimplemented @@ -73,9 +82,46 @@ func FromEnv() (Config, error) { if !reg.SelfContained && cfg.Backend.Database == "" && cfg.Backend.DSN == "" { return Config{}, fmt.Errorf("SWH_DATABASE (or SWH_DSN) is required for backend %q", cfg.Backend.Kind) } + // Last, so a configuration broken in more than one way reports the backend + // first: the token cannot be what stops a server that names no real backend. + if err := checkListenerIsAuthenticated(cfg.AuthToken); err != nil { + return Config{}, err + } return cfg, nil } +// checkListenerIsAuthenticated refuses to resolve a configuration that would +// run SQL against the bound database for anyone who can reach the listen port. +// The listen address says nothing about who that is, so the token is the only +// thing this server can enforce itself. This is authentication only: it asks +// whether the caller holds the deployment's token, never what that caller may +// do — that is settled before the request arrives. +func checkListenerIsAuthenticated(token string) error { + anonymous, err := envBool("SWH_ALLOW_ANONYMOUS") + if err != nil { + return err + } + if token != "" { + // Both set is a contradiction, not a preference: "here is a credential" + // and "accept callers with no credential" cannot both be the intent, and + // silently picking one leaves the operator believing the other. It is + // also the shape a half-finished migration takes — a token added to the + // secret while the manifest still carries the opt-out. + if anonymous { + return fmt.Errorf("SWH_AUTH_TOKEN and SWH_ALLOW_ANONYMOUS=true are mutually exclusive: " + + "unset SWH_ALLOW_ANONYMOUS to enforce the token, or unset SWH_AUTH_TOKEN to accept anonymous callers") + } + return nil + } + if anonymous { + return nil + } + return fmt.Errorf("SWH_AUTH_TOKEN is required: without it every caller that can reach the listener can run SQL against the bound database. "+ + "Set SWH_AUTH_TOKEN to a shared secret and have clients send it as the %q gRPC metadata header, "+ + "or set SWH_ALLOW_ANONYMOUS=true if the listener is confined to a private boundary enforced elsewhere (cluster NetworkPolicy / service-mesh mTLS)", + auth.MetadataKey) +} + func env(key, def string) string { if v := os.Getenv(key); v != "" { return v @@ -98,6 +144,21 @@ func envInt(key string, def int64) (int64, error) { return n, nil } +// envBool parses key as a boolean, or returns false when it is unset. A +// malformed value is an error for the same reason as envInt: "ture" must not +// read as the opposite of what the operator typed. +func envBool(key string) (bool, error) { + v := os.Getenv(key) + if v == "" { + return false, nil + } + b, err := strconv.ParseBool(v) + if err != nil { + return false, fmt.Errorf("%s: %q is not a boolean (want true or false)", key, v) + } + return b, nil +} + // envDuration parses key as a Go duration, or returns def when it is unset. A // malformed value is an error for the same reason as envInt — and a unitless // number ("90") is malformed, so it can never be read as a silent zero. diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 6be65d4..b99af96 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -28,16 +28,21 @@ func init() { } // clearEnv blanks every SWH_* variable so a test describes the whole -// environment rather than inheriting the developer's shell. +// environment rather than inheriting the developer's shell. The one thing it +// then sets is a token: resolution refuses an unauthenticated listener, and +// that is orthogonal to what most tests here assert. The auth tests clear it +// again to describe their own posture. func clearEnv(t *testing.T) { t.Helper() for _, k := range []string{ "SWH_LISTEN", "SWH_BACKEND", "SWH_DATABASE", "SWH_DATASET", "SWH_LOCATION", "SWH_DSN", "SWH_HOST", "SWH_PORT", "SWH_USER", "SWH_PASSWORD", "SWH_ACCOUNT", "SWH_CREDENTIALS_FILE", "SWH_MAX_QUERY_BYTES", "SWH_QUERY_TIMEOUT", + "SWH_AUTH_TOKEN", "SWH_ALLOW_ANONYMOUS", } { t.Setenv(k, "") } + t.Setenv("SWH_AUTH_TOKEN", "a-per-run-secret") } func TestSelfContainedBackendsNeedNoDatabase(t *testing.T) { @@ -138,3 +143,89 @@ func TestUnsetLimitsFallBackToDefaults(t *testing.T) { require.Zero(t, cfg.Backend.QueryTimeout) require.Zero(t, cfg.Backend.Port) } + +// TestFromEnvRefusesAnUnauthenticatedListener is the startup guard: with +// neither a token nor an explicit opt-out the server does not resolve, and the +// message names both ways out and the header a client must send. +func TestFromEnvRefusesAnUnauthenticatedListener(t *testing.T) { + clearEnv(t) + t.Setenv("SWH_AUTH_TOKEN", "") + + _, err := FromEnv() + require.Error(t, err) + for _, want := range []string{"SWH_AUTH_TOKEN", "x-codefly-token", "SWH_ALLOW_ANONYMOUS"} { + require.ErrorContains(t, err, want) + } +} + +// A whitespace-only token is not a token: it would install interceptors nobody +// can satisfy, or be taken for a configured credential while naming nothing. +func TestFromEnvTreatsABlankTokenAsAbsent(t *testing.T) { + clearEnv(t) + t.Setenv("SWH_AUTH_TOKEN", " \n") + + _, err := FromEnv() + require.ErrorContains(t, err, "SWH_AUTH_TOKEN is required") +} + +// A secret delivered through a file or a block scalar carries a trailing +// newline. The enforced token must be what a client can actually send, or every +// caller is refused while the server logs that auth is on. +func TestFromEnvTrimsTheToken(t *testing.T) { + clearEnv(t) + t.Setenv("SWH_AUTH_TOKEN", "s3cr3t\n") + + cfg, err := FromEnv() + require.NoError(t, err) + require.Equal(t, "s3cr3t", cfg.AuthToken) +} + +// A token together with the anonymous opt-out is the shape of a half-finished +// migration; picking either silently leaves the operator believing the other. +func TestFromEnvRejectsATokenWithTheAnonymousOptOut(t *testing.T) { + clearEnv(t) + t.Setenv("SWH_ALLOW_ANONYMOUS", "true") + + _, err := FromEnv() + require.ErrorContains(t, err, "mutually exclusive") +} + +func TestFromEnvAcceptsAnExplicitAnonymousOptOut(t *testing.T) { + clearEnv(t) + t.Setenv("SWH_AUTH_TOKEN", "") + t.Setenv("SWH_ALLOW_ANONYMOUS", "true") + + cfg, err := FromEnv() + require.NoError(t, err) + require.Empty(t, cfg.AuthToken) +} + +// A misspelled opt-out is an error, not a silent false: the refusal that +// followed would blame the missing token for a server whose operator did set the +// opt-out, and send them to provision a secret they chose not to have. +func TestFromEnvRejectsAMalformedAnonymousOptOut(t *testing.T) { + for _, value := range []string{"ture", "yes", "2"} { + t.Run(value, func(t *testing.T) { + clearEnv(t) + t.Setenv("SWH_AUTH_TOKEN", "") + t.Setenv("SWH_ALLOW_ANONYMOUS", value) + + _, err := FromEnv() + require.ErrorContains(t, err, "SWH_ALLOW_ANONYMOUS") + require.ErrorContains(t, err, "not a boolean") + }) + } +} + +// A configuration broken in more than one way reports the backend first: the +// operator fixing the token should not then meet an error that was there all +// along. +func TestFromEnvReportsTheBackendBeforeTheToken(t *testing.T) { + clearEnv(t) + t.Setenv("SWH_AUTH_TOKEN", "") + t.Setenv("SWH_BACKEND", "bigquery") + + _, err := FromEnv() + require.ErrorContains(t, err, `unknown backend kind "bigquery"`) + require.NotContains(t, err.Error(), "SWH_AUTH_TOKEN") +} From 03495af39f84ee717bbfe349a5729c6406753993 Mon Sep 17 00:00:00 2001 From: crisfiorini-oss <301773486+crisfiorini-oss@users.noreply.github.com> Date: Thu, 8 Oct 2026 17:01:42 -0400 Subject: [PATCH 03/10] feat(sbom): add the command that writes the gateway image's SBOM evidence (#12) Composing this gateway as a codefly service means shipping it as an image, and an image owes image-scope SBOM evidence under codefly-dev/core's docs/sbom.md: one CycloneDX inventory per shipped platform, bound to the digest scanned. This is the part of that evidence that refers to no image yet, so it lands first. internal/imageevidence holds the one list of shipped platforms and the subjects derived from it, so the agent's Builder.SBOM, the release command and the drift tests read the same description. For a pushed image the subjects come from core's sbom.ExpectedFromImageReference, which also refuses a reference that is not digest-pinned; for an image only the local daemon holds, the subject carries the daemon's image ID, the identity such a scan binds to. cmd/image-sbom scans each subject with core's sbom.Image and writes the documents and an index, failing the whole run when any platform fails rather than publishing partial coverage. The service identity it records defaults to the agent's publisher/name in agent.codefly.yaml, so no name is typed in a workflow. Reuse: the scanner, the subject derivation, the source selector and the CycloneDX encoder are imported from core. Core exports no helper that writes a scanned image's documents to disk or resolves a locally built image's ID, so Collect, WriteIndex and localImageID are adapted from codefly-dev/service-object-storage's internal/imageevidence (the same three functions), and the package comment says so. go.mod now requires codefly-dev/core v0.16.0, the release the later workflow pins resolve to; core requires grpc v1.83.2, so that moves with it. Verified: the command ran against service-object-storage's published multi-architecture image and wrote a CycloneDX document for each of linux/amd64 and linux/arm64, bound to the child manifest digests. Not verified: -local mode, which needs syft on PATH. Co-Authored-By: Claude Sonnet 5.5 --- cmd/image-sbom/main.go | 72 ++++++ go.mod | 45 +++- go.sum | 226 +++++++++++++++++-- internal/imageevidence/imageevidence.go | 191 ++++++++++++++++ internal/imageevidence/imageevidence_test.go | 84 +++++++ 5 files changed, 593 insertions(+), 25 deletions(-) create mode 100644 cmd/image-sbom/main.go create mode 100644 internal/imageevidence/imageevidence.go create mode 100644 internal/imageevidence/imageevidence_test.go diff --git a/cmd/image-sbom/main.go b/cmd/image-sbom/main.go new file mode 100644 index 0000000..be50af2 --- /dev/null +++ b/cmd/image-sbom/main.go @@ -0,0 +1,72 @@ +// Command image-sbom writes CycloneDX evidence for the images this service +// ships, through the same shared scanner Builder.SBOM uses. +// +// The release pipeline runs it so published evidence comes from the agent's own +// code path. Generating it with a separate scanner invocation produced +// documents that were not bound to the image digest and that drifted from the +// contract the agent implements. Run it from the repository root: the service +// identity it records is read from agent.codefly.yaml there. +package main + +import ( + "context" + "flag" + "fmt" + "os" + + "github.com/codefly-dev/core/resources" + "github.com/codefly-dev/core/shared" + + "github.com/codefly-dev/service-warehouse/internal/imageevidence" +) + +func main() { + image := flag.String("image", "", "image reference to inventory") + out := flag.String("out", "sbom", "directory to write CycloneDX documents into") + local := flag.Bool("local", false, "inventory an image held by the local Docker daemon rather than a registry image") + service := flag.String("service", "", "service identity recorded on each subject (default: the agent's publisher/name from agent.codefly.yaml)") + flag.Parse() + + if *image == "" { + fmt.Fprintln(os.Stderr, "image-sbom: -image is required") + os.Exit(2) + } + if *service == "" { + identity, err := agentIdentity() + if err != nil { + fmt.Fprintf(os.Stderr, "image-sbom: %v\n", err) + os.Exit(1) + } + *service = identity + } + + ctx := context.Background() + subjects, err := imageevidence.Subjects(ctx, *service, *image, *local) + if err != nil { + fmt.Fprintf(os.Stderr, "image-sbom: %v\n", err) + os.Exit(1) + } + documents, err := imageevidence.Collect(ctx, *out, subjects) + if err != nil { + fmt.Fprintf(os.Stderr, "image-sbom: %v\n", err) + os.Exit(1) + } + if err := imageevidence.WriteIndex(*out, documents); err != nil { + fmt.Fprintf(os.Stderr, "image-sbom: %v\n", err) + os.Exit(1) + } + for _, document := range documents { + fmt.Printf("%s %s %s\n", document.Platform, document.Digest, document.Path) + } +} + +// agentIdentity reads the identity a release has no workspace to take from: the +// publisher and name the manifest declares, which is also what a workspace +// resolves the agent by. +func agentIdentity() (string, error) { + agent, err := resources.LoadFromFs[resources.Agent](shared.Embed(os.DirFS("."))) + if err != nil { + return "", fmt.Errorf("read agent.codefly.yaml (run from the repository root, or pass -service): %w", err) + } + return agent.Publisher + "/" + agent.Name, nil +} diff --git a/go.mod b/go.mod index f277afc..e41790f 100644 --- a/go.mod +++ b/go.mod @@ -3,15 +3,50 @@ module github.com/codefly-dev/service-warehouse go 1.27.0 require ( + github.com/codefly-dev/core v0.16.0 github.com/stretchr/testify v1.12.1 - google.golang.org/grpc v1.83.1 + google.golang.org/grpc v1.83.2 google.golang.org/protobuf v1.36.12 ) require ( + buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.36.12-20260825204119-511051f7f437.2 // indirect + buf.build/go/protovalidate v1.4.0 // indirect + cel.dev/cel-go v0.32.0 // indirect + cel.dev/expr v0.25.3 // indirect + github.com/Masterminds/semver v1.5.0 // indirect + github.com/Masterminds/semver/v3 v3.5.0 // indirect + github.com/antlr4-go/antlr/v4 v4.13.1 // indirect + github.com/go-openapi/analysis v1.0.0 // indirect + github.com/go-openapi/errors v0.22.8 // indirect + github.com/go-openapi/jsonpointer v1.0.1 // indirect + github.com/go-openapi/jsonreference v1.0.2 // indirect + github.com/go-openapi/loads v0.25.3 // indirect + github.com/go-openapi/spec v1.0.1 // indirect + github.com/go-openapi/strfmt v0.27.2 // indirect + github.com/go-openapi/swag/conv v0.29.1 // indirect + github.com/go-openapi/swag/jsonutils v0.29.1 // indirect + github.com/go-openapi/swag/loading v0.29.1 // indirect + github.com/go-openapi/swag/mangling v0.29.1 // indirect + github.com/go-openapi/swag/pools v0.29.1 // indirect + github.com/go-openapi/swag/stringutils v0.29.1 // indirect + github.com/go-openapi/swag/typeutils v0.29.1 // indirect + github.com/go-openapi/swag/yamlutils v0.29.1 // indirect + github.com/go-viper/mapstructure/v2 v2.5.0 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/hashicorp/errwrap v1.1.0 // indirect + github.com/hashicorp/go-multierror v1.1.1 // indirect + github.com/mitchellh/mapstructure v1.5.0 // indirect + github.com/oklog/ulid/v2 v2.1.2 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/yoheimuta/go-protoparser/v4 v4.14.2 // indirect go.yaml.in/yaml/v3 v3.0.5 // indirect - golang.org/x/net v0.55.0 // indirect - golang.org/x/sys v0.45.0 // indirect - golang.org/x/text v0.37.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa // indirect + golang.org/x/exp v0.0.0-20260820142414-ca536658362e // indirect + golang.org/x/net v0.59.0 // indirect + golang.org/x/sys v0.48.0 // indirect + golang.org/x/term v0.46.0 // indirect + golang.org/x/text v0.42.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20260819154853-08b0e4226688 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260825221802-da73d73af1c5 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index d46fd74..6edcaa5 100644 --- a/go.sum +++ b/go.sum @@ -1,42 +1,228 @@ +buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.36.12-20260825204119-511051f7f437.2 h1:NbnlmV26O7oZ1iM5tsCI+GEx+3ZSdrhvnKQ/eWSrLiY= +buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.36.12-20260825204119-511051f7f437.2/go.mod h1:TCt1lluMFnctISJXvkIQ4x3ABrPuUKCWKyjKdkJNBpw= +buf.build/go/protovalidate v1.4.0 h1:UjLrYbt5VX7+TMOs2+pG5FhZhIG1mSfK4EIopbb4LcM= +buf.build/go/protovalidate v1.4.0/go.mod h1:8vJfzNT6NIG2qm3uFsJDXMlRmG+bQJzbcIn1Aa0vPGs= +cel.dev/cel-go v0.32.0 h1:irvpFKr5EuGPyxeME03ERh0rii1TX+BDAnB9eL3IvNk= +cel.dev/cel-go v0.32.0/go.mod h1:DnVip7tpJSsgZymwfT+m1tnEVy3ivAjSMXPx12YrMkU= +cel.dev/expr v0.25.3 h1:A2jO8jwOugrrovveCWfj0KEZOfqiLgAcwjpHPhzIGw0= +cel.dev/expr v0.25.3/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4= +github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3QEww= +github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= +github.com/Masterminds/semver/v3 v3.5.0 h1:kQceYJfbupGfZOKZQg0kou0DgAKhzDg2NZPAwZ/2OOE= +github.com/Masterminds/semver/v3 v3.5.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= +github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= +github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= +github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow= +github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4= +github.com/antlr4-go/antlr/v4 v4.13.1 h1:SqQKkuVZ+zWkMMNkjy5FZe5mr5WURWnlpmOuzYWrPrQ= +github.com/antlr4-go/antlr/v4 v4.13.1/go.mod h1:GKmUxMtwp6ZgGwZSva4eWPC5mS6vUAmOABFgjdkM7Nw= +github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ= +github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= +github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= +github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= +github.com/brianvoe/gofakeit/v6 v6.28.0 h1:Xib46XXuQfmlLS2EXRuJpqcw8St6qSZz75OUo0tgAW4= +github.com/brianvoe/gofakeit/v6 v6.28.0/go.mod h1:Xj58BMSnFqcn/fAQeSK+/PLtC5kSb7FJIq4JyGa8vEs= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= -github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/cheggaaa/pb/v3 v3.2.1 h1:aprZbFRG+B7+ug76S8QZ6Y1PW168UHzOmbC3wa+aU6I= +github.com/cheggaaa/pb/v3 v3.2.1/go.mod h1:U9hSVxoKqJrZIE3PkFG1xXXNaK/Ilzg+EF/scpXzEdw= +github.com/clipperhouse/uax29/v2 v2.7.0 h1:+gs4oBZ2gPfVrKPthwbMzWZDaAFPGYK72F0NJv2v7Vk= +github.com/clipperhouse/uax29/v2 v2.7.0/go.mod h1:EFJ2TJMRUaplDxHKj1qAEhCtQPW2tJSwu5BF98AuoVM= +github.com/codefly-dev/core v0.16.0 h1:iJQ9SteSBRaGs+jeD/DqgKMsukJfTKvnqcYTjsOBjDs= +github.com/codefly-dev/core v0.16.0/go.mod h1:rp4YZkCJEOufliWQAgK3sv8YlO7Xcva7+cN7hIlAR0I= +github.com/codefly-dev/gortk v0.2.0 h1:7bOlS5valYz2zil+fZctQNcPCYBcPj86abcw9N8h1hQ= +github.com/codefly-dev/gortk v0.2.0/go.mod h1:dDWUMFgAP063OCGhTEpV7FFUj9+zTcxxO/nV80VKEwg= +github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI= +github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M= +github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE= +github.com/containerd/errdefs/pkg v0.3.0/go.mod h1:NJw6s9HwNuRhnjJhM7pylWwMyAkmCQvQ4GpJHEqRLVk= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= +github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= +github.com/docker/docker v28.5.2+incompatible h1:DBX0Y0zAjZbSrm1uzOkdr1onVghKaftjlSWt4AFexzM= +github.com/docker/docker v28.5.2+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/go-connections v0.8.1 h1:JibmG5hULs5qXSr/cp/w3Pw5fZuStt4MOHMUExb29/M= +github.com/docker/go-connections v0.8.1/go.mod h1:no1qkHdjq7kLMGUXYAduOhYPSJxxvgWBh7ogVvptn3Q= +github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= +github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/fatih/color v1.19.0 h1:Zp3PiM21/9Ld6FzSKyL5c/BULoe/ONr9KlbYVOfG8+w= +github.com/fatih/color v1.19.0/go.mod h1:zNk67I0ZUT1bEGsSGyCZYZNrHuTkJJB+r6Q9VuMi0LE= +github.com/felixge/httpsnoop v1.1.0 h1:3YtUj32ZZkqZtt3sZZsClsymw/QDuVfpNhoA31zeORc= +github.com/felixge/httpsnoop v1.1.0/go.mod h1:Zqxgdd+1Rkcz8euOqdr7lqgCRJztwr5hp9vDSi5UZCE= +github.com/fsnotify/fsnotify v1.10.1 h1:b0/UzAf9yR5rhf3RPm9gf3ehBPpf0oZKIjtpKrx59Ho= +github.com/fsnotify/fsnotify v1.10.1/go.mod h1:TLheqan6HD6GBK6PrDWyDPBaEV8LspOxvPSjC+bVfgo= +github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= +github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= +github.com/go-logr/logr v1.4.4 h1:tG4xh9yMsRCAiodLVTxyrkzSZ9+o0L1Kg/+cPVcbP/8= +github.com/go-logr/logr v1.4.4/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE= +github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78= +github.com/go-openapi/analysis v1.0.0 h1:sNvbAGCJqUTqIAodr9IVqJMmuZas3YS9ms1dGK9yiJ4= +github.com/go-openapi/analysis v1.0.0/go.mod h1:NhYjJ57fnE+bcE7UwrJyMkhWA3Dfz7TdiBfTVAnos4Y= +github.com/go-openapi/errors v0.22.8 h1:oP7sW7TWc3wFFjrzzj0nI83H2qMBkNjNfSd+XRejk/I= +github.com/go-openapi/errors v0.22.8/go.mod h1:BuUoHcYrU6E7V9gfj1I5wLQqgtIHnup/alXZ8KdgQ0w= +github.com/go-openapi/jsonpointer v1.0.1 h1:2KxywRmNwJkT/FMBa3iRNHEaAxSJvjqoufQZy3au1Mg= +github.com/go-openapi/jsonpointer v1.0.1/go.mod h1:wI7ZYsFmbIi9nBXOZqgDaS/bqOchRGZjqxFli7FBYxY= +github.com/go-openapi/jsonreference v1.0.2 h1:oS4et8FOf3p3UQxEo4Xt0esijmBUM+F259Xl72OSZsc= +github.com/go-openapi/jsonreference v1.0.2/go.mod h1:TbUNSOo+fcorZjFaNoiSDSoaNnnZtqJtLGR1PuvE/Cs= +github.com/go-openapi/loads v0.25.3 h1:V+jKy/thXWdLJUuYC8sZX2dICyAa8M3DokF70jj34P0= +github.com/go-openapi/loads v0.25.3/go.mod h1:LgLyCSOLBL2Qnj0Ps1oo2YH8jZoEKsMeVZCc+ALIxFo= +github.com/go-openapi/spec v1.0.1 h1:lj2vdGpNDcVgwRc6qXdw6qt/KQpCtSa9tnUH6vpDPDk= +github.com/go-openapi/spec v1.0.1/go.mod h1:M//GWQGtDUAjnP37gE6fInLgaczB+FatoipV3H1fYw8= +github.com/go-openapi/strfmt v0.27.2 h1:SG32SlbwNy92s0KJiVxt2joJeFdqIYHvwrA0OU6HqzQ= +github.com/go-openapi/strfmt v0.27.2/go.mod h1:M4CKsMO0Fb8qR10+1Ra75wCKNNquy+Vj+4LWZrhTo2E= +github.com/go-openapi/swag v0.28.0 h1:xkgbOSKj6DZziNpyqRRAOt3GJGtgjgsd2RoyT30VWuw= +github.com/go-openapi/swag v0.28.0/go.mod h1:4qYnT3Cqr1p1VknOdPo70evN4rgQnAg6jwApHyxSGIg= +github.com/go-openapi/swag/cmdutils v0.28.0 h1:7TOeNtkYru1SG8Y34tDh9WBbLsMqGnptuxWiHREPZ4Q= +github.com/go-openapi/swag/cmdutils v0.28.0/go.mod h1:Sm1MVFMkF6guJJ+pQqHnQA3N0j9qALV3NxzDSv6bETM= +github.com/go-openapi/swag/conv v0.29.1 h1:AC4Eh/5c/eUDOUCzzsRC9ghmFgOSBHeRMGIngY0ZUGA= +github.com/go-openapi/swag/conv v0.29.1/go.mod h1:S1X7/ZrBEZOC0Wc8AGxjbcGS92l3WEjA7aPtpl+RaqM= +github.com/go-openapi/swag/fileutils v0.28.0 h1:Z04XWQD7R8Eq+7GnOrjovBxPPmZzsS4gt2H2GPGIViU= +github.com/go-openapi/swag/fileutils v0.28.0/go.mod h1:VvJFZLTZS0AI854gEQz5tk7dBESdLjiNUMSZ/th2ry8= +github.com/go-openapi/swag/jsonutils v0.29.1 h1:AFCxs0eQZ24/QyfhVHM2t49rMz7Vv3XCsZQI6yrNy+c= +github.com/go-openapi/swag/jsonutils v0.29.1/go.mod h1:u3+sCfJpttDpcmS5kpm0yxL6GK0eWgODsx8Yw8fcqNM= +github.com/go-openapi/swag/jsonutils/fixtures_test v0.29.1 h1:BiiXE31Bx9SfpsMmOQj5KYpUhTZBpLVriVhJDuLuY2o= +github.com/go-openapi/swag/jsonutils/fixtures_test v0.29.1/go.mod h1:julgTUKZ9/D0j6O7GKajmRs+812FWxQg/mMpGunWSjg= +github.com/go-openapi/swag/loading v0.29.1 h1:FCv5fG8UhTdDJa2R7w+5O9Ekpcbw7tt0nFWvmDKGBjc= +github.com/go-openapi/swag/loading v0.29.1/go.mod h1:N0ESuem4p2oedKal8EJhciqnJ9Q9Wmt83L1CRB3Fouw= +github.com/go-openapi/swag/mangling v0.29.1 h1:lHALtvYCdxVnRl4GrHmFPwfBTZYIObqdGNSKyu/8D6I= +github.com/go-openapi/swag/mangling v0.29.1/go.mod h1:SAop9pB7PUjQ/CGCNf/JmCKTRK+GDO+RqE9UHqC/N6s= +github.com/go-openapi/swag/netutils v0.28.0 h1:YXN6TALEi2pzts8/8GNm6T61HTAZsieukGZidap989k= +github.com/go-openapi/swag/netutils v0.28.0/go.mod h1:J+WYyFMLtvtCGqa6jLv+YNUmIKI3ZRQRrvfNDMoQoEQ= +github.com/go-openapi/swag/pools v0.29.1 h1:NRogYxdEW9SjRM4mkAOji9iefO4MRXq3p/ZJcoQbUKg= +github.com/go-openapi/swag/pools v0.29.1/go.mod h1:leDcaghjkRAhCuCRv9NfJU5f0mjoU3cT/XZObhMk3pc= +github.com/go-openapi/swag/stringutils v0.29.1 h1:1ykunK7iJQk1uOO7+oUH1ukbsK85fFCOiCFMOVSY+F0= +github.com/go-openapi/swag/stringutils v0.29.1/go.mod h1:7fSqZ+z8Qc0tOfAAK0jVa5qFGrnIlRi6n7NeGGrr1vc= +github.com/go-openapi/swag/typeutils v0.29.1 h1:Nzv9nhnlLCRBPQqfOX+7lB6Guju370or8StT+lIOf6M= +github.com/go-openapi/swag/typeutils v0.29.1/go.mod h1:hxpgDZJVBkBsi/d3MIUosafoFdE5exaQRmVp0zwu3YE= +github.com/go-openapi/swag/yamlutils v0.29.1 h1:69w3tsBajm7MR/fejLy7HD/3J68Ys1SeeZMEzZ3w2sk= +github.com/go-openapi/swag/yamlutils v0.29.1/go.mod h1:rgsp3vT/QdWzKwn43CigDwjOGIenPyTZMKnxEM8jZOA= +github.com/go-openapi/testify/enable/yaml/v2 v2.7.0 h1:wPW6YRgx3+SID1yUy/Xwa17L8kFEaEKod2VRbJDZNUs= +github.com/go-openapi/testify/enable/yaml/v2 v2.7.0/go.mod h1:mI1M88etYbc3PhgHsWQK2kwvNwW5aGFqMPbmib+SGIs= +github.com/go-openapi/testify/v2 v2.7.0 h1:bycOreEj6wfBvijg3YFogZ/sFjTCDmQnwSodSzHa3X8= +github.com/go-openapi/testify/v2 v2.7.0/go.mod h1:SgsVHtfooshd0tublTtJ50FPKhujf47YRqauXXOUxfw= +github.com/go-viper/mapstructure/v2 v2.5.0 h1:vM5IJoUAy3d7zRSVtIwQgBj7BiWtMPfmPEgAXnvj1Ro= +github.com/go-viper/mapstructure/v2 v2.5.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= +github.com/gofrs/flock v0.13.1 h1:jjREztyBeSKBZYAC+mgc1laB+xsgy4kYMf3FbKF2UBo= +github.com/gofrs/flock v0.13.1/go.mod h1:sf4BFiHwnvgxa25DlQoDqXQnwRMEOwqxRq37P6MzzmE= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/gnostic-models v0.6.9 h1:MU/8wDLif2qCXZmzncUQ/BOfxWfthHi63KqpoNbWqVw= +github.com/google/gnostic-models v0.6.9/go.mod h1:CiWsm0s6BSQd1hRn8/QmxqB6BesYcbSZxsz9b0KuDBw= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/go-github/v89 v89.0.0 h1:35bEK5XoEcF3PZrlVbl9XN63f5BcJRA/UGkxeC9xPg0= +github.com/google/go-github/v89 v89.0.0/go.mod h1:QLcbU0ipeAqQuR5KSg8c2lql4Qk1EwJ2dWz/0rP4Nho= +github.com/google/go-querystring v1.2.0 h1:yhqkPbu2/OH+V9BfpCVPZkNmUXhb2gBxJArfhIxNtP0= +github.com/google/go-querystring v1.2.0/go.mod h1:8IFJqpSRITyJ8QhQ13bmbeMBDfmeEJZD5A0egEOmkqU= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= +github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= +github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= +github.com/lufia/plan9stats v0.0.0-20260330125221-c963978e514e h1:Q6MvJtQK/iRcRtzAscm/zF23XxJlbECiGPyRicsX+Ak= +github.com/lufia/plan9stats v0.0.0-20260330125221-c963978e514e/go.mod h1:autxFIvghDt3jPTLoqZ9OZ7s9qTGNAWmYCjVFWPX/zg= +github.com/mattn/go-colorable v0.1.15 h1:+u9SLTRGnXv73cEsnsmoZBom+dMU88B2M0aDcWy0/jY= +github.com/mattn/go-colorable v0.1.15/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= +github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsReI= +github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A= +github.com/mattn/go-runewidth v0.0.27 h1:Feg/Oou5zI/wnpgDF6omIU0OokC9GxLC/WRknhVlIR0= +github.com/mattn/go-runewidth v0.0.27/go.mod h1:3qAiGCV4Koz/yuveO58qUefmUTRm8r0IGEXZ9jeHp/8= +github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= +github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= +github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= +github.com/moby/patternmatcher v0.6.1 h1:qlhtafmr6kgMIJjKJMDmMWq7WLkKIo23hsrpR3x084U= +github.com/moby/patternmatcher v0.6.1/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc= +github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 h1:n6/2gBQ3RWajuToeY6ZtZTIKv2v7ThUy5KKusIT0yc0= +github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00/go.mod h1:Pm3mSP3c5uWn86xMLZ5Sa7JB9GsEZySvHYXCTK4E9q4= +github.com/oklog/ulid/v2 v2.1.2 h1:IEclFb9JNvzYA6MW2SCxbLzcHTVsfqm3PrqGQJH5zec= +github.com/oklog/ulid/v2 v2.1.2/go.mod h1:rcEKHmBBKfef9DhnvX7y1HZBYxjXb0cP5ExxNsTT1QQ= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040= +github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M= +github.com/pborman/getopt v0.0.0-20170112200414-7148bc3a4c30/go.mod h1:85jBQOZwpVEaDAr341tbn15RS4fCAsIst0qp7i8ex1o= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 h1:o4JXh1EVt9k/+g42oCprj/FisM4qX9L3sZB3upGN2ZU= +github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= +github.com/rodaine/protogofakeit v0.1.1 h1:ZKouljuRM3A+TArppfBqnH8tGZHOwM/pjvtXe9DaXH8= +github.com/rodaine/protogofakeit v0.1.1/go.mod h1:pXn/AstBYMaSfc1/RqH3N82pBuxtWgejz1AlYpY1mI0= +github.com/santhosh-tekuri/jsonschema/v6 v6.0.3 h1:1EYB5IzjZawrrnELUi78f9fPu57HuXjmddZPjrls/28= +github.com/santhosh-tekuri/jsonschema/v6 v6.0.3/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU= +github.com/shirou/gopsutil/v3 v3.24.5 h1:i0t8kL+kQTvpAYToeuiVk3TgDeKOFioZO3Ztz/iZ9pI= +github.com/shirou/gopsutil/v3 v3.24.5/go.mod h1:bsoOS1aStSs9ErQ1WWfxllSeS1K5D+U30r2NfcubMVk= +github.com/shoenig/go-m1cpu v0.2.1 h1:yqRB4fvOge2+FyRXFkXqsyMoqPazv14Yyy+iyccT2E4= +github.com/shoenig/go-m1cpu v0.2.1/go.mod h1:KkDOw6m3ZJQAPHbrzkZki4hnx+pDRR1Lo+ldA56wD5w= github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE= github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg= +github.com/tklauser/go-sysconf v0.4.0 h1:7H0uAN+7RkwWRaxhYXDLqa5V3LPrJeV8wmD9dRUgPQU= +github.com/tklauser/go-sysconf v0.4.0/go.mod h1:8mTNWyog7H+MpKijp4VmKJAd2bbYQ2zuUwkYRbUArPI= +github.com/tklauser/numcpus v0.12.0 h1:NR85qdvHA9pFse3x3weVZ0r0ST8R6l5RHbZrlRaqob4= +github.com/tklauser/numcpus v0.12.0/go.mod h1:ABHeXzJnr/qqwguhClkZKT1/8VABcYrsyUiUGobwWJg= +github.com/xlab/treeprint v1.2.0 h1:HzHnuAF1plUN2zGlAFHbSQP2qJ0ZAD3XF5XD7OesXRQ= +github.com/xlab/treeprint v1.2.0/go.mod h1:gj5Gd3gPdKtR1ikdDK6fnFLdmIS0X30kTTuNd/WEJu0= +github.com/yoheimuta/go-protoparser/v4 v4.14.2 h1:/P/LlX1CF9NaTWEltGcIZVvNlPbhABuAnBtAWpb3+74= +github.com/yoheimuta/go-protoparser/v4 v4.14.2/go.mod h1:AHNNnSWnb0UoL4QgHPiOAg2BniQceFscPI5X/BZNHl8= +github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= +github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= -go.opentelemetry.io/otel v1.44.0 h1:JjwHmHpA4iZ3wBxluu2fbbE7j4kqlE8jXyAyPXH7HqU= -go.opentelemetry.io/otel v1.44.0/go.mod h1:BMgjTHL9WPRlRjL2oZCBTL4whCGtXch2H4BhOPIAyYc= -go.opentelemetry.io/otel/metric v1.44.0 h1:1w0gILTcHdr3YI+ixLyjemwrVnsMURbTZFrSYCdDdmc= -go.opentelemetry.io/otel/metric v1.44.0/go.mod h1:8O7hanEPBNgEMmybD3s2VBKcgWOCsA6tzHBPODAiquo= -go.opentelemetry.io/otel/sdk v1.44.0 h1:nHYwb9lK+fJPU/dnT6s7W7Z8itMWyqrnVfbheVYrZ58= -go.opentelemetry.io/otel/sdk v1.44.0/go.mod h1:Osuydd3Se74nqjAKxid74N5eC+jfEqfTegHRnq58oK0= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.71.0 h1:B2h3uqicet1CT2N5TOFhS+Gq++9i0/CLmaxvhmhtP5s= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.71.0/go.mod h1:dylvB+ZiiwMvsDij9O84Uy7SijLgHMX4mbkncds+4Sw= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.70.0 h1:LMuyCAyfalSjDyjdC65nK6N0zoTT63+E/u95X0JovZI= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.70.0/go.mod h1:085m8qbm4hgc8rZWGDEa4vmyyo2c3nPxUslYUKUIU04= +go.opentelemetry.io/otel v1.46.0 h1:FHt5/CDyVxi/8IM1CH7VE/rRgq3kLHa2mSTVMO8AWyc= +go.opentelemetry.io/otel v1.46.0/go.mod h1:Gj3SEScelsNC45tp4nSxRYlS+f5iez7W8XPMCt905kE= +go.opentelemetry.io/otel/metric v1.46.0 h1:yBnkXvgV7AXFILZc5K6IZe/CBFF3OS7BJ8ov6/lj0K8= +go.opentelemetry.io/otel/metric v1.46.0/go.mod h1:iPmdWqifKUdzziPkvvzIJXITl56fQx2mGM/DHLB3/2o= +go.opentelemetry.io/otel/sdk v1.46.0 h1:h5CNQQjEbuQXY/JfZtgt3i7HVFV3aHPO2OAwO2eTYPI= +go.opentelemetry.io/otel/sdk v1.46.0/go.mod h1:GAERFXFt5SYCEB+YiKUbMBeza6UaDH7GmGOZEfh2gSM= go.opentelemetry.io/otel/sdk/metric v1.44.0 h1:3LlKgI+VjbVsjNRFZJZAJ30WjXC5VkNRks6si09iEfI= go.opentelemetry.io/otel/sdk/metric v1.44.0/go.mod h1:5B5pMARnXxKhltooO4xUuCBorl65a4EpnTalObqOigA= -go.opentelemetry.io/otel/trace v1.44.0 h1:jxF5CsGYCe74MCRx2X4g7WsY/VBKRqqpNvXlX/6gtIk= -go.opentelemetry.io/otel/trace v1.44.0/go.mod h1:oLl1jrMQAVo6v3GAggN+1VH9VIz9iUSvW53sW1Q8PIE= +go.opentelemetry.io/otel/trace v1.46.0 h1:OULy7ccdJnZtJ0UDYFOIGaCmiWzJ8Vi2G/Rsu60qs1c= +go.opentelemetry.io/otel/trace v1.46.0/go.mod h1:J7GAXweO77XSFkB/rmAqk9D6ihszhFjLU+d9WuUxDLI= +go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI= +go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU= go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= -golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8= -golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww= -golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY= -golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= -golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc= -golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38= +golang.org/x/exp v0.0.0-20260820142414-ca536658362e h1:01Ju2A/fZKkci4zqx0eZxw//DnRYOnBiGJG14hFBhO8= +golang.org/x/exp v0.0.0-20260820142414-ca536658362e/go.mod h1:zeBbvyFKDaLwa7CH/zI8KXt7gTl14SF7sO08Pl5jBCM= +golang.org/x/net v0.59.0 h1:5zfYln+w5XCxwrnMMJPufRgNoXEaGxl0wo5GqPXyues= +golang.org/x/net v0.59.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg= +golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= +golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= +golang.org/x/term v0.46.0 h1:3+OXuTbaKDgwk8jTi3aSLHRlmWqHEUDUtxnbFigO4YE= +golang.org/x/term v0.46.0/go.mod h1:+K02xbkittuwc0Am4abfA3Fc+XRGXkvBXNO88NCXPoc= +golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI= +golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E= gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa h1:mZHHdPZl0dbGHCflZgAq/Q468DWVFcU2whhB2KAo8fk= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= -google.golang.org/grpc v1.83.1 h1:HIO0+BEtBP6soyqvqC8sNUjZ7bTs+0hFQuFF+RAy++Y= -google.golang.org/grpc v1.83.1/go.mod h1:kDyl6SKsiHKt0uylY5gtn5cEjkrIOhQOGDgIc4JGwzQ= +google.golang.org/genproto/googleapis/api v0.0.0-20260819154853-08b0e4226688 h1:ax2KzoSRIZU/M0cIxri3pKxy99vniH1PVxWC6si/eZI= +google.golang.org/genproto/googleapis/api v0.0.0-20260819154853-08b0e4226688/go.mod h1:1RJ9BQGyNdZwkGc1eTqkErfRZ6RJyYPHZo73BZ1vQqI= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260825221802-da73d73af1c5 h1:1VUiZAXyC+zmiFYi+WLtBzr68Cj8wOofHjjrA/kkizc= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260825221802-da73d73af1c5/go.mod h1:DjtHYE8FKJLivXcBEjGwndXfIC23G0VpXiXKqG179uA= +google.golang.org/grpc v1.83.2 h1:EManeRomTObA0BU7I8vXgg/78uE5MJ9M8B39EX2WscU= +google.golang.org/grpc v1.83.2/go.mod h1:YPI1hK3kDked6iHvgX3tR0y+nX/qpMFKhPgFsokw1S8= google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc= google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/evanphx/json-patch.v4 v4.13.0 h1:czT3CmqEaQ1aanPc5SdlgQrrEIb8w/wwCvWWnfEbYzo= +gopkg.in/evanphx/json-patch.v4 v4.13.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +k8s.io/kube-openapi v0.0.0-20241212222426-2c72e554b1e7 h1:hcha5B1kVACrLujCKLbr8XWMxCxzQx42DY8QKYJrDLg= +k8s.io/kube-openapi v0.0.0-20241212222426-2c72e554b1e7/go.mod h1:GewRfANuJ70iYzvn+i4lezLDAFzvjxZYK1gn1lWcfas= +sigs.k8s.io/kustomize/api v0.21.1 h1:lzqbzvz2CSvsjIUZUBNFKtIMsEw7hVLJp0JeSIVmuJs= +sigs.k8s.io/kustomize/api v0.21.1/go.mod h1:f3wkKByTrgpgltLgySCntrYoq5d3q7aaxveSagwTlwI= +sigs.k8s.io/kustomize/kyaml v0.21.1 h1:IVlbmhC076nf6foyL6Taw4BkrLuEsXUXNpsE+ScX7fI= +sigs.k8s.io/kustomize/kyaml v0.21.1/go.mod h1:hmxADesM3yUN2vbA5z1/YTBnzLJ1dajdqpQonwBL1FQ= +sigs.k8s.io/yaml v1.5.0 h1:M10b2U7aEUY6hRtU870n2VTPgR5RZiL/I6Lcc2F4NUQ= +sigs.k8s.io/yaml v1.5.0/go.mod h1:wZs27Rbxoai4C0f8/9urLZtZtF3avA3gKvGyPdDqTO4= diff --git a/internal/imageevidence/imageevidence.go b/internal/imageevidence/imageevidence.go new file mode 100644 index 0000000..2981f7b --- /dev/null +++ b/internal/imageevidence/imageevidence.go @@ -0,0 +1,191 @@ +// Package imageevidence holds the one description of the images this service +// ships: the platforms the release pipeline publishes, the subjects that +// describe them, and the CycloneDX documents inventorying them. +// +// The agent's Builder.SBOM, the release command (cmd/image-sbom), and the drift +// test all read it from here. A second copy of the platform list is how a +// platform gets added to the pipeline and silently shipped with no evidence, so +// there is only one. +// +// What core exports is imported, not redeclared: the scanner (sbom.Image), the +// subject derivation for a pushed image (sbom.ExpectedFromImageReference), the +// scanner source selector and the CycloneDX encoder. Core exports no helper +// that writes a scanned image's documents to disk or resolves a locally built +// image's ID, so Collect, WriteIndex and localImageID below are this +// package's, adapted from codefly-dev/service-object-storage's +// internal/imageevidence (the same three functions); a fix to one belongs in +// the other until core grows an exported equivalent. +package imageevidence + +import ( + "context" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + + "github.com/codefly-dev/core/agents/services/sbom" + builderv0 "github.com/codefly-dev/core/generated/go/codefly/services/builder/v0" +) + +// Platforms are the platforms the release pipeline publishes the gateway image +// for, and therefore the platforms evidence has to cover: a multi-architecture +// image is not inventoried by scanning one of its children. It is asserted +// against .github/workflows/publish-gateway-image.yml. +var Platforms = []string{"linux/amd64", "linux/arm64"} + +// Role is the gateway's purpose within the service, carried on every subject. +const Role = "runtime" + +// Subjects describes the images the service ships. A published image is +// multi-architecture and contributes one subject per shipped platform; core +// derives them from the digest-pinned reference, which is what binds each scan +// to the image that was built and which core refuses to accept as a bare tag. A +// local image was built and never pushed, so the daemon holds exactly one +// platform of it and the subject names none: the platform is read back from the +// image rather than asserted by the caller. +// +// A local subject carries the image ID instead, because the daemon resolves +// whatever reference it is handed to that ID: a pin sitting only in the +// reference is not the identity such a scan binds evidence to. +func Subjects(ctx context.Context, service, reference string, local bool) ([]*builderv0.ImageSubject, error) { + if !local { + return sbom.ExpectedFromImageReference(sbom.PublishedImage{ + Service: service, + Role: Role, + Reference: reference, + Platforms: Platforms, + }) + } + id, err := localImageID(ctx, reference) + if err != nil { + return nil, err + } + return []*builderv0.ImageSubject{{ + Reference: reference, + Digest: id, + Role: Role, + Service: service, + Source: builderv0.ImageSourceKind_IMAGE_SOURCE_KIND_DOCKER_DAEMON, + }}, nil +} + +// localImageID reports the identity the Docker daemon holds an image under. +func localImageID(ctx context.Context, reference string) (string, error) { + command := exec.CommandContext(ctx, "docker", "image", "inspect", reference, "--format", "{{.Id}}") + var stderr strings.Builder + command.Stderr = &stderr + out, err := command.Output() + if err != nil { + // An absent image, an absent daemon and an absent docker are different + // things to go fix, and only docker's own message tells them apart. + return "", fmt.Errorf("resolve local image %s: %w: %s", reference, err, strings.TrimSpace(stderr.String())) + } + id := strings.TrimSpace(string(out)) + if !strings.HasPrefix(id, "sha256:") { + return "", fmt.Errorf("local image %s reports id %q, which is not a sha256 digest", reference, id) + } + return id, nil +} + +// Document is one platform's inventory, written to disk and named by the digest +// it is bound to. +type Document struct { + Reference string + Digest string + Platform string + Path string + SHA256 string +} + +// Collect inventories every subject through the shared scanner and writes one +// CycloneDX document each. The document carries the scanned digest in its own +// root component, so an exported artifact still names the image it describes +// rather than relying on the filename to say so. +// +// A single failed scan fails the whole call: publishing evidence for some +// platforms while silently omitting others is the false coverage claim this +// evidence exists to prevent. +func Collect(ctx context.Context, dir string, subjects []*builderv0.ImageSubject) ([]Document, error) { + if len(subjects) == 0 { + return nil, fmt.Errorf("no image subjects to inventory") + } + if err := os.MkdirAll(dir, 0o755); err != nil { + return nil, fmt.Errorf("create evidence directory: %w", err) + } + documents := make([]Document, 0, len(subjects)) + for _, subject := range subjects { + result, err := sbom.Image(ctx, sbom.ImageRequest{ + Reference: subject.GetReference(), + Platform: subject.GetPlatform(), + Source: sbom.SourceOf(subject), + }) + if err != nil { + return nil, fmt.Errorf("inventory %s: %w", describe(subject), err) + } + encoded, err := sbom.MarshalCycloneDXJSON(result.Bom) + if err != nil { + return nil, fmt.Errorf("encode %s: %w", describe(subject), err) + } + path := filepath.Join(dir, documentName(subject.GetReference(), result.Platform, result.Digest)) + if err := os.WriteFile(path, encoded, 0o644); err != nil { + return nil, fmt.Errorf("write %s: %w", path, err) + } + documents = append(documents, Document{ + Reference: subject.GetReference(), + Digest: result.Digest, + Platform: result.Platform, + Path: path, + SHA256: result.SHA256, + }) + } + return documents, nil +} + +// WriteIndex records which image and platform each document covers, so a +// release asset is readable without opening every document. +func WriteIndex(dir string, documents []Document) error { + var index strings.Builder + for _, document := range documents { + fmt.Fprintf(&index, "%s %s@%s %s %s\n", + document.Platform, referenceName(document.Reference), document.Digest, + filepath.Base(document.Path), document.SHA256) + } + return os.WriteFile(filepath.Join(dir, "index.txt"), []byte(index.String()), 0o644) +} + +// documentName names a document after the repository it inventories, the +// platform and the digest it is bound to, none of which is typed here. +func documentName(reference, platform, digest string) string { + repository := referenceName(reference) + return fmt.Sprintf("%s-%s-%s.cdx.json", + repository[strings.LastIndex(repository, "/")+1:], + strings.ReplaceAll(platform, "/", "-"), + strings.TrimPrefix(digest, "sha256:")) +} + +func describe(subject *builderv0.ImageSubject) string { + if platform := subject.GetPlatform(); platform != "" { + return subject.GetReference() + " on " + platform + } + return subject.GetReference() +} + +// referenceName strips any tag or digest so the index names the repository and +// the digest actually scanned, rather than the tag that was asked for. +func referenceName(reference string) string { + if base, _, found := strings.Cut(reference, "@"); found { + reference = base + } + if slash := strings.LastIndex(reference, "/"); slash >= 0 { + if colon := strings.LastIndex(reference[slash:], ":"); colon >= 0 { + return reference[:slash+colon] + } + return reference + } + if colon := strings.LastIndex(reference, ":"); colon >= 0 { + return reference[:colon] + } + return reference +} diff --git a/internal/imageevidence/imageevidence_test.go b/internal/imageevidence/imageevidence_test.go new file mode 100644 index 0000000..c1f191f --- /dev/null +++ b/internal/imageevidence/imageevidence_test.go @@ -0,0 +1,84 @@ +package imageevidence + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/codefly-dev/core/agents/services/sbom" + "github.com/stretchr/testify/require" +) + +const pinned = "ghcr.io/codefly-dev/service-warehouse@sha256:1205da0743e56b68c12c754438476e8fdd0f05cdb08ce1c9687ddb8d8f0e1669" + +func TestSubjectsAreOnePerShippedPlatformAndPinned(t *testing.T) { + subjects, err := Subjects(context.Background(), "codefly.dev/warehouse", pinned, false) + require.NoError(t, err) + + var platforms []string + for _, subject := range subjects { + platforms = append(platforms, subject.GetPlatform()) + require.Equal(t, pinned, subject.GetReference()) + require.Equal(t, Role, subject.GetRole()) + require.Equal(t, "codefly.dev/warehouse", subject.GetService()) + require.Equal(t, sbom.SourceRegistry, sbom.SourceOf(subject)) + require.NoError(t, sbom.RequirePinned(subject)) + } + require.ElementsMatch(t, Platforms, platforms) +} + +// An evidence request that names no service would match evidence belonging to +// any service, so core refuses it; this package does not paper over that. +func TestSubjectsRefuseAnUnnamedServiceAndAFloatingReference(t *testing.T) { + _, err := Subjects(context.Background(), "", pinned, false) + require.Error(t, err) + _, err = Subjects(context.Background(), "codefly.dev/warehouse", "ghcr.io/codefly-dev/service-warehouse:latest", false) + require.Error(t, err) +} + +// A local subject carries the image ID the daemon holds, so an image the daemon +// does not hold can only be reported, never turned into a subject pinned to +// nothing. The resolved case needs a real image and is asserted by the e2e +// inventory. +func TestALocalSubjectNeedsAnImageTheDaemonHolds(t *testing.T) { + const absent = "swh-imageevidence-unit-test-absent:none" + subjects, err := Subjects(context.Background(), "codefly.dev/warehouse", absent, true) + require.Error(t, err, "got %d subjects", len(subjects)) + require.Contains(t, err.Error(), absent) +} + +func TestCollectRefusesToReportCoverageOfNothing(t *testing.T) { + _, err := Collect(context.Background(), t.TempDir(), nil) + require.Error(t, err) +} + +func TestDocumentsAreNamedAfterTheImageThePlatformAndTheDigest(t *testing.T) { + require.Equal(t, "service-warehouse-linux-arm64-1205da.cdx.json", + documentName(pinned, "linux/arm64", "sha256:1205da")) + require.Equal(t, "service-warehouse-linux-amd64-abc.cdx.json", + documentName("service-warehouse:e2e", "linux/amd64", "sha256:abc")) +} + +func TestReferenceNameDropsTheTagAndTheDigest(t *testing.T) { + for reference, want := range map[string]string{ + pinned: "ghcr.io/codefly-dev/service-warehouse", + "ghcr.io/codefly-dev/service-warehouse:0.0.1": "ghcr.io/codefly-dev/service-warehouse", + "localhost:5000/service-warehouse:e2e": "localhost:5000/service-warehouse", + "service-warehouse": "service-warehouse", + "service-warehouse:e2e": "service-warehouse", + } { + require.Equal(t, want, referenceName(reference), reference) + } +} + +func TestTheIndexNamesEveryDocumentWithItsPlatformAndDigest(t *testing.T) { + dir := t.TempDir() + require.NoError(t, WriteIndex(dir, []Document{{ + Reference: pinned, Digest: "sha256:aa", Platform: "linux/amd64", + Path: filepath.Join(dir, "a.cdx.json"), SHA256: "bb", + }})) + index, err := os.ReadFile(filepath.Join(dir, "index.txt")) + require.NoError(t, err) + require.Equal(t, "linux/amd64 ghcr.io/codefly-dev/service-warehouse@sha256:aa a.cdx.json bb\n", string(index)) +} From 632feeafeb92cc2a82aae53df6b3d98df86c9806 Mon Sep 17 00:00:00 2001 From: crisfiorini-oss <301773486+crisfiorini-oss@users.noreply.github.com> Date: Thu, 8 Oct 2026 17:03:02 -0400 Subject: [PATCH 04/10] feat(agent): compose the gateway as codefly.dev/warehouse, with its image and SBOM evidence (#12) A workspace can now compose this gateway the way it composes codefly-dev/service-object-storage: the repo-root main package is a codefly service agent (Service, Runtime, Builder on codefly-dev/core's agents package), declared by agent.codefly.yaml at 0.0.1 and released by a tag workflow as a GitHub release asset a consumer pins by version. This closes #12. The agent - Runtime starts the gateway container with a fresh x-codefly-token secret per run (an operator may pin one), publishes the port, and waits until the gateway's health service reports serving and Capabilities accepts the token. That attests the process and the credential, not the warehouse: the gateway probes no backend. A rejected token fails at once instead of burning the budget. - Consumers receive the "warehouse" configuration group: connection, endpoint and a secret-marked token. - Builder renders a Deployment, Service and a dedicated ServiceAccount (so a cloud warehouse can grant exactly this workload), gRPC health probes on the overall service, SWH_ALLOW_ANONYMOUS=true as service-object-storage does for its deployed profile. It refuses what it cannot deliver (SWH_AUTH_TOKEN, SWH_CREDENTIALS_FILE) and any value the manifest cannot carry. - The agent links no backend, so no cgo driver reaches its CGO_ENABLED=0 release build; a test holds that. The image and its evidence (the same commit, so the gate never reads no-image) - Dockerfile builds cmd/service-warehouse with cgo on a Debian 13 builder onto distroless/cc, not static: the embedded DuckDB engine is a cgo driver. - gateway-image.json pins the image by digest; the registry comes from core (resources.ImageRegistry), the repository name from the workflows' own github.repository. It records no digest until the first image is published, and the agent, Deploy, the audit and the release all refuse an empty one rather than run :latest. - publish-gateway-image.yml pushes the image by digest before the tag exists; release.yml tags only after the tag matches agent.codefly.yaml, the recorded digest was built for that version, and evidence for every platform was written. - Builder.SBOM serves image scope through core's SBOMImages and is checked against sbom.ValidateCoverage. ValidationCapabilities.image_sbom is served and deliberately not advertised (core docs/sbom.md rollout gate); a test pins it. - distribution_test.go no longer claims "no image". It lists every image- producing path, fails on one that is not accounted for or no longer produces its signal, and fails if the tests holding the evidence are gone. README and AGENTS.md state the new distribution, the agent and the release order. Reuse: core's resources.PublishedImage/ImageRegistry, sbom.Image, sbom.ExpectedFromImageReference, sbom.ValidateCoverage, agents.AuthMetadataKey (held equal to internal/auth's copy by a test), dockerrun, the kustomize conformance helper and the agent registration are imported. Copied from service-object-storage: the four generic deployment templates (namespace and kustomization base, kustomization and secret overlay), and the agent, workflow and test structure, adapted, without MinIO. Core's endpoint model no longer has the "external" visibility service-object-storage declares; the endpoint here is "internal", its replacement for cross-module reach. Classification: feature; nothing here works around a missing fix elsewhere. Release workflows pin codefly-dev/core's reusable workflows at the v0.16.0 commit that go.mod requires. Verified: build, vet, race tests under go1.27.0; the image builds for linux/arm64 and linux/amd64 and starts under both; the agent Runtime run against the built image (e2e tag) starts it, refuses a missing or wrong token, hands out the secret token and removes the container on Destroy; the release's lock check and tag guard were run against the real files; each guard was seen to fail when its property was broken. Not verified: the image-scope SBOM against a real warehouse image (needs syft on PATH, absent here), the multi-platform publish and the release workflows (never run; GH_PAT is needed), codefly agent ci past its source stage (the installed CLI's packager carries Go 1.26.4, so its build stage fails as documented in agent-ci.yml), and the image with the DuckDB driver linked (not on this branch). Co-Authored-By: Claude Sonnet 5.5 --- .dockerignore | 5 + .github/workflows/agent-ci.yml | 27 + .github/workflows/ci.yml | 72 +++ .github/workflows/publish-gateway-image.yml | 80 +++ .github/workflows/release.yml | 161 ++++++ .gitignore | 1 + .goreleaser.yaml | 41 ++ AGENTS.md | 63 +- Dockerfile | 24 + README.md | 149 +++-- agent.codefly.yaml | 10 + builder.go | 291 ++++++++++ cmd/service-warehouse/distribution_test.go | 92 ++- composition_test.go | 546 ++++++++++++++++++ gateway-image.json | 4 + gateway_image.go | 93 +++ gateway_image_test.go | 173 ++++++ go.mod | 63 +- go.sum | 103 +++- main.go | 224 +++++++ readiness_test.go | 129 +++++ runtime.go | 356 ++++++++++++ runtime_e2e_test.go | 248 ++++++++ sbom_e2e_test.go | 91 +++ sbom_test.go | 541 +++++++++++++++++ templates/agent/README.md.tmpl | 12 + .../kustomize/base/deployment.yaml.tmpl | 119 ++++ .../kustomize/base/kustomization.yaml.tmpl | 9 + .../kustomize/base/namespace.yaml.tmpl | 8 + .../kustomize/base/service.yaml.tmpl | 12 + .../environment/kustomization.yaml.tmpl | 8 + .../overlays/environment/secret.yaml.tmpl | 12 + templates/factory/GETTING_STARTED.md.tmpl | 5 + .../configurations/local/warehouse.secret.env | 5 + 34 files changed, 3718 insertions(+), 59 deletions(-) create mode 100644 .dockerignore create mode 100644 .github/workflows/agent-ci.yml create mode 100644 .github/workflows/publish-gateway-image.yml create mode 100644 .github/workflows/release.yml create mode 100644 .goreleaser.yaml create mode 100644 Dockerfile create mode 100644 agent.codefly.yaml create mode 100644 builder.go create mode 100644 composition_test.go create mode 100644 gateway-image.json create mode 100644 gateway_image.go create mode 100644 gateway_image_test.go create mode 100644 main.go create mode 100644 readiness_test.go create mode 100644 runtime.go create mode 100644 runtime_e2e_test.go create mode 100644 sbom_e2e_test.go create mode 100644 sbom_test.go create mode 100644 templates/agent/README.md.tmpl create mode 100644 templates/deployment/kustomize/base/deployment.yaml.tmpl create mode 100644 templates/deployment/kustomize/base/kustomization.yaml.tmpl create mode 100644 templates/deployment/kustomize/base/namespace.yaml.tmpl create mode 100644 templates/deployment/kustomize/base/service.yaml.tmpl create mode 100644 templates/deployment/kustomize/overlays/environment/kustomization.yaml.tmpl create mode 100644 templates/deployment/kustomize/overlays/environment/secret.yaml.tmpl create mode 100644 templates/factory/GETTING_STARTED.md.tmpl create mode 100644 templates/factory/configurations/local/warehouse.secret.env diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..9050b4d --- /dev/null +++ b/.dockerignore @@ -0,0 +1,5 @@ +.git +*.md +templates +dist +*.duckdb diff --git a/.github/workflows/agent-ci.yml b/.github/workflows/agent-ci.yml new file mode 100644 index 0000000..8c3f8d3 --- /dev/null +++ b/.github/workflows/agent-ci.yml @@ -0,0 +1,27 @@ +# Full-lifecycle agent validation: `codefly agent ci` runs the source, release, +# and generated-service conformance gates against this agent, through the shared +# reusable workflow every codefly service agent uses. +# +# Manual (workflow_dispatch) until a run of it has been seen green on this +# repository. service-object-storage parks the same workflow because the CLI's +# pinned go source-packager predated Go 1.27, and the release/build stage failed +# with `go.mod requires go >= 1.27.0 (GOTOOLCHAIN=local)` +# (codefly-dev/cli#499, since closed). Run locally with a CLI whose packager +# still carries Go 1.26.4, `codefly agent ci` passes its manifest and source +# stages here and fails that same build stage; the CLI release named below has +# not been run against this repository. Switch the trigger to `pull_request` once the first +# dispatch passes. +name: agent-ci + +on: + workflow_dispatch: + +jobs: + agent-ci: + # Pinned to an immutable core commit SHA, not @main (see release.yml): core's + # v0.16.0, the release go.mod requires. Bump the two together. + uses: codefly-dev/core/.github/workflows/agent-ci.yml@e54bc852f3d40cd1aeafc0ced2369a8c13a51bb8 + with: + # The CLI release the gate runs with. The go source-packager it installs + # is the one the CLI pins, so bumping this is how that pin moves. + codefly-cli-version: "0.1.181" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 57a46e7..d4d9cee 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,3 +17,75 @@ jobs: - run: go build ./... - run: go vet ./... - run: go test -race ./... + + # Runs the agent's Runtime against the gateway image this repository builds: + # the container starts, enforces the token the agent minted, and answers a + # consumer who presents it. The unit tests cannot say whether the image the + # Dockerfile produces serves what the agent expects of it. + agent-runtime-e2e: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + - uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5 + with: + go-version: "1.27.0" + cache: true + - name: build gateway image + run: docker build -t service-warehouse:e2e . + - name: end-to-end run through the agent runtime + env: + SWH_GATEWAY_IMAGE: service-warehouse:e2e + run: go test -tags e2e -count=1 -timeout 300s -run TestRuntimeEndToEnd . + + # Inventories a real gateway image through the agent's own image-SBOM path and + # checks it against the shared coverage contract, so a change that would ship + # an uninventoried image fails here rather than at release. + image-sbom: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + - uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5 + with: + go-version: "1.27.0" + cache: true + - name: build gateway image + run: docker build -t service-warehouse:e2e . + # The daemon holds this image and no registry serves it, so the scan needs + # syft on PATH: the managed containerized scanner has no Docker access. + - uses: anchore/sbom-action/download-syft@3ad7283483fc7af8ff2b4ea19663c2d5ca935e26 # v0.24.2 + - name: assert the image contract against a real image + env: + SWH_GATEWAY_IMAGE: service-warehouse:e2e + run: go test -tags e2e -count=1 -timeout 300s -run TestImageSBOM . + - name: write evidence through the release command + run: go run ./cmd/image-sbom -image service-warehouse:e2e -local -out sbom + # The local image is single-platform and reached through the daemon. The + # image a release publishes is neither: it is multi-architecture and + # resolved through the registry. Without this step the two-platform path + # the release depends on would never execute in CI. It inventories the + # digest gateway-image.json records — the image the agent actually names — + # so a lock pointing at something that does not resolve, or that ships + # fewer platforms than evidence has to cover, fails here rather than at + # release time. A lock with no digest is the tree before its first image + # was published: there is nothing to inventory, release.yml refuses to tag + # it, and the agent refuses to run it, so this is the one place it is + # allowed to pass, and says so. + - name: inventory the published multi-architecture image + run: | + set -euo pipefail + digest=$(jq -er '.digest' gateway-image.json) + if [ -z "$digest" ]; then + echo "::notice::gateway-image.json records no digest: no gateway image has been published yet, so there is no published image to inventory" + exit 0 + fi + go run ./cmd/image-sbom -image "$(jq -er '.name + "@" + .digest' gateway-image.json)" -out sbom-published + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: gateway-image-sbom + path: | + sbom/ + sbom-published/ + # A skipped scan writes nothing; failing here keeps a green job from + # standing in for evidence that was never produced. sbom/ is always + # written, so this fails only when the local scan did not run. + if-no-files-found: error diff --git a/.github/workflows/publish-gateway-image.yml b/.github/workflows/publish-gateway-image.yml new file mode 100644 index 0000000..37409f2 --- /dev/null +++ b/.github/workflows/publish-gateway-image.yml @@ -0,0 +1,80 @@ +name: publish-gateway-image + +# Builds the gateway image and pushes it by digest, carrying no tag. Nothing +# resolves it yet: release.yml creates the tags consumers use, from the digest +# gateway-image.json records. +# +# It runs before the release tag rather than from it. The agent pins the digest +# it runs, and a digest is only known once the image is built — so a tag cut +# before the build can never carry it. Publishing here, recording the digest in +# a commit, and tagging afterwards is what lets the released agent name the +# image that was actually built for its version. +on: + workflow_dispatch: + inputs: + version: + description: Upcoming stable agent version reported by codefly publish --dry-run (without v) + required: true + type: string + +env: + # The repository this runs in, never a typed name: the agent derives the same + # name from core's image registry and gatewayImageName, and + # TestGatewayImageNameIsThisRepository holds the two together. + IMAGE: ghcr.io/${{ github.repository }} + +jobs: + publish: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3 + - uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Validate the upcoming agent version + id: version + env: + VERSION: ${{ inputs.version }} + run: | + set -euo pipefail + if [[ ! "$VERSION" =~ ^(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)$ ]]; then + echo "version must be a stable semantic version without a v prefix" >&2 + exit 1 + fi + echo "version=$VERSION" >> "$GITHUB_OUTPUT" + # The version label is how the release tells this image apart from the one + # published for an earlier version: both are digests in the same + # repository, and a lock left unrefreshed would otherwise publish the + # previous release's bytes under the new tag. + - uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6 + id: build + with: + context: . + platforms: linux/amd64,linux/arm64 + labels: org.opencontainers.image.version=${{ steps.version.outputs.version }} + outputs: type=image,name=${{ env.IMAGE }},push-by-digest=true,name-canonical=true,push=true + - name: Record the digest + env: + DIGEST: ${{ steps.build.outputs.digest }} + run: | + set -euo pipefail + mkdir -p release-evidence + jq -n --arg name "$IMAGE" --arg digest "$DIGEST" '{name: $name, digest: $digest}' > release-evidence/gateway-image.json + { + echo 'Commit this as `gateway-image.json`, then tag the release:' + echo + echo '```json' + cat release-evidence/gateway-image.json + echo '```' + } >> "$GITHUB_STEP_SUMMARY" + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: gateway-image-lock-${{ steps.version.outputs.version }} + path: release-evidence/gateway-image.json + if-no-files-found: error diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..a2ff2bd --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,161 @@ +name: release + +# Everything a `v*` tag publishes, in dependency order: the tags that resolve +# the gateway container the agent runs, then the GitHub release carrying the +# agent binary. The agent pins the gateway digest gateway-image.json records, so +# a release published without those tags leaves consumers unable to resolve the +# image by the version they asked for. One workflow, so a failure anywhere stops +# what follows. +# +# The image itself is built and pushed by digest before the tag, by +# publish-gateway-image.yml: a digest cannot be recorded in a commit the tag has +# already been cut from. +on: + push: + tags: + - 'v*' + +permissions: + contents: read + +env: + # The repository this runs in, never a typed name (see + # publish-gateway-image.yml). + IMAGE: ghcr.io/${{ github.repository }} + +jobs: + # A tag whose tests fail publishes nothing. go-service-release.yml gates the + # binary on its own test job, but a caller cannot depend on a job inside the + # workflow it calls, so the image tags need this gate of their own — + # otherwise a failing test still leaves an untested image tagged :latest. + # Its own job rather than a step in gateway-image: that job holds packages: + # write and an authenticated ghcr session, and the whole test tree has no + # business running under a registry-write token. + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + - uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5 + with: + go-version-file: go.mod + cache: true + - run: go test ./... + + gateway-image: + needs: test + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3 + - uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Derive version from tag + id: version + run: echo "version=${GITHUB_REF_NAME#v}" >> "$GITHUB_OUTPUT" + # The image tag comes from the git tag, while the agent reports the + # version of the embedded agent.codefly.yaml and resolves its gateway by + # the digest recorded beside it. A tag ahead of the file publishes one + # image and ships an agent that calls itself the previous release, which + # exists, so nothing fails at run time and a consumer pinned to one + # version runs another. + - name: Tag matches the agent version + env: + VERSION: ${{ steps.version.outputs.version }} + run: | + set -euo pipefail + embedded=$(awk '$1 == "version:" {print $2}' agent.codefly.yaml) + if [ "$embedded" != "$VERSION" ]; then + echo "release tag $VERSION does not match agent.codefly.yaml $embedded" >&2 + exit 1 + fi + # The recorded digest is the one the agent runs and reports evidence for, + # so it has to be an image published for this version and not the one left + # over from the previous release — both are digests in the same repository + # and neither fails to resolve. publish-gateway-image.yml stamps the + # version it built for onto every platform of the image, which is what + # tells them apart. A lock that records no digest at all is the tree + # before its first image was published: there is nothing to tag, and + # `jq -e` alone would let an empty string through. + - name: Locked gateway image was published for this version + id: locked + env: + VERSION: ${{ steps.version.outputs.version }} + run: | + set -euo pipefail + name=$(jq -er '.name' gateway-image.json) + digest=$(jq -er '.digest' gateway-image.json) + if [ "$name" != "$IMAGE" ]; then + echo "gateway-image.json names $name, but this workflow publishes $IMAGE" >&2 + exit 1 + fi + if [[ ! "$digest" =~ ^sha256:[0-9a-f]{64}$ ]]; then + echo "gateway-image.json records no sha256 digest (got \"$digest\"): no gateway image has been published;" >&2 + echo "run publish-gateway-image.yml and commit the digest it prints before tagging" >&2 + exit 1 + fi + labelled=$(docker buildx imagetools inspect "$name@$digest" --format '{{json .Image}}' | + jq -ec '[.. | objects | .Labels? // empty | ."org.opencontainers.image.version"? // empty] | unique') + if [ "$labelled" != "$(jq -nc --arg v "$VERSION" '[$v]')" ]; then + echo "gateway-image.json records $digest, built for version(s) $labelled, not $VERSION;" >&2 + echo "run publish-gateway-image.yml and commit the digest it prints before tagging" >&2 + exit 1 + fi + echo "digest=$digest" >> "$GITHUB_OUTPUT" + - uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5 + with: + go-version-file: go.mod + cache: true + - uses: anchore/sbom-action/download-syft@3ad7283483fc7af8ff2b4ea19663c2d5ca935e26 # v0.24.2 + # Evidence for every platform the image ships, generated by the same code + # path the agent's Builder.SBOM uses, so each document is bound to the child + # manifest digest inside the document itself. It scans the recorded digest + # rather than a tag, so the evidence describes exactly the bytes that are + # about to be tagged. + - name: image SBOM evidence + env: + DIGEST: ${{ steps.locked.outputs.digest }} + run: go run ./cmd/image-sbom -image "$IMAGE@$DIGEST" -out sbom + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: gateway-image-sbom-${{ steps.version.outputs.version }} + path: sbom/ + if-no-files-found: error + # Only now does the image become resolvable by the tags consumers use. A + # platform that resolved to no manifest, or a scan that failed, has already + # failed the job above, leaving the digest untagged and unreferenced. + - name: Publish the release tags + env: + VERSION: ${{ steps.version.outputs.version }} + DIGEST: ${{ steps.locked.outputs.digest }} + run: | + set -euo pipefail + docker buildx imagetools create \ + --tag "$IMAGE:$VERSION" \ + --tag "$IMAGE:latest" \ + "$IMAGE@$DIGEST" + for published in "$IMAGE:$VERSION" "$IMAGE:latest"; do + resolved=$(docker buildx imagetools inspect "$published" --format '{{json .Manifest}}' | jq -er '.digest') + if [ "$resolved" != "$DIGEST" ]; then + echo "$published resolves to $resolved, want $DIGEST" >&2 + exit 1 + fi + done + + # The binary ships last: only once the image it points at is resolvable and + # inventoried is the GitHub release that carries the agent worth creating. + release: + needs: gateway-image + # Pinned to an immutable core commit SHA, not @main: this call carries + # secrets: inherit, so a mutable ref would hand an upstream change this + # repo's GH_PAT with no commit here. It is core's v0.16.0, the release go.mod + # requires; bump the two together. This version of the workflow keeps the + # release PAT out of jobs that run unreviewed code and refuses a tag not + # reachable from the default branch. + uses: codefly-dev/core/.github/workflows/go-service-release.yml@e54bc852f3d40cd1aeafc0ced2369a8c13a51bb8 + secrets: inherit diff --git a/.gitignore b/.gitignore index 4a6f7b5..5b363d0 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ /service-warehouse /dist/ *.duckdb +/.codefly/ diff --git a/.goreleaser.yaml b/.goreleaser.yaml new file mode 100644 index 0000000..23f41ab --- /dev/null +++ b/.goreleaser.yaml @@ -0,0 +1,41 @@ +version: 2 + +sboms: + - id: archives + artifacts: archive + documents: + - "${artifact}.sbom.json" + disable: false + +project_name: service-warehouse + +# The release asset is the codefly agent binary: the main package at the repo +# root, which is what makes codefly.dev/warehouse resolvable. It is built without +# cgo and must stay so: the gateway (cmd/service-warehouse) is the binary that +# carries the warehouse drivers, and ships as the container image the release +# workflow publishes before this one runs. TestAgentLinksNoBackendDriver holds +# the drivers out of this package. +builds: + - id: darwin + main: . + env: + - CGO_ENABLED=0 + goos: + - darwin + goarch: + - amd64 + - arm64 + - id: linux + main: . + env: + - CGO_ENABLED=0 + goos: + - linux + goarch: + - amd64 + - arm64 + +archives: + - id: default + formats: [tar.gz] + name_template: "{{ .ProjectName }}_{{ .Version }}_{{ .Os }}_{{ .Arch }}" diff --git a/AGENTS.md b/AGENTS.md index 81883c0..44826e7 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -22,6 +22,14 @@ this file needs a fix in the same PR. - **Depends on:** the warehouse drivers and the gRPC contract, nothing else. The SBOM scanner and the image-SBOM contract live in `codefly-dev/core` and are not reimplemented here. +- **The agent is a second program in this module.** The repo-root `main` + package (`main.go`, `runtime.go`, `builder.go`) is the codefly agent that + lets a workspace compose the gateway; it links `codefly-dev/core`, and + `cmd/service-warehouse` must not. The agent links no backend, so no cgo + driver reaches its `CGO_ENABLED=0` release build + (`TestAgentLinksNoBackendDriver`). Like the gateway, it authorizes nothing: it + starts the gateway, hands out the one token it enforces, and renders the + manifest. Three rules follow from that: @@ -58,8 +66,10 @@ preferences; each is a rule an agent broke at real cost. become a fix by working, by being small, by being local, or by the real fix belonging to someone else. 4. **Never hardcode what the system resolves.** `internal/config/config.go` is - the entire configuration surface and every knob is an `SWH_*` variable — a - listen address, a DSN, a credentials path, a query cap. If you are typing a + the entire gateway configuration surface and every knob is an `SWH_*` + variable — a listen address, a DSN, a credentials path, a query cap. (The + agent has one of its own, `SWH_GATEWAY_IMAGE`, a locally built image to run + instead of the pinned one; it never reaches the gateway.) If you are typing a connection string, a port, or a credential into Go source, a test, or a manifest, you are encoding something true only on your machine for the next ten minutes. A test that needs a backend uses `mem`, which needs nothing. @@ -78,9 +88,9 @@ preferences; each is a rule an agent broke at real cost. ## Build and test -Derived from [`.github/workflows/ci.yml`](.github/workflows/ci.yml): those -three commands are the whole gate, so a green local run of all three is the -definition of done. CI pins the Go version as a literal in that workflow +Derived from [`.github/workflows/ci.yml`](.github/workflows/ci.yml): the +`build-test` job is those three commands, so a green local run of all three is +the definition of done. CI pins the Go version as a literal in that workflow (`go-version: "1.27.0"`) rather than reading `go.mod`, so bumping `go.mod` alone moves your toolchain and not CI's — change both. @@ -90,6 +100,27 @@ go vet ./... go test -race ./... ``` +Two more CI jobs need Docker and are not part of that gate. They build the image +(`docker build -t service-warehouse:e2e .`) and run the agent against it, behind +the `e2e` build tag: + +```sh +SWH_GATEWAY_IMAGE=service-warehouse:e2e go test -tags e2e -count=1 -run TestRuntimeEndToEnd . +SWH_GATEWAY_IMAGE=service-warehouse:e2e go test -tags e2e -count=1 -run TestImageSBOM . # needs syft on PATH +``` + +**A green e2e run is not evidence that anything ran.** With +`SWH_GATEWAY_IMAGE` unset both tests call `t.Skip`, the package still prints +`ok`, and `go test` exits 0. The SBOM one also skips without `syft`. Read `-v` +output for `--- PASS`, never `ok` alone. + +`codefly agent ci` is the full agent conformance gate. Its workflow +(`agent-ci.yml`) is `workflow_dispatch` only until a run of it has been seen +green here. Run locally from the repo root, it needs a CLI whose go +source-packager carries Go 1.27: with one that carries 1.26.4 the manifest and +source stages pass and the build stage fails on `go.mod requires go >= 1.27.0`, +which is the CLI's gap, not this repo's. + Run the server locally against the zero-dependency backend — it needs no database, no warehouse credential, and no network. It still refuses an unauthenticated listener, so a local run states that choice (or sets @@ -113,6 +144,21 @@ version is pinned anywhere in the repo (codefly-dev/service-warehouse#7). If `buf generate` leaves you with a diff that is only those banner lines, revert it — you changed your toolchain, not the contract. +## Releases + +A `v*` tag publishes the gateway image's tags first and the agent binary second, +from one workflow, in that order. The image itself is published *before* the +tag, because the agent pins its digest and a digest cannot be recorded in a +commit the tag was already cut from; `README.md` ("Releasing") has the four +steps. `gateway-image.json` records no digest until the first image exists, and +every consumer of it — the agent, `Deploy`, the release — refuses an empty one +rather than run `:latest`. The tag must match `version:` in `agent.codefly.yaml` +or the job fails by design. + +Core is pinned in two places that move together: the `codefly-dev/core` version +in `go.mod` and the commit SHA of core's reusable workflows in `release.yml` and +`agent-ci.yml` (never `@main`: `release.yml` passes `secrets: inherit`). + ## Where the depth is Keep this file short; depth belongs in the file that owns the subject. @@ -123,9 +169,10 @@ Keep this file short; depth belongs in the file that owns the subject. - `internal/backend/backend.go` — the provider-agnostic interface every backend implements, and the portable type model. - [`cmd/service-warehouse/distribution_test.go`](cmd/service-warehouse/distribution_test.go) - — gates the claim that this repo ships no container image. It fails the day - the repo gains a way to build or deploy one; that is the day image SBOM - coverage is owed, and the gate is scoped to this repo only. + — gates the claim that this repo ships one image, the gateway, and that its + SBOM evidence exists. It lists every path that builds or deploys an image and + fails on one that is not accounted for, so a second image is the day more + image SBOM coverage is owed. The gate is scoped to this repo only. - A repeated multi-step procedure belongs in a `.claude/skills//SKILL.md` of its own, or in a nested `AGENTS.md` next to the code it describes — not in another paragraph here. There are none yet: this repo has three diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..84825ec --- /dev/null +++ b/Dockerfile @@ -0,0 +1,24 @@ +# The warehouse gateway image. A client speaks only the codefly/warehouse/v0 +# gRPC contract; every backend is compiled in and selected at runtime by +# SWH_BACKEND. The image ships no warehouse credentials: those arrive as +# environment at deploy time. +# +# cgo is on, and the runtime image carries glibc and libstdc++ (distroless/cc). +# The embedded DuckDB engine is a cgo driver (internal/backend/duckdb), so a +# CGO_ENABLED=0 build onto distroless/static would keep building until the +# driver lands and then stop. Both stages name Debian 13 explicitly: the binary +# links against the build stage's glibc and needs one at least as new in the +# image it runs in, which "golang:1.27" moving to the next Debian would break. +FROM golang:1.27-trixie AS build +WORKDIR /src +COPY go.mod go.sum ./ +RUN go mod download +COPY . . +RUN CGO_ENABLED=1 GOOS=linux go build -trimpath -ldflags="-s -w" \ + -o /out/service-warehouse ./cmd/service-warehouse + +FROM gcr.io/distroless/cc-debian13:nonroot +COPY --from=build /out/service-warehouse /service-warehouse +# The gRPC listen port; SWH_LISTEN can override it. +EXPOSE 9465 +ENTRYPOINT ["/service-warehouse"] diff --git a/README.md b/README.md index 465f01a..8feb675 100644 --- a/README.md +++ b/README.md @@ -118,52 +118,111 @@ This is authentication only — whether the caller holds the deployment's token. The server still authorizes nothing: which caller may run which statement is settled before the request arrives. -## Distribution and SBOM evidence - -This repository **publishes no container image**. It builds to the -`service-warehouse` Go binary; nothing here builds, pushes, or deploys an image, -and the tree carries no Codefly agent manifest and no `Builder` implementation — -so no `Builder.SBOM` RPC is served and none is claimed. +## Running as a codefly service + +A workspace composes this gateway as `codefly.dev/warehouse`, the same way it +composes [`service-object-storage`](https://github.com/codefly-dev/service-object-storage) +as `codefly.dev/object-storage`. The agent is the repo-root `main` package +([`main.go`](main.go), [`runtime.go`](runtime.go), [`builder.go`](builder.go)), +declared by [`agent.codefly.yaml`](agent.codefly.yaml); it is released as a GitHub +release asset a consumer pins by version. The gateway it runs is a container +image, pinned by digest in [`gateway-image.json`](gateway-image.json). + +**What a consumer receives.** The configuration group `warehouse`, with +`connection` (`grpc://host:port`), `endpoint` (`host:port`) and `token`, a +secret value to send as the `x-codefly-token` metadata header. The agent mints a +fresh token for every local run unless an operator pins one, so a run revokes the +previous run's. + +**Locally** (`codefly run`) the agent starts the gateway container, publishes its +port, waits until the gateway reports itself serving *and* accepts the agent's +token, and hands the endpoint and token to consumers. That wait attests the +process and the credential, not the warehouse: the gateway does not probe its +backend and no RPC reports that. Set `backend:` in `service.codefly.yaml` to +choose the engine; left unset, the gateway's own default applies, which today is +`duckdb` and refuses to start, so a local run presently needs `backend: mem` +(the catalog-only backend). The agent delivers `SWH_BACKEND`, `SWH_DATABASE`, +`SWH_DATASET` and `SWH_LOCATION`, and refuses `SWH_CREDENTIALS_FILE`, which +nothing could project into the container. `SWH_GATEWAY_IMAGE` makes the agent +run a locally built image instead of the pinned one (`docker build -t +service-warehouse:local .`); it configures the agent, not the gateway. + +**Deployed** the agent renders a Deployment, Service and its own ServiceAccount +(so a cloud warehouse can grant exactly this workload through a workload +identity), with gRPC health probes on the gateway's overall service. The +deployed listener is `SWH_ALLOW_ANONYMOUS=true`: caller identity is enforced by +the cluster (NetworkPolicy and service-mesh mTLS), because nothing delivers a +secret to the consumers' `warehouse` configuration for a deployment. A configured +`SWH_AUTH_TOKEN` or `SWH_CREDENTIALS_FILE` is therefore refused rather than +rendered, and so is a value the manifest cannot carry. The rest of the `SWH_*` +surface reaches the gateway by the environment's service configuration being +bound onto the container by name, after the render. -Under the fleet image-SBOM contract ([`codefly-dev/core` `docs/sbom.md`][sbom], -released in `v0.3.29`) that is the `NO_IMAGE_REASON_NO_IMAGE` case: a service -that legitimately ships no image, which is a different thing from an agent whose -implementation is missing (`UNSUPPORTED`). A source or lockfile inventory never -counts as image coverage, so the absence of an image is recorded here as a -status — the Go module's dependency list does not satisfy it. +## Distribution and SBOM evidence -`TestRepositoryShipsNoImage` in +This repository publishes **one container image**, the gateway +(`ghcr.io/codefly-dev/service-warehouse`, for `linux/amd64` and `linux/arm64`), +and the agent binary that runs it. The service ships no other image: it has no +init, migration or sidecar image, and the warehouse behind it is not run by this +service. + +Under the fleet image-SBOM contract ([`codefly-dev/core` `docs/sbom.md`][sbom]) +that image owes evidence, and the agent serves it. `Builder.SBOM` at image scope +returns one CycloneDX inventory per shipped platform, covering OS packages and +installed application dependencies, each bound to the image digest and platform +that was scanned and attributed to the service. The scanner is core's +(`sbom.Image`, through `BuilderWrapper.SBOMImages`), and the response is checked +against core's `sbom.ValidateCoverage`. + +- [`internal/imageevidence`](internal/imageevidence/imageevidence.go) holds the + one list of shipped platforms and the subjects derived from it; the agent, the + release command and the tests all read it. +- [`cmd/image-sbom`](cmd/image-sbom/main.go) writes the documents. A release runs + it against the digest [`gateway-image.json`](gateway-image.json) records and + attaches them as the `gateway-image-sbom-` workflow artifact, before + the tags consumers resolve are created, so an image whose scan failed is never + pullable by the version it was built for. +- CI builds the image and checks the same contract against it (`image-sbom`), and + runs the agent against it (`agent-runtime-e2e`). +- `ValidationCapabilities.image_sbom` is **served but not advertised**: core + holds the advertisement back until every consumer that evaluates it runs a core + that can represent the phase, and a present `Validation` is authoritative for + every operation it omits. `TestImageSBOMIsServedButNotAdvertised` marks it. + +`TestEveryImageThisRepositoryShipsIsCovered` in [`cmd/service-warehouse/distribution_test.go`](cmd/service-warehouse/distribution_test.go) -gates that status. It fails when this repository gains a way to **build** an -image (a Dockerfile, an `agent.codefly.yaml`, a compose file, or a workflow -running `docker build`/`docker buildx`/`docker/build-push-action`) *or* a way to -**deploy** one (a Kubernetes manifest, Helm values, or a Kustomize overlay -naming an image). Both matter: evidence binds to the digest actually built or -selected for deployment, so an image built elsewhere and deployed from here owes -coverage just the same. +gates this. It lists every path that builds an image (a Dockerfile, an +`agent.codefly.yaml`, a compose file, a workflow running +`docker build`/`docker buildx`/`docker/build-push-action`) or deploys one (a +Kubernetes manifest, Helm values, a Kustomize overlay naming an image), and fails +on any that is not accounted for, or on one that stopped producing its signal, +or when the tests holding the evidence are gone. A second image therefore has to +arrive with its evidence and with an edit to that list. The gate is scoped to this repository, which is the limit of what it can prove. If another repository ever packages this binary into an image, the service ships -an image that nothing here can see — that case has to be recorded where that +an image that nothing here can see, and that case has to be recorded where that image is built. -Introducing image distribution means this lands **with** the image rather than -after it: +### Releasing -- a valid CycloneDX SBOM for each final runtime image — OS packages and - installed application dependencies — covering every shipped platform and every - service-owned runtime, init, migration, and sidecar image; -- evidence bound to the immutable digest and platform actually built or - deployed, served at image scope through the shared contract - (`BuilderWrapper.SBOMImages`) and checked by `sbom.ValidateCoverage`; -- the SBOM artifact, checksum, image digest, platform, and service identity - carried into the build/release report and retrievable with the image; -- failures reported as failures — a failed scan, a missing image, a stale digest - or an omitted platform must never read as complete coverage. +The digest is only known once the image is built, so the order is fixed, and +`release.yml` refuses to tag a tree that skips a step: -The scanner itself stays in `core`; this repository does not reimplement it. +1. Find the upcoming version with `codefly publish --dry-run`. +2. Run the `publish-gateway-image` workflow with that version (no `v`). It + pushes the image by digest, carrying no tag, and prints `gateway-image.json`. +3. Commit that file in a reviewed change. Until it is committed the lock records + no digest, and the agent, the deployment and the release all refuse to name an + image. +4. `codefly publish` tags the release. The tag workflow tests, checks that the + tag matches `agent.codefly.yaml`, that the recorded digest was built for this + version, inventories it, and only then creates the `:` and `:latest` + tags and the GitHub release carrying the agent. -[sbom]: https://github.com/codefly-dev/core/blob/v0.3.29/docs/sbom.md +The release call needs a `GH_PAT` secret in the repository (`secrets: inherit`). + +[sbom]: https://github.com/codefly-dev/core/blob/v0.16.0/docs/sbom.md ## Develop @@ -173,9 +232,20 @@ codefly generate proto --proto ./proto --output ./gen go build ./... go vet ./... -go test ./... # unit tests (mem backend + bufconn server) +go test -race ./... # unit tests (mem backend + bufconn server) and the agent's ``` +The agent's end-to-end tests need Docker and are behind the `e2e` tag; the SBOM +one also needs `syft` on `PATH`: + +```bash +docker build -t service-warehouse:e2e . +SWH_GATEWAY_IMAGE=service-warehouse:e2e go test -tags e2e -count=1 . +``` + +With `SWH_GATEWAY_IMAGE` unset they skip, and a skipped run still prints `ok`: +read `-v` for `--- PASS`. + `buf` is never run on the host. `codefly generate proto` runs it inside the versioned proto companion image (`ghcr.io/codefly-dev/proto`), so the plugin versions and the `goimports` pass are the image's, fixed by its tag — two @@ -203,4 +273,11 @@ internal/auth/ caller authentication (the x-codefly-token interce internal/config/ env configuration internal/serr/ normalized error model cmd/service-warehouse/ the server binary +cmd/image-sbom/ writes the image's SBOM evidence (run by the release) +internal/imageevidence/ the shipped platforms and the SBOM subjects derived from them +main.go runtime.go builder.go the codefly agent (codefly.dev/warehouse) that runs the gateway +agent.codefly.yaml the agent's manifest and version +gateway-image.json the gateway image the agent pins, by digest +Dockerfile the gateway image +templates/ the agent's README, factory files and Kubernetes manifests ``` diff --git a/agent.codefly.yaml b/agent.codefly.yaml new file mode 100644 index 0000000..49b4ab1 --- /dev/null +++ b/agent.codefly.yaml @@ -0,0 +1,10 @@ +publisher: codefly.dev +kind: codefly:service +name: warehouse +version: 0.0.1 +source: + directory: . + agent: codefly.dev/go:0.0.56 +release: + owner: workflow + workflow: release.yml diff --git a/builder.go b/builder.go new file mode 100644 index 0000000..f73dc39 --- /dev/null +++ b/builder.go @@ -0,0 +1,291 @@ +package main + +import ( + "context" + "embed" + "fmt" + "regexp" + + "github.com/codefly-dev/core/agents/communicate" + "github.com/codefly-dev/core/agents/services" + v0 "github.com/codefly-dev/core/generated/go/codefly/base/v0" + builderv0 "github.com/codefly-dev/core/generated/go/codefly/services/builder/v0" + "github.com/codefly-dev/core/resources" + "github.com/codefly-dev/core/standards" + "github.com/codefly-dev/core/wool" + + "github.com/codefly-dev/service-warehouse/internal/imageevidence" +) + +type Builder struct { + *services.DefaultBuilder + *Service +} + +// deploymentTemplateParameters carries the values the Kubernetes templates need +// beyond the resolved configuration. An empty value renders no variable at all, +// leaving the choice to the gateway. +type deploymentTemplateParameters struct { + Backend string + Database string + Dataset string + Location string + // ServicePort is the port the Kubernetes Service publishes: the in-cluster + // port core allocated to the gRPC endpoint, the one every consumer is told + // to dial. It forwards to ContainerPort. + ServicePort uint32 + // ContainerPort is the port the gateway listens on inside its container, + // which the template also hands the gateway as SWH_LISTEN. + ContainerPort uint32 +} + +// renderedValue is what a configuration value may look like to be written into a +// manifest. The templates place these inside double-quoted YAML scalars, so a +// quote, a newline or a template action in one would not be a value but a way to +// write the rest of the manifest. Every real value (a backend kind, a BigQuery +// project, a dataset, a region) fits; one that does not is bound onto the +// container by name by the CLI instead of rendered here. +var renderedValue = regexp.MustCompile(`^[A-Za-z0-9._:/@-]*$`) + +func NewBuilder() *Builder { + service := NewService() + return &Builder{ + DefaultBuilder: services.NewDefaultBuilder(service.Builder), + Service: service, + } +} + +func (s *Builder) Load(ctx context.Context, req *builderv0.LoadRequest) (*builderv0.LoadResponse, error) { + defer s.Wool.Catch() + return s.Builder.LoadService(ctx, req, services.BuilderLoad{ + Settings: s.Settings, + Requirements: requirements, + FactoryTemplates: factoryFS, + ResolveEndpoints: func(ctx context.Context, endpoints []*v0.Endpoint) error { + endpoint, err := resolveServingGRPCEndpoint(ctx, endpoints) + if err != nil { + return err + } + s.GrpcEndpoint = endpoint + return nil + }, + }) +} + +func (s *Builder) Audit(ctx context.Context, req *builderv0.AuditRequest) (*builderv0.AuditResponse, error) { + defer s.Wool.Catch() + ctx = s.Wool.Inject(ctx) + image, err := publishedGatewayImage() + if err != nil { + return s.Builder.AuditError(err) + } + return s.Builder.AuditContainer(ctx, req, image.FullName()) +} + +// SBOM answers a source-scope request with the package inventory this agent has +// always produced, and an image-scope request with evidence bound to the digest +// and platform of every image the service ships. A caller that resolved the +// images itself passes them as subjects; an empty subject list asks the agent to +// enumerate its own. +func (s *Builder) SBOM(ctx context.Context, req *builderv0.SBOMRequest) (*builderv0.SBOMResponse, error) { + defer s.Wool.Catch() + ctx = s.Wool.Inject(ctx) + if req.GetScope() != builderv0.SBOMScope_SBOM_SCOPE_IMAGE { + image, err := publishedGatewayImage() + if err != nil { + return s.Builder.SBOMError(err) + } + return s.Builder.SBOMContainer(ctx, image.FullName()) + } + if subjects := req.GetSubjects(); len(subjects) > 0 { + return s.Builder.SBOMImages(ctx, subjects) + } + subjects, err := s.imageSubjects(ctx) + if err != nil { + return s.Builder.SBOMImageError(err) + } + return s.Builder.SBOMImages(ctx, subjects) +} + +// imageSubjects enumerates the images this service ships, and the way to reach +// them. The gateway is the only one: the service has no init, migration or +// sidecar image, and the warehouse behind it is not run by this service. +// +// The subjects themselves come from imageevidence, which the release command +// reads too, so the evidence a release publishes describes the same images the +// agent reports. +func (s *Builder) imageSubjects(ctx context.Context) ([]*builderv0.ImageSubject, error) { + image, overridden, err := effectiveGatewayImage() + if err != nil { + return nil, err + } + return imageevidence.Subjects(ctx, s.Base.Unique(), image.FullName(), overridden) +} + +func (s *Builder) Deploy(ctx context.Context, req *builderv0.DeploymentRequest) (*builderv0.DeploymentResponse, error) { + defer s.Wool.Catch() + ctx = s.Wool.Inject(ctx) + image, err := publishedGatewayImage() + if err != nil { + return s.Builder.DeployError(err) + } + s.Base.SetDockerImage(image) + + // Resolve the effective configuration from the deployment request; without + // this the template only ever sees the zero-value config. + if err := s.LoadConfiguration(ctx, req.GetConfiguration()); err != nil { + return s.Builder.DeployError(err) + } + + parameters := &deploymentTemplateParameters{ + Backend: s.conf.backend, + Database: s.conf.database, + Dataset: s.conf.dataset, + Location: s.conf.location, + ContainerPort: gatewayContainerPort, + } + servicePort, err := s.servicePort(ctx, req.GetNetworkMappings()) + if err != nil { + return s.Builder.DeployError(err) + } + parameters.ServicePort = servicePort + + // A backend kind is not validated against a list here: the gateway owns the + // set of backends it compiles in and refuses an unknown one at startup, + // naming the registered kinds, and a second list in the agent would drift + // from it. What this layer owns is the manifest, so it checks that every + // value is one the manifest can carry. + for _, value := range []struct{ name, value string }{ + {"SWH_BACKEND", parameters.Backend}, + {"SWH_DATABASE", parameters.Database}, + {"SWH_DATASET", parameters.Dataset}, + {"SWH_LOCATION", parameters.Location}, + } { + if !renderedValue.MatchString(value.value) { + return s.Builder.DeployError(fmt.Errorf("%s %q cannot be rendered into the manifest: "+ + "it may hold only letters, digits and . _ : / @ -; bind a value outside that set onto the container "+ + "through the environment's service configuration instead", value.name, value.value)) + } + } + // A credentials-file path cannot be honored by this deployment: nothing + // projects the file into the pod, and the configuration channel carries the + // path, never its contents. Emitting the variable anyway names a path that + // does not exist, and the gateway then fails on its first query after + // passing every probe. Keyless auth is the remedy where the platform has it + // (a workload identity bound to the pod's ServiceAccount); elsewhere the + // file has to be mounted by an overlay this agent does not own. Either way, + // reject the path rather than render a manifest that lies about it. + if s.conf.credentialsFile != "" { + return s.Builder.DeployError(fmt.Errorf("SWH_CREDENTIALS_FILE cannot be delivered by this deployment: "+ + "nothing here mounts %s into the pod, so the gateway would name a path that does not exist; "+ + "unset it and bind the workload's Kubernetes ServiceAccount to a cloud identity (workload identity), "+ + "or mount the file from an overlay of your own and patch SWH_CREDENTIALS_FILE onto the container there", + s.conf.credentialsFile)) + } + // A configured gateway token cannot be honored by this deployment: the + // manifest carries no secret values, and CreateConnectionConfiguration only + // emits a token the Runtime generated, so consumers would receive none. + // Accepting it would render a manifest that either crash-loops the gateway or, + // worse, starts a gateway enforcing a credential every consumer lacks. Reject + // it where the manifest is owned. + if s.conf.authToken != "" { + return s.Builder.DeployError(fmt.Errorf("SWH_AUTH_TOKEN cannot be delivered by this deployment: " + + "the emitted Secret carries no configuration values and consumers would receive no credential; " + + "caller identity in the deployed profile is enforced by the cluster (NetworkPolicy and service-mesh mTLS)")) + } + return s.Builder.DeployKustomize(ctx, req, services.KustomizeDeployment{ + EnvironmentVariables: s.EnvironmentVariables, + Templates: deploymentFS, + Parameters: parameters, + PodOverlay: workloadPodOverlay(), + }) +} + +// servicePort is the port the rendered Service publishes for the gRPC endpoint: +// the in-cluster port core allocated to it, which is what every consumer is +// given to dial. With no container mapping for this endpoint in the request (a +// render with no network input) it keeps the container port, so that render is +// unchanged. +func (s *Builder) servicePort(ctx context.Context, mappings []*v0.NetworkMapping) (uint32, error) { + if s.GrpcEndpoint != nil { + for _, mapping := range mappings { + endpoint := mapping.GetEndpoint() + if endpoint.GetApi() != standards.GRPC || + endpoint.GetModule() != s.GrpcEndpoint.GetModule() || + endpoint.GetService() != s.GrpcEndpoint.GetService() || + endpoint.GetName() != s.GrpcEndpoint.GetName() { + continue + } + // An external endpoint is reached through its DNS entry, never this + // Service, and core gives it no container view. + if resources.IsExternalEndpoint(endpoint) { + continue + } + instance, err := resources.FindNetworkInstanceInNetworkMappings(ctx, mappings, endpoint, resources.NewContainerNetworkAccess()) + if err != nil { + return 0, err + } + return instance.GetPort(), nil + } + } + return gatewayContainerPort, nil +} + +// workloadPodOverlay gives the gateway its own Kubernetes ServiceAccount, named +// after the service (core defaults an unnamed account to the service's DNS +// name) and rendered in the namespace the deployment targets. Running under the +// namespace default would make the gateway's cloud identity the one every other +// workload in that namespace shares: keyless warehouses (workload identity on +// GKE, and the AKS/EKS equivalents) grant on the Kubernetes principal, so a +// grant to `default` is a grant to all of them. With a dedicated account the +// warehouse's IAM can name exactly this workload. +// +// The account carries no annotations: a direct workload-identity grant needs +// none. An environment that declares a runtime identity for this service has +// the CLI project it onto this same account (same name), merging annotations. +func workloadPodOverlay() *services.PodTemplateOverlay { + return &services.PodTemplateOverlay{ServiceAccount: &services.WorkloadServiceAccount{}} +} + +func (s *Builder) Create(ctx context.Context, req *builderv0.CreateRequest) (*builderv0.CreateResponse, error) { + defer s.Wool.Catch() + if err := s.Templates(ctx, s.Information, services.WithFactory(factoryFS)); err != nil { + return s.Builder.CreateError(err) + } + if err := s.CreateEndpoints(ctx); err != nil { + return s.Builder.CreateErrorf(err, "cannot create endpoints") + } + return s.Builder.CreateResponse(ctx, s.Settings) +} + +func (s *Builder) CreateEndpoints(ctx context.Context) error { + grpc, err := resources.LoadGrpcAPI(ctx, nil) + if err != nil { + return s.Wool.Wrapf(err, "cannot load grpc api") + } + endpoint := s.Base.BaseEndpoint(standards.GRPC) + // Reachable by the other modules of the workspace that depend on it. Core's + // reach vocabulary is private, internal and public: the cross-module reach + // service-object-storage declares as "external" is "internal" here, and + // "external" is now a location, which this service's endpoint is not. + endpoint.Visibility = resources.VisibilityInternal + s.GrpcEndpoint, err = resources.NewAPI(ctx, endpoint, resources.ToGrpcAPI(grpc)) + if err != nil { + return s.Wool.Wrapf(err, "cannot create grpc endpoint") + } + s.Endpoints = []*v0.Endpoint{s.GrpcEndpoint} + s.Wool.Debug("created endpoints", wool.Field("endpoints", resources.MakeManyEndpointSummary(s.Endpoints))) + return nil +} + +func (s *Builder) Communicate(stream builderv0.Builder_CommunicateServer) error { + asker := communicate.NewQuestionAsker(stream) + _, err := asker.RunSequence(nil) + return err +} + +//go:embed templates/factory +var factoryFS embed.FS + +//go:embed templates/deployment +var deploymentFS embed.FS diff --git a/cmd/service-warehouse/distribution_test.go b/cmd/service-warehouse/distribution_test.go index 1876dc2..a39d77f 100644 --- a/cmd/service-warehouse/distribution_test.go +++ b/cmd/service-warehouse/distribution_test.go @@ -121,23 +121,93 @@ func contentSignal(path, rel string) (string, error) { return "", nil } -// TestRepositoryShipsNoImage is the gate behind the documented no-image status. -// The moment this repository builds, publishes, or deploys an image, the status -// is false and image-scope SBOM evidence has to land with the image. -func TestRepositoryShipsNoImage(t *testing.T) { +// shippedImage is every path in this repository that builds, publishes, or +// deploys an image, with what imageSignals says about it. The service ships one +// image, the gateway, so these are the pieces of that one image: the recipe that +// builds it, the workflows that build, tag and test it, the manifest of the +// agent that runs it, and the template that deploys it. +// +// A path not listed is an image this repository ships without evidence for it, +// and a second Dockerfile, a compose file or another workflow building an image +// lands here first. The list is exact in both directions: a listed path that no +// longer produces its signal means the image changed shape, and the evidence has +// to be re-checked against it. +var shippedImage = []string{ + ".github/workflows/ci.yml: workflow runs docker build", + ".github/workflows/publish-gateway-image.yml: workflow runs docker/build-push-action", + ".github/workflows/release.yml: workflow runs docker build", + "Dockerfile: container image build definition", + "agent.codefly.yaml: Codefly agent manifest, whose Builder ships image build recipes", + "templates/deployment/kustomize/base/deployment.yaml.tmpl: manifest names a container image to deploy", +} + +// imageCoverage are the tests, in the agent's package at the repository root, +// that hold the gateway image's SBOM evidence: one subject per shipped platform, +// the evidence validated against the shared contract, and the platforms the +// pipeline builds held to the platforms the evidence covers. They are looked up +// by name so that deleting one fails here, where the image is accounted for. +var imageCoverage = []string{ + "TestImageSubjectsCoverEveryPublishedPlatform", + "TestImageEvidenceSatisfiesTheCoverageContract", + "TestShippedPlatformsMatchTheReleasePipeline", +} + +// TestEveryImageThisRepositoryShipsIsCovered is the gate behind the documented +// distribution status. This repository builds, publishes and deploys one +// container image, the gateway, so image-scope SBOM evidence is owed and is +// served by the agent's Builder.SBOM (codefly-dev/core docs/sbom.md: +// BuilderWrapper.SBOMImages, checked by sbom.ValidateCoverage). What this holds +// is that nothing else joins it without that evidence: any image-producing path +// outside shippedImage fails, and the tests that hold the evidence have to exist. +func TestEveryImageThisRepositoryShipsIsCovered(t *testing.T) { found, err := imageSignals(repoRoot) if err != nil { t.Fatal(err) } + listed := map[string]bool{} + for _, signal := range shippedImage { + listed[signal] = true + } + seen := map[string]bool{} for _, signal := range found { - t.Errorf("image-producing path found: %s", signal) + seen[signal] = true + if !listed[signal] { + t.Errorf("image-producing path not accounted for: %s", signal) + } } - if len(found) > 0 { - t.Log("service-warehouse documents no image, so nothing here publishes image SBOM evidence. " + - "Shipping or deploying an image requires a CycloneDX SBOM per final image — OS packages " + - "and application dependencies — bound to the digest and platform actually shipped, served " + - "through the fleet contract (codefly-dev/core docs/sbom.md: BuilderWrapper.SBOMImages, " + - "checked by sbom.ValidateCoverage), then README.md and this gate updated to match.") + for _, signal := range shippedImage { + if !seen[signal] { + t.Errorf("accounted-for image path no longer produces its signal: %s", signal) + } + } + if t.Failed() { + t.Log("service-warehouse ships one image, the gateway. A new way to build or deploy an image " + + "requires a CycloneDX SBOM per final image — OS packages and application dependencies — bound " + + "to the digest and platform actually shipped, served through the fleet contract (codefly-dev/core " + + "docs/sbom.md: BuilderWrapper.SBOMImages, checked by sbom.ValidateCoverage), then README.md and " + + "this list updated to match.") + } + + covered := map[string]bool{} + tests, err := filepath.Glob(filepath.Join(repoRoot, "*_test.go")) + if err != nil { + t.Fatal(err) + } + for _, name := range tests { + content, err := os.ReadFile(name) + if err != nil { + t.Fatal(err) + } + for _, test := range imageCoverage { + if bytes.Contains(content, []byte("func "+test+"(")) { + covered[test] = true + } + } + } + for _, test := range imageCoverage { + if !covered[test] { + t.Errorf("the image's SBOM coverage test %s is gone from the agent package", test) + } } } diff --git a/composition_test.go b/composition_test.go new file mode 100644 index 0000000..7a108e6 --- /dev/null +++ b/composition_test.go @@ -0,0 +1,546 @@ +package main + +import ( + "context" + "os" + "path/filepath" + "slices" + "strings" + "testing" + + "github.com/codefly-dev/core/agents/services" + agenttesting "github.com/codefly-dev/core/agents/testing" + basev0 "github.com/codefly-dev/core/generated/go/codefly/base/v0" + agentv0 "github.com/codefly-dev/core/generated/go/codefly/services/agent/v0" + builderv0 "github.com/codefly-dev/core/generated/go/codefly/services/builder/v0" + "github.com/codefly-dev/core/resources" + "github.com/codefly-dev/core/standards" + "github.com/stretchr/testify/require" + "gopkg.in/yaml.v3" +) + +// withPublishedLock makes the agent see a gateway image that was published, for +// the tests of what it does with one. They have to mean the same thing before +// the first image exists and after, and the file on disk decides neither. +func withPublishedLock(t *testing.T) { + t.Helper() + previous := gatewayImageLockJSON + gatewayImageLockJSON = []byte(`{"name": "` + registry + `/service-warehouse", "digest": "` + digest + `"}`) + t.Cleanup(func() { gatewayImageLockJSON = previous }) +} + +func TestNewService_EmbedsBase(t *testing.T) { + svc := NewService() + require.NotNil(t, svc.Base, "services.Base embedding broken") + require.NotNil(t, svc.Settings) +} + +func TestLoadConfigurationDefaults(t *testing.T) { + svc := NewService() + require.NoError(t, svc.LoadConfiguration(context.Background(), nil)) + // Nothing is defaulted here: an unset value is the gateway's to decide, and + // a default typed in the agent would be a second opinion about it. + require.Equal(t, resolved{}, svc.conf) +} + +func configuration(values map[string]string) *basev0.Configuration { + var configured []*basev0.ConfigurationValue + for key, value := range values { + configured = append(configured, &basev0.ConfigurationValue{Key: key, Value: value}) + } + return &basev0.Configuration{Infos: []*basev0.ConfigurationInformation{{ + Name: configurationGroup, + ConfigurationValues: configured, + }}} +} + +func TestLoadConfigurationRuntimeOverridesSettings(t *testing.T) { + svc := NewService() + svc.Backend, svc.Database, svc.Dataset, svc.WarehouseLocation = "duckdb", "local.duckdb", "scratch", "eu" + + require.NoError(t, svc.LoadConfiguration(context.Background(), configuration(map[string]string{ + "SWH_BACKEND": "bigquery", + "SWH_DATABASE": "example-project", + "SWH_DATASET": "analytics", + "SWH_LOCATION": "us-central1", + }))) + require.Equal(t, resolved{backend: "bigquery", database: "example-project", dataset: "analytics", location: "us-central1"}, svc.conf) + + // A value the environment does not supply keeps the service's own setting. + require.NoError(t, svc.LoadConfiguration(context.Background(), configuration(map[string]string{"SWH_DATASET": "other"}))) + require.Equal(t, resolved{backend: "duckdb", database: "local.duckdb", dataset: "other", location: "eu"}, svc.conf) +} + +// A secret or a path delivered through a YAML block scalar or an env file +// arrives with a trailing newline. Untrimmed, the token is handed to consumers +// in one form and enforced in another, and a path names nothing. +func TestLoadConfigurationNormalizesWhitespace(t *testing.T) { + svc := NewService() + require.NoError(t, svc.LoadConfiguration(context.Background(), configuration(map[string]string{ + "SWH_AUTH_TOKEN": "pinned-token\n", + "SWH_CREDENTIALS_FILE": " /secrets/key.json\n", + "SWH_BACKEND": "mem\n", + }))) + require.Equal(t, "pinned-token", svc.conf.authToken) + require.Equal(t, "/secrets/key.json", svc.conf.credentialsFile) + require.Equal(t, "mem", svc.conf.backend) +} + +func TestResolveGatewayToken(t *testing.T) { + rt := NewRuntime() + + pinned := "operator-pinned" + rt.conf.authToken = pinned + got, err := rt.resolveGatewayToken() + require.NoError(t, err) + require.Equal(t, pinned, got, "an operator who pins a token is not silently overridden") + + rt.conf.authToken = "" + first, err := rt.resolveGatewayToken() + require.NoError(t, err) + second, err := rt.resolveGatewayToken() + require.NoError(t, err) + require.Len(t, first, 48, "24 random bytes, hex encoded") + require.NotEqual(t, first, second, "each run mints a fresh token, which is what revokes the last run's") +} + +func TestResolveServingGRPCEndpoint(t *testing.T) { + endpoint, err := resolveServingGRPCEndpoint(context.Background(), []*basev0.Endpoint{{Name: "grpc", Api: "grpc"}}) + require.NoError(t, err) + require.Equal(t, "grpc", endpoint.Name) + + _, err = resolveServingGRPCEndpoint(context.Background(), nil) + require.Error(t, err, "expected error when no grpc endpoint is present") +} + +func TestConnectionString(t *testing.T) { + require.Equal(t, "grpc://localhost:9465", NewService().createConnectionString("localhost:9465")) +} + +func headlessService(t *testing.T) *Service { + t.Helper() + svc := NewService() + identity := &basev0.ServiceIdentity{Workspace: "workspace", Module: "module", Name: "warehouse", Version: "1.2.3", WorkspacePath: t.TempDir(), RelativeToWorkspace: "."} + require.NoError(t, svc.Base.HeadlessLoad(context.Background(), identity)) + return svc +} + +func connectionValues(t *testing.T, svc *Service) map[string]*basev0.ConfigurationValue { + t.Helper() + conf, err := svc.CreateConnectionConfiguration(context.Background(), nil, + &basev0.NetworkInstance{Address: "localhost:9465", Access: resources.NewNativeNetworkAccess()}) + require.NoError(t, err) + require.Len(t, conf.Infos, 1) + require.Equal(t, configurationGroup, conf.Infos[0].Name) + values := map[string]*basev0.ConfigurationValue{} + for _, value := range conf.Infos[0].ConfigurationValues { + values[value.Key] = value + } + return values +} + +// TestConnectionConfigurationCarriesTokenAsSecret pins how a consumer receives +// the gateway credential: as its own secret-marked value, never folded into the +// connection string or endpoint that get logged and templated everywhere. +func TestConnectionConfigurationCarriesTokenAsSecret(t *testing.T) { + svc := headlessService(t) + const token = "b6f1a0c9d8e7f6a5b4c3d2e1f0a9b8c7" + svc.gatewayToken = token + + values := connectionValues(t, svc) + + require.Contains(t, values, "token") + require.Equal(t, token, values["token"].Value) + require.True(t, values["token"].Secret, "token must be marked secret so it is never logged or displayed") + for _, key := range []string{"connection", "endpoint"} { + require.NotContains(t, values[key].GetValue(), token, "%s leaks the token", key) + } + require.Equal(t, "grpc://localhost:9465", values["connection"].Value) + require.Equal(t, "localhost:9465", values["endpoint"].Value) +} + +// The Builder shares the Service but never generates a token: it must not +// advertise an empty credential a consumer would then try to present. +func TestConnectionConfigurationWithoutTokenOmitsIt(t *testing.T) { + require.NotContains(t, connectionValues(t, headlessService(t)), "token") +} + +func TestAgentInformationDocumentsTheConfigurationGroup(t *testing.T) { + info, err := NewService().GetAgentInformation(context.Background(), &agentv0.AgentInformationRequest{}) + require.NoError(t, err) + require.Len(t, info.GetConfigurationDetails(), 1) + group := info.GetConfigurationDetails()[0] + require.Equal(t, configurationGroup, group.GetName()) + var fields []string + for _, field := range group.GetFields() { + fields = append(fields, field.GetName()) + } + require.Equal(t, []string{"connection", "endpoint", "token"}, fields) + require.Contains(t, info.GetReadMe(), "warehouse") +} + +func TestSettings_YAMLRoundTrip(t *testing.T) { + var s Settings + require.NoError(t, yaml.Unmarshal([]byte("backend: bigquery\ndatabase: example-project\ndataset: analytics\nlocation: eu\n"), &s)) + require.Equal(t, Settings{Backend: "bigquery", Database: "example-project", Dataset: "analytics", WarehouseLocation: "eu"}, s) +} + +func TestDeploymentTemplates(t *testing.T) { + cases := map[string]deploymentTemplateParameters{ + "nothing chosen": {ServicePort: gatewayContainerPort, ContainerPort: gatewayContainerPort}, + "bigquery": { + Backend: "bigquery", Database: "example-project", Dataset: "analytics", Location: "us-central1", + ServicePort: gatewayContainerPort, ContainerPort: gatewayContainerPort, + }, + "another service port": {Backend: "mem", ServicePort: 9090, ContainerPort: gatewayContainerPort}, + } + for name, params := range cases { + t.Run(name, func(t *testing.T) { + dir := agenttesting.AssertKustomizeTemplatesWithOverlay(t, deploymentFS, ¶ms, workloadPodOverlay()) + manifest, err := os.ReadFile(filepath.Join(dir, "base", "deployment.yaml")) + require.NoError(t, err) + body := string(manifest) + + env := deploymentEnv(t, body) + for key, want := range map[string]string{ + "SWH_BACKEND": params.Backend, "SWH_DATABASE": params.Database, + "SWH_DATASET": params.Dataset, "SWH_LOCATION": params.Location, + } { + if want == "" { + require.NotContains(t, env, key, "an unset value must render no variable, leaving the choice to the gateway") + continue + } + require.Equal(t, want, env[key]) + } + require.Equal(t, ":9465", env["SWH_LISTEN"]) + // The gateway refuses an unauthenticated listener unless the manifest + // says so, so a deployment that stops emitting this opt-out without + // also delivering SWH_AUTH_TOKEN would crash-loop. + require.Equal(t, "true", env["SWH_ALLOW_ANONYMOUS"], "the deployed profile must declare its unauthenticated listener") + require.NotContains(t, env, "SWH_AUTH_TOKEN", "the manifest must not carry the gateway credential") + // No rendering names a credentials path: nothing in this manifest + // mounts one, so an emitted path would point at a file the pod does + // not have. Deploy rejects a configured path instead. Matched on the + // bare name so that explaining the absence in rendered YAML fails too. + require.NotContains(t, body, "SWH_CREDENTIALS_FILE") + requireProbeSemantics(t, body) + }) + } +} + +// deploymentEnv reads the gateway container's literal environment from a +// rendered Deployment, refusing a key rendered twice: the CLI's binding of the +// environment's configuration replaces literals by name. +func deploymentEnv(t *testing.T, body string) map[string]string { + t.Helper() + var deployment struct { + Spec struct { + Template struct { + Spec struct { + Containers []struct { + Env []struct { + Name string `yaml:"name"` + Value string `yaml:"value"` + } `yaml:"env"` + } `yaml:"containers"` + } `yaml:"spec"` + } `yaml:"template"` + } `yaml:"spec"` + } + require.NoError(t, yaml.Unmarshal([]byte(body), &deployment)) + require.Len(t, deployment.Spec.Template.Spec.Containers, 1) + env := map[string]string{} + for _, e := range deployment.Spec.Template.Spec.Containers[0].Env { + require.NotContains(t, env, e.Name, "env %s rendered twice; the CLI binding refuses a repeated key", e.Name) + env[e.Name] = e.Value + } + return env +} + +// requireProbeSemantics pins what the rendered probes attest. A TCP probe +// passes as soon as the gRPC port is bound, and the gateway's health service +// reports the process and not the warehouse, so all three probes check the +// overall service: naming one would attest something the gateway does not +// report, and a probe that followed a warehouse outage would drain every replica +// at once, since they share one warehouse and one credential set. +func requireProbeSemantics(t *testing.T, body string) { + t.Helper() + require.NotContains(t, body, "tcpSocket", "probes must not settle for a bound socket") + + var deployment struct { + Spec struct { + Template struct { + Spec struct { + Containers []map[string]any `yaml:"containers"` + } `yaml:"spec"` + } `yaml:"template"` + } `yaml:"spec"` + } + require.NoError(t, yaml.Unmarshal([]byte(body), &deployment)) + require.Len(t, deployment.Spec.Template.Spec.Containers, 1) + container := deployment.Spec.Template.Spec.Containers[0] + for _, name := range []string{"startupProbe", "readinessProbe", "livenessProbe"} { + probe, ok := container[name].(map[string]any) + require.True(t, ok, "%s is missing", name) + grpc, isGRPC := probe["grpc"].(map[string]any) + require.True(t, isGRPC, "%s is not a grpc probe: %v", name, probe) + require.Equal(t, gatewayContainerPort, grpc["port"], name) + require.NotContains(t, grpc, "service", "%s must check the overall service", name) + } +} + +// newDeployBuilder returns a Builder wired for a headless Deploy call, against a +// gateway image that was published. +func newDeployBuilder(t *testing.T, ctx context.Context) *Builder { + t.Helper() + withPublishedLock(t) + builder := NewBuilder() + identity := &basev0.ServiceIdentity{Workspace: "workspace", Module: "module", Name: "warehouse", Version: "1.2.3", WorkspacePath: t.TempDir(), RelativeToWorkspace: "."} + require.NoError(t, builder.Base.HeadlessLoad(ctx, identity)) + builder.Base.Information = &services.Information{Service: resources.ToServiceWithCase(resources.ServiceIdentityFromProto(identity))} + builder.Base.EnvironmentVariables.SetIdentity(identity) + return builder +} + +// deployRequest builds a Kubernetes DeploymentRequest carrying the given +// warehouse configuration values, rendering into destination. +func deployRequest(destination string, values map[string]string, profile builderv0.KubernetesOutputProfile) *builderv0.DeploymentRequest { + return &builderv0.DeploymentRequest{ + Environment: &basev0.Environment{Name: "test", Fixture: "dev-admin"}, + Deployment: &builderv0.Deployment{Kind: &builderv0.Deployment_Kubernetes{ + Kubernetes: &builderv0.KubernetesDeployment{ + Namespace: "codefly", + Destination: destination, + Profile: profile, + }, + }}, + Configuration: configuration(values), + } +} + +const ( + ephemeral = builderv0.KubernetesOutputProfile_KUBERNETES_OUTPUT_PROFILE_EPHEMERAL_LOCAL_APPLY_V1 + restricted = builderv0.KubernetesOutputProfile_KUBERNETES_OUTPUT_PROFILE_RESTRICTED_PORTABLE_V1 +) + +func renderedDeployment(t *testing.T, destination string) string { + t.Helper() + body, err := os.ReadFile(filepath.Join(destination, "base", "deployment.yaml")) + require.NoError(t, err) + return string(body) +} + +// TestDeployResolvesConfigurationIntoTheManifest drives the real Builder.Deploy +// path and asserts the rendered manifest names the configured values: the +// regression guard for a Deploy that never loaded the configuration, leaving the +// manifest on its zero value whatever the environment said. Every key goes +// through the full config-key -> resolved -> template wiring, so a typo'd map key +// fails here rather than shipping green. +func TestDeployResolvesConfigurationIntoTheManifest(t *testing.T) { + ctx := context.Background() + builder := newDeployBuilder(t, ctx) + destination := t.TempDir() + + resp, err := builder.Deploy(ctx, deployRequest(destination, map[string]string{ + "SWH_BACKEND": "bigquery", + "SWH_DATABASE": "example-project", + "SWH_DATASET": "analytics", + "SWH_LOCATION": "us-central1", + }, ephemeral)) + require.NoError(t, err) + require.Equal(t, builderv0.DeploymentStatus_SUCCESS, resp.GetState().GetState(), resp.GetState().GetMessage()) + + env := deploymentEnv(t, renderedDeployment(t, destination)) + require.Equal(t, "bigquery", env["SWH_BACKEND"]) + require.Equal(t, "example-project", env["SWH_DATABASE"]) + require.Equal(t, "analytics", env["SWH_DATASET"]) + require.Equal(t, "us-central1", env["SWH_LOCATION"]) + require.Equal(t, "warehouse", env["CODEFLY__SERVICE"], "the CLI binds environment configuration onto the container declaring this") +} + +// The composed service's own settings are what a render with no environment +// values carries: the environment's configuration is bound onto the container +// after the render, replacing the SWH_* literals by name. +func TestDeployRendersTheServiceSettingsWhenTheEnvironmentSuppliesNone(t *testing.T) { + ctx := context.Background() + builder := newDeployBuilder(t, ctx) + builder.Settings.Backend = "bigquery" + destination := t.TempDir() + + resp, err := builder.Deploy(ctx, deployRequest(destination, nil, restricted)) + require.NoError(t, err) + require.Equal(t, builderv0.DeploymentStatus_SUCCESS, resp.GetState().GetState(), resp.GetState().GetMessage()) + require.Equal(t, "bigquery", deploymentEnv(t, renderedDeployment(t, destination))["SWH_BACKEND"]) +} + +func TestDeployRefusesWhatItHasNoWayToDeliver(t *testing.T) { + ctx := context.Background() + for name, tc := range map[string]struct { + values map[string]string + want string + }{ + "a credentials file nothing mounts": {map[string]string{"SWH_CREDENTIALS_FILE": "/keys/sa.json"}, "SWH_CREDENTIALS_FILE cannot be delivered"}, + "a token the consumers would lack": {map[string]string{"SWH_AUTH_TOKEN": "pinned"}, "SWH_AUTH_TOKEN cannot be delivered"}, + "a value that would write the manifest": { + map[string]string{"SWH_DATASET": "x\"\n - name: SWH_AUTH_TOKEN"}, "SWH_DATASET", + }, + "a templating action": {map[string]string{"SWH_DATABASE": "{{ .Image }}"}, "SWH_DATABASE"}, + } { + t.Run(name, func(t *testing.T) { + builder := newDeployBuilder(t, ctx) + destination := t.TempDir() + + resp, err := builder.Deploy(ctx, deployRequest(destination, tc.values, ephemeral)) + require.NoError(t, err) + require.Equal(t, builderv0.DeploymentStatus_ERROR, resp.GetState().GetState()) + require.Contains(t, resp.GetState().GetMessage(), tc.want) + entries, err := os.ReadDir(destination) + require.NoError(t, err) + require.Empty(t, entries, "a refused deploy must write nothing") + }) + } +} + +// With no digest recorded there is no image to deploy, and rendering one under +// a floating name is exactly what pinning exists to prevent. +func TestDeployRefusesWhileNoGatewayImageIsPublished(t *testing.T) { + ctx := context.Background() + builder := newDeployBuilder(t, ctx) + gatewayImageLockJSON = []byte(`{"name": "` + registry + `/service-warehouse", "digest": ""}`) + destination := t.TempDir() + + resp, err := builder.Deploy(ctx, deployRequest(destination, nil, restricted)) + require.NoError(t, err) + require.Equal(t, builderv0.DeploymentStatus_ERROR, resp.GetState().GetState()) + require.Contains(t, resp.GetState().GetMessage(), "no gateway image has been published") +} + +// TestDeployRestrictedBindsTheWorkloadIdentity pins what infra has to grant +// against: in the restricted profile a hosted cell renders, the pod runs under +// its own ServiceAccount, named after the service, in the target namespace, so a +// warehouse IAM member for that Kubernetes principal authorizes exactly this +// workload and nothing else; and the image is the digest the lock records. +func TestDeployRestrictedBindsTheWorkloadIdentity(t *testing.T) { + ctx := context.Background() + builder := newDeployBuilder(t, ctx) + destination := t.TempDir() + + resp, err := builder.Deploy(ctx, deployRequest(destination, map[string]string{ + "SWH_BACKEND": "bigquery", "SWH_DATABASE": "example-cell-platform", + }, restricted)) + require.NoError(t, err) + require.Equal(t, builderv0.DeploymentStatus_SUCCESS, resp.GetState().GetState(), resp.GetState().GetMessage()) + + files := map[string]string{} + require.NoError(t, filepath.WalkDir(destination, func(path string, d os.DirEntry, err error) error { + if err != nil || d.IsDir() { + return err + } + data, err := os.ReadFile(path) + rel, _ := filepath.Rel(destination, path) + files[rel] = string(data) + return err + })) + // No Namespace and no Secret in a restricted tree; nothing secret-shaped. + for rel, body := range files { + require.NotContains(t, body, "kind: Namespace", rel) + require.NotContains(t, body, "kind: Secret", rel) + require.NotContains(t, body, "SWH_CREDENTIALS_FILE", rel) + require.NotContains(t, body, "SWH_PASSWORD", rel) + require.NotContains(t, body, "envFrom", rel) + } + + account, ok := files[filepath.Join("base", "serviceaccount.yaml")] + require.True(t, ok, "restricted render must carry the workload's ServiceAccount; files: %v", slices.Sorted(func(yield func(string) bool) { + for name := range files { + if !yield(name) { + return + } + } + })) + var sa struct { + Kind string `yaml:"kind"` + Metadata struct { + Name string `yaml:"name"` + Namespace string `yaml:"namespace"` + Annotations map[string]string `yaml:"annotations"` + } `yaml:"metadata"` + } + require.NoError(t, yaml.Unmarshal([]byte(account), &sa)) + require.Equal(t, "ServiceAccount", sa.Kind) + require.Equal(t, "warehouse", sa.Metadata.Name) + require.Equal(t, "codefly", sa.Metadata.Namespace) + require.Empty(t, sa.Metadata.Annotations, "a direct workload-identity grant needs no cloud-account annotation") + + body := files[filepath.Join("base", "deployment.yaml")] + var deployment struct { + Spec struct { + Template struct { + Spec struct { + ServiceAccountName string `yaml:"serviceAccountName"` + Containers []struct { + Image string `yaml:"image"` + } `yaml:"containers"` + } `yaml:"spec"` + } `yaml:"template"` + } `yaml:"spec"` + } + require.NoError(t, yaml.Unmarshal([]byte(body), &deployment)) + require.Equal(t, "warehouse", deployment.Spec.Template.Spec.ServiceAccountName) + require.Equal(t, registry+"/service-warehouse@"+digest, deployment.Spec.Template.Spec.Containers[0].Image) + requireProbeSemantics(t, body) +} + +// TestDeployServicePublishesAllocatedPort: core allocates the gRPC endpoint an +// in-cluster port and hands consumers that port, so the Service must publish it +// and forward it to the container's listener. +func TestDeployServicePublishesAllocatedPort(t *testing.T) { + ctx := context.Background() + builder := newDeployBuilder(t, ctx) + builder.GrpcEndpoint = &basev0.Endpoint{Name: "grpc", Module: "module", Service: "warehouse", Api: standards.GRPC} + destination := t.TempDir() + req := deployRequest(destination, nil, restricted) + instance := resources.NewNetworkInstance("warehouse.codefly.svc.cluster.local", 9090) + instance.Access = resources.NewContainerNetworkAccess() + req.NetworkMappings = []*basev0.NetworkMapping{ + {Endpoint: builder.GrpcEndpoint, Instances: []*basev0.NetworkInstance{instance}}, + // Another service's endpoint is never published by this Service. + { + Endpoint: &basev0.Endpoint{Name: "grpc", Module: "module", Service: "other", Api: standards.GRPC}, + Instances: []*basev0.NetworkInstance{instance}, + }, + } + resp, err := builder.Deploy(ctx, req) + require.NoError(t, err) + require.Equal(t, builderv0.DeploymentStatus_SUCCESS, resp.GetState().GetState(), resp.GetState().GetMessage()) + service, err := os.ReadFile(filepath.Join(destination, "base", "service.yaml")) + require.NoError(t, err) + require.Contains(t, string(service), " - name: grpc\n port: 9090\n targetPort: 9465\n") +} + +// Without a mapping for the endpoint the Service keeps publishing the container +// port. +func TestDeployServiceFallsBackToContainerPort(t *testing.T) { + ctx := context.Background() + builder := newDeployBuilder(t, ctx) + destination := t.TempDir() + + resp, err := builder.Deploy(ctx, deployRequest(destination, nil, ephemeral)) + require.NoError(t, err) + require.Equal(t, builderv0.DeploymentStatus_SUCCESS, resp.GetState().GetState(), resp.GetState().GetMessage()) + service, err := os.ReadFile(filepath.Join(destination, "base", "service.yaml")) + require.NoError(t, err) + require.Contains(t, string(service), " - name: grpc\n port: 9465\n targetPort: 9465\n") +} + +// The endpoint a Create scaffolds must be one core's endpoint model accepts, and +// reachable from the modules that depend on this service. +func TestCreateEndpointsDeclaresAnInternalGRPCEndpoint(t *testing.T) { + builder := newDeployBuilder(t, context.Background()) + require.NoError(t, builder.CreateEndpoints(context.Background())) + require.Len(t, builder.Endpoints, 1) + endpoint := builder.Endpoints[0] + require.Equal(t, standards.GRPC, endpoint.GetApi()) + require.Equal(t, resources.VisibilityInternal, endpoint.GetVisibility()) + require.True(t, strings.HasPrefix(endpoint.GetService(), "warehouse"), endpoint.GetService()) +} diff --git a/gateway-image.json b/gateway-image.json new file mode 100644 index 0000000..24d8284 --- /dev/null +++ b/gateway-image.json @@ -0,0 +1,4 @@ +{ + "name": "ghcr.io/codefly-dev/service-warehouse", + "digest": "" +} diff --git a/gateway_image.go b/gateway_image.go new file mode 100644 index 0000000..823b964 --- /dev/null +++ b/gateway_image.go @@ -0,0 +1,93 @@ +package main + +import ( + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "os" + "strings" + + "github.com/codefly-dev/core/resources" +) + +// gatewayImageName is the repository, inside core's image registry, this +// service's gateway image is published to. The release workflows derive the same +// name from the repository they run in (ghcr.io//), and +// TestGatewayImageNameIsThisRepository holds the two together. +const gatewayImageName = "service-warehouse" + +// errGatewayImageUnpublished means gateway-image.json records no digest: the +// image has not been published for any version yet. It is a state the tree +// legitimately sits in between merging the agent and publishing its first +// image, and the only safe reading of it is a refusal. A reference with no +// digest and no tag is the registry's :latest, which is whatever was pushed +// last and which the evidence contract refuses to inventory. +var errGatewayImageUnpublished = errors.New("no gateway image has been published: gateway-image.json records no digest. " + + "Run publish-gateway-image.yml for the upcoming version and commit the digest it prints") + +// gatewayImageLock is the recorded identity of the published gateway image. It +// carries no tag: the tag is the agent version, and a second copy of it is one +// more thing to disagree with agent.codefly.yaml. +type gatewayImageLock struct { + Name string `json:"name"` + Digest string `json:"digest"` +} + +// parseGatewayImageLock returns the image the lock pins. The registry comes from +// core, never from the lock: a lock naming another registry is the lock being +// wrong, not an instruction to run an image from there. +func parseGatewayImageLock(content []byte) (*resources.DockerImage, error) { + var lock gatewayImageLock + if err := json.Unmarshal(content, &lock); err != nil { + return nil, fmt.Errorf("parse gateway image lock: %w", err) + } + image := resources.PublishedImage(gatewayImageName, "") + if want := image.Repository + "/" + image.Name; lock.Name != want { + return nil, fmt.Errorf("gateway image lock names %q, but this service's gateway image is %q", lock.Name, want) + } + if lock.Digest == "" { + return nil, errGatewayImageUnpublished + } + if !isSHA256Digest(lock.Digest) { + return nil, fmt.Errorf("gateway image digest %q must be a sha256 digest", lock.Digest) + } + image.Digest = lock.Digest + return &image, nil +} + +// isSHA256Digest reports whether digest is a complete sha256 image digest. +func isSHA256Digest(digest string) bool { + algorithm, encoded, found := strings.Cut(digest, ":") + decoded, err := hex.DecodeString(encoded) + return found && algorithm == "sha256" && err == nil && len(decoded) == 32 +} + +// gatewayImageOverrideEnv names a locally built gateway image to run instead of +// the published one. It configures the agent, not the gateway: the gateway's own +// surface is the SWH_* variables internal/config reads. +const gatewayImageOverrideEnv = "SWH_GATEWAY_IMAGE" + +// publishedGatewayImage is the image gateway-image.json pins, whatever the +// override says. A deployment, an audit and a source-scope inventory describe +// the image a release ships, which a locally built stand-in is not. +func publishedGatewayImage() (*resources.DockerImage, error) { + return parseGatewayImageLock(gatewayImageLockJSON) +} + +// effectiveGatewayImage resolves the gateway image the agent actually runs, and +// reports whether it came from the override. An override is built locally and +// never pushed, so it is reachable through the Docker daemon rather than a +// registry and carries the single platform the daemon holds. +// +// The published image is pinned to the digest gateway-image.json records. A tag +// serves whatever was pushed to it last, so an agent naming one would run, and +// report evidence for, an image nobody checked was the one built for this +// version. +func effectiveGatewayImage() (*resources.DockerImage, bool, error) { + if override := os.Getenv(gatewayImageOverrideEnv); override != "" { + return &resources.DockerImage{Name: override}, true, nil + } + image, err := publishedGatewayImage() + return image, false, err +} diff --git a/gateway_image_test.go b/gateway_image_test.go new file mode 100644 index 0000000..01abe0e --- /dev/null +++ b/gateway_image_test.go @@ -0,0 +1,173 @@ +package main + +import ( + "fmt" + "os" + "os/exec" + "regexp" + "strings" + "testing" + + "github.com/codefly-dev/core/agents" + "github.com/codefly-dev/core/resources" + "github.com/stretchr/testify/require" + + "github.com/codefly-dev/service-warehouse/internal/auth" + "github.com/codefly-dev/service-warehouse/internal/config" + + // The mem backend is registered here, in a test, so config.FromEnv can + // resolve a configuration without a warehouse. The agent itself imports no + // backend: see TestAgentLinksNoBackendDriver. + _ "github.com/codefly-dev/service-warehouse/internal/backend/mem" +) + +const ( + registry = "ghcr.io/codefly-dev" + digest = "sha256:1205da0743e56b68c12c754438476e8fdd0f05cdb08ce1c9687ddb8d8f0e1669" +) + +func TestParseGatewayImageLock(t *testing.T) { + image, err := parseGatewayImageLock([]byte(`{ + "name": "` + registry + `/service-warehouse", + "digest": "` + digest + `" + }`)) + require.NoError(t, err) + require.Equal(t, registry+"/service-warehouse@"+digest, image.FullName()) +} + +// A lock the agent accepts but that names no immutable image is the failure +// pinning exists to prevent: it produces subjects whose evidence describes +// whatever the registry served, so every malformed shape is refused at load +// rather than carried into a scan. +func TestParseGatewayImageLockRejectsAnythingButASHA256Digest(t *testing.T) { + name := registry + "/service-warehouse" + for label, lock := range map[string]string{ + "no name": `{"digest": "` + digest + `"}`, + "another image": `{"name": "` + registry + `/service-object-storage", "digest": "` + digest + `"}`, + "another registry": `{"name": "registry.example.com/codefly-dev/service-warehouse", "digest": "` + digest + `"}`, + "tag as digest": `{"name": "` + name + `", "digest": "0.0.3"}`, + "other algorithm": `{"name": "` + name + `", "digest": "sha512:1205da07"}`, + "truncated": `{"name": "` + name + `", "digest": "sha256:1205da07"}`, + "not hex": `{"name": "` + name + `", "digest": "sha256:zzzzda0743e56b68c12c754438476e8fdd0f05cdb08ce1c9687ddb8d8f0e1669"}`, + "not json": `sha256:1205da07`, + } { + t.Run(label, func(t *testing.T) { + _, err := parseGatewayImageLock([]byte(lock)) + require.Error(t, err, "accepted a lock that pins no image") + require.NotErrorIs(t, err, errGatewayImageUnpublished) + }) + } +} + +// A lock with no digest is the tree before its first image was published. A +// reference with no digest and no tag is the registry's :latest, so the only +// safe reading is a refusal that says what to do. +func TestAnUnpublishedLockIsRefusedNeverRunAsLatest(t *testing.T) { + image, err := parseGatewayImageLock([]byte(`{"name": "` + registry + `/service-warehouse", "digest": ""}`)) + require.ErrorIs(t, err, errGatewayImageUnpublished) + require.Nil(t, image) + require.Contains(t, err.Error(), "publish-gateway-image.yml") +} + +// The embedded copy is what the agent runs; the file on disk is what the +// release reads to tag and to inventory. They are the same bytes, and a test +// reading the file keeps the embed from silently going stale. +func TestGatewayImageMatchesTheLockOnDisk(t *testing.T) { + lock, err := os.ReadFile("gateway-image.json") + require.NoError(t, err) + require.Equal(t, string(lock), string(gatewayImageLockJSON)) + + want, wantErr := parseGatewayImageLock(lock) + got, gotErr := publishedGatewayImage() + require.Equal(t, wantErr, gotErr) + if wantErr == nil { + require.Equal(t, want.FullName(), got.FullName()) + } +} + +// The override names a locally built image for a run. It must never decide what +// a deployment or an audit describes, which is the image a release ships. +func TestTheOverrideDecidesTheRunButNotThePublishedImage(t *testing.T) { + t.Setenv(gatewayImageOverrideEnv, "service-warehouse:local") + + image, overridden, err := effectiveGatewayImage() + require.NoError(t, err) + require.True(t, overridden) + require.Equal(t, "service-warehouse:local", image.FullName()) + + published, err := publishedGatewayImage() + if err == nil { + require.NotEqual(t, "service-warehouse:local", published.FullName()) + } else { + require.ErrorIs(t, err, errGatewayImageUnpublished) + } + + t.Setenv(gatewayImageOverrideEnv, "") + _, overridden, _ = effectiveGatewayImage() + require.False(t, overridden) +} + +// The image's name is derived, not typed: the agent takes the registry from core +// and the repository name from gatewayImageName, while the workflows publish to +// the repository they run in. This is where those meet, so that renaming the +// repository, or the constant, fails here instead of publishing one image and +// running another. +func TestGatewayImageNameIsThisRepository(t *testing.T) { + gomod, err := os.ReadFile("go.mod") + require.NoError(t, err) + module := regexp.MustCompile(`(?m)^module github\.com/(\S+)$`).FindSubmatch(gomod) + require.NotNil(t, module, "go.mod names no github.com module") + repository := string(module[1]) + + image := resources.PublishedImage(gatewayImageName, "") + require.Equal(t, "ghcr.io/"+repository, image.Repository+"/"+image.Name, + "the agent runs an image other than the one this repository publishes") + + lock, err := os.ReadFile("gateway-image.json") + require.NoError(t, err) + require.Contains(t, string(lock), `"name": "ghcr.io/`+repository+`"`) + + for _, workflow := range []string{"publish-gateway-image.yml", "release.yml"} { + require.Contains(t, workflowBody(t, workflow), "IMAGE: ghcr.io/${{ github.repository }}", workflow) + } +} + +// The container port is stated once in the agent and handed to the gateway as +// SWH_LISTEN and to the templates as the container port. The gateway's own +// default and the image's EXPOSE are the other two statements of it, and none of +// the three is derived from another. +func TestGatewayContainerPortIsTheGatewaysDefault(t *testing.T) { + t.Setenv("SWH_BACKEND", "mem") + t.Setenv("SWH_ALLOW_ANONYMOUS", "true") + t.Setenv("SWH_AUTH_TOKEN", "") + t.Setenv("SWH_LISTEN", "") + + cfg, err := config.FromEnv() + require.NoError(t, err) + require.Equal(t, fmt.Sprintf(":%d", gatewayContainerPort), cfg.ListenAddr) + + dockerfile, err := os.ReadFile("Dockerfile") + require.NoError(t, err) + require.Regexp(t, regexp.MustCompile(fmt.Sprintf(`(?m)^EXPOSE %d$`, gatewayContainerPort)), string(dockerfile)) +} + +// The agent presents the token under the key the gateway enforces. internal/auth +// redeclares the key because the gateway must not link core's agent runtime; the +// agent does link it, so this is where a divergence is caught. +func TestTheTokenKeyIsTheOneCoreAuthenticatesAgentsWith(t *testing.T) { + require.Equal(t, agents.AuthMetadataKey, auth.MetadataKey) +} + +// The agent is released without cgo (.goreleaser.yaml) and must stay free of the +// gateway's backends: the embedded DuckDB engine is a cgo driver, and linking it +// here would turn the release's CGO_ENABLED=0 build into one that fails the day +// the driver lands. +func TestAgentLinksNoBackendDriver(t *testing.T) { + out, err := exec.Command("go", "list", "-deps", "-f", "{{.ImportPath}}", ".").Output() + require.NoError(t, err) + for _, dependency := range strings.Fields(string(out)) { + require.False(t, strings.Contains(dependency, "service-warehouse/internal/backend") || + strings.Contains(dependency, "duckdb"), + "the agent links %s, a backend the gateway carries", dependency) + } +} diff --git a/go.mod b/go.mod index e41790f..0d39d12 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,7 @@ require ( github.com/stretchr/testify v1.12.1 google.golang.org/grpc v1.83.2 google.golang.org/protobuf v1.36.12 + gopkg.in/yaml.v3 v3.0.1 ) require ( @@ -14,9 +15,32 @@ require ( buf.build/go/protovalidate v1.4.0 // indirect cel.dev/cel-go v0.32.0 // indirect cel.dev/expr v0.25.3 // indirect + github.com/Azure/go-ansiterm v0.0.0-20261006220739-8c912ac31dd4 // indirect github.com/Masterminds/semver v1.5.0 // indirect github.com/Masterminds/semver/v3 v3.5.0 // indirect + github.com/Microsoft/go-winio v0.6.2 // indirect + github.com/VividCortex/ewma v1.2.0 // indirect github.com/antlr4-go/antlr/v4 v4.13.1 // indirect + github.com/blang/semver v3.5.1+incompatible // indirect + github.com/blang/semver/v4 v4.0.0 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/cheggaaa/pb/v3 v3.2.1 // indirect + github.com/clipperhouse/uax29/v2 v2.7.0 // indirect + github.com/codefly-dev/gortk v0.2.0 // indirect + github.com/containerd/errdefs v1.0.0 // indirect + github.com/containerd/errdefs/pkg v0.3.0 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/distribution/reference v0.6.0 // indirect + github.com/docker/docker v28.5.2+incompatible // indirect + github.com/docker/go-connections v0.8.1 // indirect + github.com/docker/go-units v0.5.0 // indirect + github.com/fatih/color v1.19.0 // indirect + github.com/felixge/httpsnoop v1.1.0 // indirect + github.com/fsnotify/fsnotify v1.10.1 // indirect + github.com/go-errors/errors v1.4.2 // indirect + github.com/go-logr/logr v1.4.4 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/go-ole/go-ole v1.3.0 // indirect github.com/go-openapi/analysis v1.0.0 // indirect github.com/go-openapi/errors v0.22.8 // indirect github.com/go-openapi/jsonpointer v1.0.1 // indirect @@ -24,22 +48,55 @@ require ( github.com/go-openapi/loads v0.25.3 // indirect github.com/go-openapi/spec v1.0.1 // indirect github.com/go-openapi/strfmt v0.27.2 // indirect + github.com/go-openapi/swag v0.28.0 // indirect + github.com/go-openapi/swag/cmdutils v0.28.0 // indirect github.com/go-openapi/swag/conv v0.29.1 // indirect + github.com/go-openapi/swag/fileutils v0.28.0 // indirect github.com/go-openapi/swag/jsonutils v0.29.1 // indirect github.com/go-openapi/swag/loading v0.29.1 // indirect github.com/go-openapi/swag/mangling v0.29.1 // indirect + github.com/go-openapi/swag/netutils v0.28.0 // indirect github.com/go-openapi/swag/pools v0.29.1 // indirect github.com/go-openapi/swag/stringutils v0.29.1 // indirect github.com/go-openapi/swag/typeutils v0.29.1 // indirect github.com/go-openapi/swag/yamlutils v0.29.1 // indirect github.com/go-viper/mapstructure/v2 v2.5.0 // indirect + github.com/gofrs/flock v0.13.1 // indirect + github.com/google/gnostic-models v0.6.9 // indirect + github.com/google/go-github/v89 v89.0.0 // indirect + github.com/google/go-querystring v1.2.0 // indirect github.com/google/uuid v1.6.0 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect + github.com/kr/text v0.2.0 // indirect + github.com/lufia/plan9stats v0.0.0-20260330125221-c963978e514e // indirect + github.com/mattn/go-colorable v0.1.15 // indirect + github.com/mattn/go-isatty v0.0.24 // indirect + github.com/mattn/go-runewidth v0.0.27 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect + github.com/moby/docker-image-spec v1.3.1 // indirect + github.com/moby/patternmatcher v0.6.1 // indirect + github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 // indirect github.com/oklog/ulid/v2 v2.1.2 // indirect + github.com/opencontainers/go-digest v1.0.0 // indirect + github.com/opencontainers/image-spec v1.1.1 // indirect github.com/pkg/errors v0.9.1 // indirect + github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect + github.com/santhosh-tekuri/jsonschema/v6 v6.0.3 // indirect + github.com/shirou/gopsutil/v3 v3.24.5 // indirect + github.com/shoenig/go-m1cpu v0.2.1 // indirect + github.com/tklauser/go-sysconf v0.4.0 // indirect + github.com/tklauser/numcpus v0.12.0 // indirect + github.com/xlab/treeprint v1.2.0 // indirect github.com/yoheimuta/go-protoparser/v4 v4.14.2 // indirect + github.com/yusufpapurcu/wmi v1.2.4 // indirect + go.opentelemetry.io/auto/sdk v1.2.1 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.71.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.70.0 // indirect + go.opentelemetry.io/otel v1.46.0 // indirect + go.opentelemetry.io/otel/metric v1.46.0 // indirect + go.opentelemetry.io/otel/trace v1.46.0 // indirect + go.yaml.in/yaml/v2 v2.4.2 // indirect go.yaml.in/yaml/v3 v3.0.5 // indirect golang.org/x/exp v0.0.0-20260820142414-ca536658362e // indirect golang.org/x/net v0.59.0 // indirect @@ -48,5 +105,9 @@ require ( golang.org/x/text v0.42.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20260819154853-08b0e4226688 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20260825221802-da73d73af1c5 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect + gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect + k8s.io/kube-openapi v0.0.0-20241212222426-2c72e554b1e7 // indirect + sigs.k8s.io/kustomize/api v0.21.1 // indirect + sigs.k8s.io/kustomize/kyaml v0.21.1 // indirect + sigs.k8s.io/yaml v1.5.0 // indirect ) diff --git a/go.sum b/go.sum index 6edcaa5..800ce44 100644 --- a/go.sum +++ b/go.sum @@ -6,12 +6,18 @@ cel.dev/cel-go v0.32.0 h1:irvpFKr5EuGPyxeME03ERh0rii1TX+BDAnB9eL3IvNk= cel.dev/cel-go v0.32.0/go.mod h1:DnVip7tpJSsgZymwfT+m1tnEVy3ivAjSMXPx12YrMkU= cel.dev/expr v0.25.3 h1:A2jO8jwOugrrovveCWfj0KEZOfqiLgAcwjpHPhzIGw0= cel.dev/expr v0.25.3/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4= +dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8= +dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA= +github.com/Azure/go-ansiterm v0.0.0-20261006220739-8c912ac31dd4 h1:m2xcjRABtG288vnlV3U/PKh/HnPURxZp50RKNUwBxM8= +github.com/Azure/go-ansiterm v0.0.0-20261006220739-8c912ac31dd4/go.mod h1:MTUE7ZuZO89FAtAwozGg6AWD83pcdefnEq5gF82MB8s= github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3QEww= github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= github.com/Masterminds/semver/v3 v3.5.0 h1:kQceYJfbupGfZOKZQg0kou0DgAKhzDg2NZPAwZ/2OOE= github.com/Masterminds/semver/v3 v3.5.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= +github.com/ProtonMail/go-crypto v1.4.1 h1:9RfcZHqEQUvP8RzecWEUafnZVtEvrBVL9BiF67IQOfM= +github.com/ProtonMail/go-crypto v1.4.1/go.mod h1:e1OaTyu5SYVrO9gKOEhTc+5UcXtTUa+P3uLudwcgPqo= github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow= github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4= github.com/antlr4-go/antlr/v4 v4.13.1 h1:SqQKkuVZ+zWkMMNkjy5FZe5mr5WURWnlpmOuzYWrPrQ= @@ -22,12 +28,16 @@ github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= github.com/brianvoe/gofakeit/v6 v6.28.0 h1:Xib46XXuQfmlLS2EXRuJpqcw8St6qSZz75OUo0tgAW4= github.com/brianvoe/gofakeit/v6 v6.28.0/go.mod h1:Xj58BMSnFqcn/fAQeSK+/PLtC5kSb7FJIq4JyGa8vEs= +github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM= +github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cheggaaa/pb/v3 v3.2.1 h1:aprZbFRG+B7+ug76S8QZ6Y1PW168UHzOmbC3wa+aU6I= github.com/cheggaaa/pb/v3 v3.2.1/go.mod h1:U9hSVxoKqJrZIE3PkFG1xXXNaK/Ilzg+EF/scpXzEdw= github.com/clipperhouse/uax29/v2 v2.7.0 h1:+gs4oBZ2gPfVrKPthwbMzWZDaAFPGYK72F0NJv2v7Vk= github.com/clipperhouse/uax29/v2 v2.7.0/go.mod h1:EFJ2TJMRUaplDxHKj1qAEhCtQPW2tJSwu5BF98AuoVM= +github.com/cloudflare/circl v1.6.3 h1:9GPOhQGF9MCYUeXyMYlqTR6a5gTrgR/fBLXvUgtVcg8= +github.com/cloudflare/circl v1.6.3/go.mod h1:2eXP6Qfat4O/Yhh8BznvKnJ+uzEoTQ6jVKJRn81BiS4= github.com/codefly-dev/core v0.16.0 h1:iJQ9SteSBRaGs+jeD/DqgKMsukJfTKvnqcYTjsOBjDs= github.com/codefly-dev/core v0.16.0/go.mod h1:rp4YZkCJEOufliWQAgK3sv8YlO7Xcva7+cN7hIlAR0I= github.com/codefly-dev/gortk v0.2.0 h1:7bOlS5valYz2zil+fZctQNcPCYBcPj86abcw9N8h1hQ= @@ -36,16 +46,26 @@ github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M= github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE= github.com/containerd/errdefs/pkg v0.3.0/go.mod h1:NJw6s9HwNuRhnjJhM7pylWwMyAkmCQvQ4GpJHEqRLVk= +github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= +github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/cyphar/filepath-securejoin v0.7.0 h1:s0Y3ITPy6sQn5xt54DuYvTF8hu134ooYLUb58DX/HjE= +github.com/cyphar/filepath-securejoin v0.7.0/go.mod h1:ymLGms/u3BYaviIiuKFnUx8EkQEZeK6cInNoAPJA3o4= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= +github.com/dlclark/regexp2 v1.12.0 h1:0j4c5qQmnC6XOWNjP3PIXURXN2gWx76rd3KvgdPkCz8= +github.com/dlclark/regexp2 v1.12.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= github.com/docker/docker v28.5.2+incompatible h1:DBX0Y0zAjZbSrm1uzOkdr1onVghKaftjlSWt4AFexzM= github.com/docker/docker v28.5.2+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/go-connections v0.8.1 h1:JibmG5hULs5qXSr/cp/w3Pw5fZuStt4MOHMUExb29/M= github.com/docker/go-connections v0.8.1/go.mod h1:no1qkHdjq7kLMGUXYAduOhYPSJxxvgWBh7ogVvptn3Q= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= +github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ= github.com/fatih/color v1.19.0 h1:Zp3PiM21/9Ld6FzSKyL5c/BULoe/ONr9KlbYVOfG8+w= github.com/fatih/color v1.19.0/go.mod h1:zNk67I0ZUT1bEGsSGyCZYZNrHuTkJJB+r6Q9VuMi0LE= github.com/felixge/httpsnoop v1.1.0 h1:3YtUj32ZZkqZtt3sZZsClsymw/QDuVfpNhoA31zeORc= @@ -54,10 +74,18 @@ github.com/fsnotify/fsnotify v1.10.1 h1:b0/UzAf9yR5rhf3RPm9gf3ehBPpf0oZKIjtpKrx5 github.com/fsnotify/fsnotify v1.10.1/go.mod h1:TLheqan6HD6GBK6PrDWyDPBaEV8LspOxvPSjC+bVfgo= github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= +github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 h1:+zs/tPmkDkHx3U66DAb0lQFJrpS6731Oaa12ikc+DiI= +github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376/go.mod h1:an3vInlBmSxCcxctByoQdvwPiA7DTK7jaaFDBTtu0ic= +github.com/go-git/go-billy/v5 v5.9.0 h1:jItGXszUDRtR/AlferWPTMN4j38BQ88XnXKbilmmBPA= +github.com/go-git/go-billy/v5 v5.9.0/go.mod h1:jCnQMLj9eUgGU7+ludSTYoZL/GGmii14RxKFj7ROgHw= +github.com/go-git/go-git/v5 v5.19.2 h1:wkfn7vOlUBu8ivAWKBWisTiwJK4jYHzTF8Ndv1LyGqY= +github.com/go-git/go-git/v5 v5.19.2/go.mod h1:QqCBE1EFN5ddFmrliLQ3/ntRCUjZU3EJuwuB/jWEHjk= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.4 h1:tG4xh9yMsRCAiodLVTxyrkzSZ9+o0L1Kg/+cPVcbP/8= github.com/go-logr/logr v1.4.4/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE= github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78= github.com/go-openapi/analysis v1.0.0 h1:sNvbAGCJqUTqIAodr9IVqJMmuZas3YS9ms1dGK9yiJ4= @@ -108,23 +136,40 @@ github.com/go-viper/mapstructure/v2 v2.5.0 h1:vM5IJoUAy3d7zRSVtIwQgBj7BiWtMPfmPE github.com/go-viper/mapstructure/v2 v2.5.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= github.com/gofrs/flock v0.13.1 h1:jjREztyBeSKBZYAC+mgc1laB+xsgy4kYMf3FbKF2UBo= github.com/gofrs/flock v0.13.1/go.mod h1:sf4BFiHwnvgxa25DlQoDqXQnwRMEOwqxRq37P6MzzmE= +github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 h1:f+oWsMOmNPc8JmEHVZIycC7hBoQxHH9pNKQORJNozsQ= +github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8/go.mod h1:wcDNUvekVysuuOpQKo3191zZyTpiI6se1N1ULghS0sw= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/gnostic-models v0.6.9 h1:MU/8wDLif2qCXZmzncUQ/BOfxWfthHi63KqpoNbWqVw= github.com/google/gnostic-models v0.6.9/go.mod h1:CiWsm0s6BSQd1hRn8/QmxqB6BesYcbSZxsz9b0KuDBw= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/go-github/v89 v89.0.0 h1:35bEK5XoEcF3PZrlVbl9XN63f5BcJRA/UGkxeC9xPg0= github.com/google/go-github/v89 v89.0.0/go.mod h1:QLcbU0ipeAqQuR5KSg8c2lql4Qk1EwJ2dWz/0rP4Nho= github.com/google/go-querystring v1.2.0 h1:yhqkPbu2/OH+V9BfpCVPZkNmUXhb2gBxJArfhIxNtP0= github.com/google/go-querystring v1.2.0/go.mod h1:8IFJqpSRITyJ8QhQ13bmbeMBDfmeEJZD5A0egEOmkqU= +github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= +github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0 h1:/Tnpcb2E0Pz/tN9s3bfEY2Q8ePCEX9iuS+cneUwncnw= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0/go.mod h1:zOBXOsUaBSjKgmH4OGzV1esUpR3oUSCPYVd2cUBjKYY= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= +github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= +github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= +github.com/kevinburke/ssh_config v1.6.0 h1:J1FBfmuVosPHf5GRdltRLhPJtJpTlMdKTBjRgTaQBFY= +github.com/kevinburke/ssh_config v1.6.0/go.mod h1:q2RIzfka+BXARoNexmF9gkxEX7DmvbW9P4hIVx2Kg4M= +github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= +github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/lufia/plan9stats v0.0.0-20260330125221-c963978e514e h1:Q6MvJtQK/iRcRtzAscm/zF23XxJlbECiGPyRicsX+Ak= github.com/lufia/plan9stats v0.0.0-20260330125221-c963978e514e/go.mod h1:autxFIvghDt3jPTLoqZ9OZ7s9qTGNAWmYCjVFWPX/zg= github.com/mattn/go-colorable v0.1.15 h1:+u9SLTRGnXv73cEsnsmoZBom+dMU88B2M0aDcWy0/jY= @@ -139,8 +184,16 @@ github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3N github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= github.com/moby/patternmatcher v0.6.1 h1:qlhtafmr6kgMIJjKJMDmMWq7WLkKIo23hsrpR3x084U= github.com/moby/patternmatcher v0.6.1/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc= +github.com/moby/sys/atomicwriter v0.1.0 h1:kw5D/EqkBwsBFi0ss9v1VG3wIkVhzGvLklJ+w3A14Sw= +github.com/moby/sys/atomicwriter v0.1.0/go.mod h1:Ul8oqv2ZMNHOceF643P6FKPXeCmYtlQMvpizfsSoaWs= +github.com/moby/sys/sequential v0.7.0 h1:ASQNGNROJSuOO6LL6bPHbKvuZu6NU8P4ldPWk31zj/8= +github.com/moby/sys/sequential v0.7.0/go.mod h1:NfSTAp6V3fw4tmkD62PEcOKeZKquXT8VKCkf7aVR79o= +github.com/moby/term v0.5.2 h1:6qk3FJAFDs6i/q3W/pQ97SX192qKfZgGjCQqfCJkgzQ= +github.com/moby/term v0.5.2/go.mod h1:d3djjFCrjnB+fl8NJux+EJzu0msscUP+f8it8hPkFLc= github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 h1:n6/2gBQ3RWajuToeY6ZtZTIKv2v7ThUy5KKusIT0yc0= github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00/go.mod h1:Pm3mSP3c5uWn86xMLZ5Sa7JB9GsEZySvHYXCTK4E9q4= +github.com/morikuni/aec v1.1.0 h1:vBBl0pUnvi/Je71dsRrhMBtreIqNMYErSAbEeb8jrXQ= +github.com/morikuni/aec v1.1.0/go.mod h1:xDRgiq/iw5l+zkao76YTKzKttOp2cwPEne25HDkJnBw= github.com/oklog/ulid/v2 v2.1.2 h1:IEclFb9JNvzYA6MW2SCxbLzcHTVsfqm3PrqGQJH5zec= github.com/oklog/ulid/v2 v2.1.2/go.mod h1:rcEKHmBBKfef9DhnvX7y1HZBYxjXb0cP5ExxNsTT1QQ= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= @@ -148,24 +201,43 @@ github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3I github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040= github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M= github.com/pborman/getopt v0.0.0-20170112200414-7148bc3a4c30/go.mod h1:85jBQOZwpVEaDAr341tbn15RS4fCAsIst0qp7i8ex1o= +github.com/pjbgf/sha1cd v0.6.0 h1:3WJ8Wz8gvDz29quX1OcEmkAlUg9diU4GxJHqs0/XiwU= +github.com/pjbgf/sha1cd v0.6.0/go.mod h1:lhpGlyHLpQZoxMv8HcgXvZEhcGs0PG/vsZnEJ7H0iCM= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 h1:o4JXh1EVt9k/+g42oCprj/FisM4qX9L3sZB3upGN2ZU= github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/rodaine/protogofakeit v0.1.1 h1:ZKouljuRM3A+TArppfBqnH8tGZHOwM/pjvtXe9DaXH8= github.com/rodaine/protogofakeit v0.1.1/go.mod h1:pXn/AstBYMaSfc1/RqH3N82pBuxtWgejz1AlYpY1mI0= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/santhosh-tekuri/jsonschema/v6 v6.0.3 h1:1EYB5IzjZawrrnELUi78f9fPu57HuXjmddZPjrls/28= github.com/santhosh-tekuri/jsonschema/v6 v6.0.3/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU= +github.com/sergi/go-diff v1.4.0 h1:n/SP9D5ad1fORl+llWyN+D6qoUETXNZARKjyY2/KVCw= +github.com/sergi/go-diff v1.4.0/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= github.com/shirou/gopsutil/v3 v3.24.5 h1:i0t8kL+kQTvpAYToeuiVk3TgDeKOFioZO3Ztz/iZ9pI= github.com/shirou/gopsutil/v3 v3.24.5/go.mod h1:bsoOS1aStSs9ErQ1WWfxllSeS1K5D+U30r2NfcubMVk= github.com/shoenig/go-m1cpu v0.2.1 h1:yqRB4fvOge2+FyRXFkXqsyMoqPazv14Yyy+iyccT2E4= github.com/shoenig/go-m1cpu v0.2.1/go.mod h1:KkDOw6m3ZJQAPHbrzkZki4hnx+pDRR1Lo+ldA56wD5w= +github.com/shoenig/test v1.7.0 h1:eWcHtTXa6QLnBvm0jgEabMRN/uJ4DMV3M8xUGgRkZmk= +github.com/shoenig/test v1.7.0/go.mod h1:UxJ6u/x2v/TNs/LoLxBNJRV9DiwBBKYxXSyczsBHFoI= +github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w= +github.com/sirupsen/logrus v1.9.4/go.mod h1:ftWc9WdOfJ0a92nsE2jF5u5ZwH8Bv2zdeOC42RjbV2g= +github.com/skeema/knownhosts v1.3.2 h1:EDL9mgf4NzwMXCTfaxSD/o/a5fxDw/xL9nkU28JjdBg= +github.com/skeema/knownhosts v1.3.2/go.mod h1:bEg3iQAuw+jyiw+484wwFJoKSLwcfd7fqRy+N0QTiow= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.5.3 h1:jmXUvGomnU1o3W/V5h2VEradbpJDwGrzugQQvL0POH4= +github.com/stretchr/objx v0.5.3/go.mod h1:rDQraq+vQZU7Fde9LOZLr8Tax6zZvy4kuNKF+QYS+U0= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE= github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg= github.com/tklauser/go-sysconf v0.4.0 h1:7H0uAN+7RkwWRaxhYXDLqa5V3LPrJeV8wmD9dRUgPQU= github.com/tklauser/go-sysconf v0.4.0/go.mod h1:8mTNWyog7H+MpKijp4VmKJAd2bbYQ2zuUwkYRbUArPI= github.com/tklauser/numcpus v0.12.0 h1:NR85qdvHA9pFse3x3weVZ0r0ST8R6l5RHbZrlRaqob4= github.com/tklauser/numcpus v0.12.0/go.mod h1:ABHeXzJnr/qqwguhClkZKT1/8VABcYrsyUiUGobwWJg= +github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM= +github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw= github.com/xlab/treeprint v1.2.0 h1:HzHnuAF1plUN2zGlAFHbSQP2qJ0ZAD3XF5XD7OesXRQ= github.com/xlab/treeprint v1.2.0/go.mod h1:gj5Gd3gPdKtR1ikdDK6fnFLdmIS0X30kTTuNd/WEJu0= github.com/yoheimuta/go-protoparser/v4 v4.14.2 h1:/P/LlX1CF9NaTWEltGcIZVvNlPbhABuAnBtAWpb3+74= @@ -180,28 +252,45 @@ go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.70.0 h1:LMuyCAy go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.70.0/go.mod h1:085m8qbm4hgc8rZWGDEa4vmyyo2c3nPxUslYUKUIU04= go.opentelemetry.io/otel v1.46.0 h1:FHt5/CDyVxi/8IM1CH7VE/rRgq3kLHa2mSTVMO8AWyc= go.opentelemetry.io/otel v1.46.0/go.mod h1:Gj3SEScelsNC45tp4nSxRYlS+f5iez7W8XPMCt905kE= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.46.0 h1:OFnwLJr+pF3iHrlGSzbxyuo6/6HyBlnlN1CWEJmBVcw= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.46.0/go.mod h1:716wFneO0ov19A2beH5hjfh9AK5z/VWNAtDijp1Y0/g= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.45.0 h1:QBajQ2SrwQijzHyZbQlPsuIzpl/ll8DY6wPWsajeGcI= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.45.0/go.mod h1:08ZQLjrPLQ6R4kAXvuOvODEer5Yh4CoFvll5qB2BCI8= go.opentelemetry.io/otel/metric v1.46.0 h1:yBnkXvgV7AXFILZc5K6IZe/CBFF3OS7BJ8ov6/lj0K8= go.opentelemetry.io/otel/metric v1.46.0/go.mod h1:iPmdWqifKUdzziPkvvzIJXITl56fQx2mGM/DHLB3/2o= go.opentelemetry.io/otel/sdk v1.46.0 h1:h5CNQQjEbuQXY/JfZtgt3i7HVFV3aHPO2OAwO2eTYPI= go.opentelemetry.io/otel/sdk v1.46.0/go.mod h1:GAERFXFt5SYCEB+YiKUbMBeza6UaDH7GmGOZEfh2gSM= -go.opentelemetry.io/otel/sdk/metric v1.44.0 h1:3LlKgI+VjbVsjNRFZJZAJ30WjXC5VkNRks6si09iEfI= -go.opentelemetry.io/otel/sdk/metric v1.44.0/go.mod h1:5B5pMARnXxKhltooO4xUuCBorl65a4EpnTalObqOigA= +go.opentelemetry.io/otel/sdk/metric v1.46.0 h1:0piZ26EG4RBfebb2jhDH6ERCYHoVWduc3kLgPCwSnSE= +go.opentelemetry.io/otel/sdk/metric v1.46.0/go.mod h1:I1PbKrdVc8Qu8HYVDNtqVIwLwjNrhsV/uFuxfwg8mO4= go.opentelemetry.io/otel/trace v1.46.0 h1:OULy7ccdJnZtJ0UDYFOIGaCmiWzJ8Vi2G/Rsu60qs1c= go.opentelemetry.io/otel/trace v1.46.0/go.mod h1:J7GAXweO77XSFkB/rmAqk9D6ihszhFjLU+d9WuUxDLI= +go.opentelemetry.io/proto/otlp v1.11.0 h1:5rrYs0Ykyj50sdU/JU0x8etU+LubXWb+gED6TbEdMIk= +go.opentelemetry.io/proto/otlp v1.11.0/go.mod h1:SmVizdCOAm3XBtG1g1NnOdhW6jtddT72hLMhv8VwA8E= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI= go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU= go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= +golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M= +golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA= golang.org/x/exp v0.0.0-20260820142414-ca536658362e h1:01Ju2A/fZKkci4zqx0eZxw//DnRYOnBiGJG14hFBhO8= golang.org/x/exp v0.0.0-20260820142414-ca536658362e/go.mod h1:zeBbvyFKDaLwa7CH/zI8KXt7gTl14SF7sO08Pl5jBCM= +golang.org/x/mod v0.41.0 h1:qJmnOUb4YB+FsEuM3HcWucdZASCPGhsX6uljO6pog0c= +golang.org/x/mod v0.41.0/go.mod h1:Ek9pY8RKWXwsWvd3rQiHYtMqkjSUV+s1Rj7j4H5Ur6o= golang.org/x/net v0.59.0 h1:5zfYln+w5XCxwrnMMJPufRgNoXEaGxl0wo5GqPXyues= golang.org/x/net v0.59.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg= +golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= golang.org/x/term v0.46.0 h1:3+OXuTbaKDgwk8jTi3aSLHRlmWqHEUDUtxnbFigO4YE= golang.org/x/term v0.46.0/go.mod h1:+K02xbkittuwc0Am4abfA3Fc+XRGXkvBXNO88NCXPoc= golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI= golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E= +golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= +golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= google.golang.org/genproto/googleapis/api v0.0.0-20260819154853-08b0e4226688 h1:ax2KzoSRIZU/M0cIxri3pKxy99vniH1PVxWC6si/eZI= @@ -212,14 +301,22 @@ google.golang.org/grpc v1.83.2 h1:EManeRomTObA0BU7I8vXgg/78uE5MJ9M8B39EX2WscU= google.golang.org/grpc v1.83.2/go.mod h1:YPI1hK3kDked6iHvgX3tR0y+nX/qpMFKhPgFsokw1S8= google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc= google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/evanphx/json-patch.v4 v4.13.0 h1:czT3CmqEaQ1aanPc5SdlgQrrEIb8w/wwCvWWnfEbYzo= gopkg.in/evanphx/json-patch.v4 v4.13.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= +gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME= +gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gotest.tools/v3 v3.5.1 h1:EENdUnS3pdur5nybKYIh2Vfgc8IUNBjxDPSjtiJcOzU= +gotest.tools/v3 v3.5.1/go.mod h1:isy3WKz7GK6uNw/sbHzfKBLvlvXwUyV06n6brMxxopU= k8s.io/kube-openapi v0.0.0-20241212222426-2c72e554b1e7 h1:hcha5B1kVACrLujCKLbr8XWMxCxzQx42DY8QKYJrDLg= k8s.io/kube-openapi v0.0.0-20241212222426-2c72e554b1e7/go.mod h1:GewRfANuJ70iYzvn+i4lezLDAFzvjxZYK1gn1lWcfas= +sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= +sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= sigs.k8s.io/kustomize/api v0.21.1 h1:lzqbzvz2CSvsjIUZUBNFKtIMsEw7hVLJp0JeSIVmuJs= sigs.k8s.io/kustomize/api v0.21.1/go.mod h1:f3wkKByTrgpgltLgySCntrYoq5d3q7aaxveSagwTlwI= sigs.k8s.io/kustomize/kyaml v0.21.1 h1:IVlbmhC076nf6foyL6Taw4BkrLuEsXUXNpsE+ScX7fI= diff --git a/main.go b/main.go new file mode 100644 index 0000000..99e9b6c --- /dev/null +++ b/main.go @@ -0,0 +1,224 @@ +// Command warehouse is the codefly service agent that runs the warehouse +// gateway as a real service: a container speaking the codefly/warehouse/v0 gRPC +// contract, backed by the embedded engine locally and a cloud warehouse when +// deployed. Consumers depend on it as a gRPC service and never link a warehouse +// SDK — the gateway absorbs every backend difference. +// +// The agent mirrors the shape of the other codefly infrastructure agents +// (codefly-dev/service-object-storage, service-postgres): a Service advertises +// its configuration surface, a Runtime brings the container up for `codefly run` +// and tests, and a Builder emits the Kubernetes deployment and serves the +// image's SBOM evidence. The gateway itself is cmd/service-warehouse; this +// package is only what lets a workspace compose it. +package main + +import ( + "context" + "embed" + "fmt" + "strings" + + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + + "github.com/codefly-dev/core/agents" + "github.com/codefly-dev/core/agents/services" + "github.com/codefly-dev/core/builders" + basev0 "github.com/codefly-dev/core/generated/go/codefly/base/v0" + agentv0 "github.com/codefly-dev/core/generated/go/codefly/services/agent/v0" + "github.com/codefly-dev/core/resources" + runnersbase "github.com/codefly-dev/core/runners/base" + "github.com/codefly-dev/core/shared" + "github.com/codefly-dev/core/standards" + "github.com/codefly-dev/core/templates" +) + +// agent is loaded from the embedded agent.codefly.yaml (name + version). +var agent = shared.Must(resources.LoadFromFs[resources.Agent](shared.Embed(infoFS))) + +var requirements = builders.NewDependencies(agent.Name, + builders.NewDependency("service.codefly.yaml"), +) + +// configurationGroup is the name of the configuration group this service +// exports to its consumers and reads its own deployment values from. +const configurationGroup = "warehouse" + +// Settings are the service.codefly.yaml knobs. They are the local defaults; +// runtime configuration (an environment's values) takes precedence. Credentials +// are never settings: they arrive as runtime configuration, never here. +type Settings struct { + // Backend selects the warehouse. Empty leaves the choice to the gateway, + // whose default is the embedded engine. A deployed environment names a cloud + // warehouse. + Backend string `yaml:"backend"` + // Database is the database the gateway is bound to (a BigQuery project, a + // Snowflake or Redshift database, an embedded engine's file). + Database string `yaml:"database"` + // Dataset is the default namespace for unqualified names. + Dataset string `yaml:"dataset"` + // WarehouseLocation is the region datasets and jobs live in, for the + // warehouses that have one. It is not named Location because services.Base + // already has a Location, the service's directory. + WarehouseLocation string `yaml:"location"` +} + +// resolved holds the effective configuration after defaults and runtime +// configuration are applied. The keys are the gateway's own SWH_* variables. +type resolved struct { + backend string + database string + dataset string + location string + // credentialsFile names a warehouse credentials file. Nothing delivers one: + // a local run would have to project it into the gateway container, and a + // deployment mounts nothing, so the Runtime and the Builder both refuse a + // configured path rather than name a file the gateway cannot open. + credentialsFile string + authToken string +} + +type Service struct { + *services.Base + *Settings + + conf resolved + + // gatewayToken is the secret every caller must present to the gateway. The + // Runtime generates one per run and hands it to consumers through the + // configuration channel as a secret value — never inside the connection + // string. It lives outside resolved because LoadConfiguration rebuilds + // resolved from the incoming configuration on every call. + gatewayToken string + + GrpcEndpoint *basev0.Endpoint +} + +func NewService() *Service { + return &Service{ + Base: services.NewServiceBase(context.Background(), agent.Of(resources.ServiceAgent)), + Settings: &Settings{}, + } +} + +func (s *Service) GetAgentInformation(ctx context.Context, _ *agentv0.AgentInformationRequest) (*agentv0.AgentInformation, error) { + readme, err := templates.ApplyTemplateFrom(ctx, shared.Embed(readmeFS), "templates/agent/README.md", s.Information) + if err != nil { + return nil, status.Error(codes.Internal, err.Error()) + } + // Validation is left unset on purpose: this agent serves image-scope SBOM + // (Builder.SBOM) without advertising ValidationCapabilities.image_sbom. A + // present Validation is authoritative for every operation, including the ones + // it omits, and advertising the capability is a separate decision from + // serving the scope: core's docs/sbom.md holds it back until every consumer + // that evaluates it runs a core whose phase bound accepts it. + // TestImageSBOMIsServedButNotAdvertised pins both halves. + return services.Advertisement{ + Backends: runnersbase.BackendSupport{Docker: true}, + ReadMe: readme, + Config: []*agentv0.ConfigurationValueDetail{ + { + Name: configurationGroup, Description: "warehouse gateway endpoint", + Fields: []*agentv0.ConfigurationValueInformation{ + {Name: "connection", Description: "grpc connection string"}, + {Name: "endpoint", Description: "host:port of the gRPC endpoint"}, + {Name: "token", Description: "shared secret sent as the x-codefly-token grpc metadata header"}, + }, + }, + }, + }.Build(), nil +} + +// resolveServingGRPCEndpoint selects the single gRPC endpoint the agent binds +// its runtime and deployment to. +func resolveServingGRPCEndpoint(ctx context.Context, endpoints []*basev0.Endpoint) (*basev0.Endpoint, error) { + grpc := resources.FindEndpointsByAPI(ctx, standards.GRPC, endpoints) + if len(grpc) == 0 { + return nil, fmt.Errorf("no grpc endpoint found") + } + return grpc[0], nil +} + +// LoadConfiguration resolves the effective configuration. Settings are the local +// defaults; runtime configuration (deployed values) takes precedence. +func (s *Service) LoadConfiguration(ctx context.Context, conf *basev0.Configuration) error { + r := resolved{ + backend: s.Backend, + database: s.Database, + dataset: s.Dataset, + location: s.WarehouseLocation, + } + if conf != nil { + for key, dst := range map[string]*string{ + "SWH_BACKEND": &r.backend, + "SWH_DATABASE": &r.database, + "SWH_DATASET": &r.dataset, + "SWH_LOCATION": &r.location, + "SWH_CREDENTIALS_FILE": &r.credentialsFile, + "SWH_AUTH_TOKEN": &r.authToken, + } { + v, err := resources.GetConfigurationValue(ctx, conf, configurationGroup, key) + if err == nil && v != "" { + *dst = v + } + } + } + // Normalized at the single boundary where it enters, for the same reason the + // gateway trims its own env read: a secret carrying a trailing newline would + // otherwise be handed to consumers in one form and enforced in another. The + // rest arrive through YAML block scalars and env files just the same, and a + // stray newline in one is not a value but a malformed manifest. + for _, field := range []*string{&r.backend, &r.database, &r.dataset, &r.location, &r.credentialsFile, &r.authToken} { + *field = strings.TrimSpace(*field) + } + s.conf = r + return nil +} + +func (s *Service) createConnectionString(address string) string { + return fmt.Sprintf("grpc://%s", address) +} + +func (s *Service) CreateConnectionConfiguration(ctx context.Context, conf *basev0.Configuration, instance *basev0.NetworkInstance) (*basev0.Configuration, error) { + defer s.Wool.Catch() + ctx = s.Wool.Inject(ctx) + + if err := s.LoadConfiguration(ctx, conf); err != nil { + return nil, s.Wool.Wrapf(err, "cannot load configuration") + } + values := []*basev0.ConfigurationValue{ + {Key: "connection", Value: s.createConnectionString(instance.Address)}, + {Key: "endpoint", Value: instance.Address}, + } + // The token travels as its own secret value: putting it in the connection + // string would leak it into every log line and manifest that carries an + // endpoint. + if s.gatewayToken != "" { + values = append(values, &basev0.ConfigurationValue{Key: "token", Value: s.gatewayToken, Secret: true}) + } + return &basev0.Configuration{ + Origin: s.Base.Unique(), + RuntimeContext: resources.RuntimeContextFromInstance(instance), + Infos: []*basev0.ConfigurationInformation{ + {Name: configurationGroup, ConfigurationValues: values}, + }, + }, nil +} + +func main() { + svc := NewService() + agents.Serve(agents.PluginRegistration{ + Agent: svc, + Runtime: NewRuntime(), + Builder: NewBuilder(), + }) +} + +//go:embed agent.codefly.yaml +var infoFS embed.FS + +//go:embed gateway-image.json +var gatewayImageLockJSON []byte + +//go:embed templates/agent +var readmeFS embed.FS diff --git a/readiness_test.go b/readiness_test.go new file mode 100644 index 0000000..b7918e6 --- /dev/null +++ b/readiness_test.go @@ -0,0 +1,129 @@ +package main + +import ( + "context" + "net" + "testing" + "time" + + basev0 "github.com/codefly-dev/core/generated/go/codefly/base/v0" + "github.com/codefly-dev/core/resources" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/credentials/insecure" + grpchealth "google.golang.org/grpc/health" + healthv1 "google.golang.org/grpc/health/grpc_health_v1" + + whv0 "github.com/codefly-dev/service-warehouse/gen/codefly/warehouse/v0" + "github.com/codefly-dev/service-warehouse/internal/auth" + "github.com/codefly-dev/service-warehouse/internal/backend" + "github.com/codefly-dev/service-warehouse/internal/backend/mem" + "github.com/codefly-dev/service-warehouse/internal/server" +) + +// startGateway serves the gateway's gRPC surface — the Warehouse service behind +// the token interceptors, the health service outside them — on a loopback port +// and returns its address and its health server. cmd/service-warehouse builds +// the same server from the same pieces; it is rebuilt here because that +// constructor is in a main package. +func startGateway(t *testing.T, token string) (string, *grpchealth.Server) { + t.Helper() + lis, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + health := grpchealth.NewServer() + srv := grpc.NewServer( + grpc.ChainUnaryInterceptor(auth.UnaryInterceptor(token)), + grpc.ChainStreamInterceptor(auth.StreamInterceptor(token)), + ) + healthv1.RegisterHealthServer(srv, health) + whv0.RegisterWarehouseServer(srv, server.New(mem.New(backend.Config{}))) + go func() { _ = srv.Serve(lis) }() + t.Cleanup(srv.Stop) + return lis.Addr().String(), health +} + +// runtimeAt wires a Runtime with just enough network state for WaitForReady to +// resolve address. +func runtimeAt(t *testing.T, address string, token string) *Runtime { + t.Helper() + rt := NewRuntime() + endpoint := &basev0.Endpoint{Name: "grpc", Api: "grpc"} + rt.GrpcEndpoint = endpoint + rt.Runtime.WithContext(resources.NewRuntimeContextNative()) + rt.NetworkMappings = []*basev0.NetworkMapping{{ + Endpoint: endpoint, + Instances: []*basev0.NetworkInstance{{Address: address, Access: resources.NewNativeNetworkAccess()}}, + }} + rt.gatewayToken = token + return rt +} + +// A readiness probe that swallowed its own error would retry a token mismatch +// for the whole budget and then report "not ready", pointing the operator at the +// gateway instead of at the credential. +func TestWaitForReadyReportsRejectedCredential(t *testing.T) { + address, _ := startGateway(t, "server-token") + rt := runtimeAt(t, address, "a-different-token") + + start := time.Now() + err := rt.WaitForReady(context.Background()) + + require.Error(t, err, "WaitForReady must fail when the gateway rejects the agent's credential") + require.Contains(t, err.Error(), "rejected the agent's credential") + require.Less(t, time.Since(start), 10*time.Second, "a rejected credential must not be retried as if transient") + + rt.gatewayToken = "server-token" + require.NoError(t, rt.WaitForReady(context.Background())) +} + +// The health service answers without a token, so a gateway that is up but +// reports itself not serving must not be called ready on the strength of +// Capabilities alone; once it serves, the wait ends. +func TestWaitForReadyWaitsForTheGatewayToServe(t *testing.T) { + address, health := startGateway(t, "server-token") + health.SetServingStatus("", healthv1.HealthCheckResponse_NOT_SERVING) + rt := runtimeAt(t, address, "server-token") + + go func() { + time.Sleep(1500 * time.Millisecond) + health.SetServingStatus("", healthv1.HealthCheckResponse_SERVING) + }() + start := time.Now() + require.NoError(t, rt.WaitForReady(context.Background())) + require.GreaterOrEqual(t, time.Since(start), time.Second, "ready was reported while the gateway said it was not serving") +} + +func TestProbeGatewayDistinguishesNotYetFromNever(t *testing.T) { + address, health := startGateway(t, "server-token") + conn, err := grpc.NewClient(address, grpc.WithTransportCredentials(insecureCredentials()), auth.DialOption("server-token")) + require.NoError(t, err) + defer conn.Close() + healthClient, warehouse := healthv1.NewHealthClient(conn), whv0.NewWarehouseClient(conn) + + reason, final := probeGateway(context.Background(), healthClient, warehouse) + require.Empty(t, reason) + require.NoError(t, final) + + health.SetServingStatus("", healthv1.HealthCheckResponse_NOT_SERVING) + reason, final = probeGateway(context.Background(), healthClient, warehouse) + require.Contains(t, reason, "NOT_SERVING") + require.NoError(t, final, "a gateway that is not serving yet is worth retrying") +} + +func TestWaitForReadyReturnsOnCancellation(t *testing.T) { + rt := runtimeAt(t, "127.0.0.1:1", "token") // nothing listens here + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(1500 * time.Millisecond) + cancel() + }() + + start := time.Now() + err := rt.WaitForReady(ctx) + + require.Error(t, err) + require.Less(t, time.Since(start), 10*time.Second, "cancellation must end the wait, not the 60s budget") +} + +func insecureCredentials() credentials.TransportCredentials { return insecure.NewCredentials() } diff --git a/runtime.go b/runtime.go new file mode 100644 index 0000000..c101d47 --- /dev/null +++ b/runtime.go @@ -0,0 +1,356 @@ +package main + +import ( + "context" + "crypto/rand" + "encoding/hex" + "fmt" + "os" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials/insecure" + healthv1 "google.golang.org/grpc/health/grpc_health_v1" + "google.golang.org/grpc/status" + + basev0 "github.com/codefly-dev/core/generated/go/codefly/base/v0" + runtimev0 "github.com/codefly-dev/core/generated/go/codefly/services/runtime/v0" + + "github.com/codefly-dev/core/agents/services" + "github.com/codefly-dev/core/resources" + dockerrun "github.com/codefly-dev/core/runners/dockerrun" + "github.com/codefly-dev/core/wool" + + whv0 "github.com/codefly-dev/service-warehouse/gen/codefly/warehouse/v0" + "github.com/codefly-dev/service-warehouse/internal/auth" +) + +// gatewayContainerPort is the port the gateway listens on inside its container. +// It is handed to the gateway as SWH_LISTEN and to the Kubernetes templates as +// the container port; TestGatewayContainerPortIsTheGatewaysDefault holds it to +// the gateway's own default and to the Dockerfile's EXPOSE. +const gatewayContainerPort = 9465 + +// gatewayContainerUser is the unprivileged uid:gid the gateway runs as. The +// published image already declares it, but SWH_GATEWAY_IMAGE can name another +// one; setting the user explicitly makes the guarantee hold for whichever image +// is run. +const gatewayContainerUser = "65532:65532" + +// readinessBudget bounds the whole wait for the gateway in Start: one overall +// deadline, so a slow attempt spends the budget instead of extending it. +const readinessBudget = 60 * time.Second + +// readinessProbeTimeout bounds a single attempt, and readinessPollInterval paces +// the retries between them. +const ( + readinessProbeTimeout = 5 * time.Second + readinessPollInterval = time.Second +) + +// containerEnvironment is the part of dockerrun.DockerEnvironment the Runtime +// owns: something it can identify and tear down. Teardown is the path that has +// to keep working when Docker does not, so it is reachable through this seam. +type containerEnvironment interface { + ContainerID() (string, error) + Shutdown(context.Context) error +} + +type Runtime struct { + *services.DefaultRuntime + *Service + + gatewayEnv containerEnvironment +} + +func NewRuntime() *Runtime { + service := NewService() + return &Runtime{ + DefaultRuntime: services.NewDefaultRuntime(service.Runtime), + Service: service, + } +} + +func (s *Runtime) Load(ctx context.Context, req *runtimev0.LoadRequest) (*runtimev0.LoadResponse, error) { + defer s.Wool.Catch() + return s.Runtime.LoadService(ctx, req, services.RuntimeLoad{ + Settings: s.Settings, + Requirements: requirements, + ResolveEndpoints: func(ctx context.Context, endpoints []*basev0.Endpoint) error { + endpoint, err := resolveServingGRPCEndpoint(ctx, endpoints) + if err != nil { + return s.Wool.Wrapf(err, "cannot find grpc endpoint") + } + s.GrpcEndpoint = endpoint + return nil + }, + }) +} + +func (s *Runtime) Init(ctx context.Context, req *runtimev0.InitRequest) (*runtimev0.InitResponse, error) { + defer s.Wool.Catch() + ctx = s.Wool.Inject(ctx) + + s.Runtime.LogInitRequest(req) + s.Runtime.WithContext(req.GetRuntimeContext()) + s.NetworkMappings = req.ProposedNetworkMappings + + w := s.Wool.In("runtime::init") + configuration := req.GetConfiguration() + + net, err := resources.FindNetworkMapping(ctx, s.NetworkMappings, s.GrpcEndpoint) + if err != nil { + return s.Runtime.InitError(err) + } + instance, err := resources.FindNetworkInstanceInNetworkMappings(ctx, s.NetworkMappings, s.GrpcEndpoint, s.Runtime.NetworkAccess()) + if err != nil { + return s.Runtime.InitError(err) + } + + if err = s.LoadConfiguration(ctx, configuration); err != nil { + return s.Runtime.InitError(err) + } + // Refuse before any container exists: a path the gateway cannot open must + // fail here with its own cause, not as a gateway that comes up and cannot + // authenticate to its warehouse. + if s.conf.credentialsFile != "" { + return s.Runtime.InitError(s.Wool.NewError("SWH_CREDENTIALS_FILE cannot be delivered to a local run: "+ + "nothing here projects %s into the gateway container, which would name a path it cannot open. "+ + "A local run is for the embedded engine, which needs no credential; "+ + "a cloud warehouse is reached from a deployment", s.conf.credentialsFile)) + } + + s.gatewayToken, err = s.resolveGatewayToken() + if err != nil { + return s.Runtime.InitError(err) + } + + if err = s.startGateway(ctx, uint16(instance.Port)); err != nil { + s.rollbackInit(ctx) + return s.Runtime.InitError(err) + } + + for _, inst := range net.Instances { + conf, errConn := s.CreateConnectionConfiguration(ctx, configuration, inst) + if errConn != nil { + return s.Runtime.InitError(errConn) + } + s.Runtime.RuntimeConfigurations = append(s.Runtime.RuntimeConfigurations, conf) + } + + w.Debug("init successful") + return s.Runtime.InitResponse() +} + +// resolveGatewayToken returns the secret the gateway will enforce and consumers +// will present. A configured token is honored so an operator who pins one is not +// silently overridden; otherwise a fresh one is generated per run, which is what +// stands between the all-interface published port and SQL against the bound +// database, and which revokes every token a previous run handed out. +// +// Generating a new token changes the gateway container's environment, and +// dockerrun fingerprints the environment, so each run deliberately recreates the +// container rather than reattaching to the previous one. Making the token stable +// to recover that reattach would silently give up the revocation property. +func (s *Runtime) resolveGatewayToken() (string, error) { + if s.conf.authToken != "" { + return s.conf.authToken, nil + } + return randomSecret() +} + +// startGateway runs the gateway container, mapping the assigned host port onto +// the gateway's listen port and handing it the resolved configuration. A value +// nobody set is not passed, so the gateway's own default decides it. +func (s *Runtime) startGateway(ctx context.Context, hostPort uint16) error { + image, _, err := effectiveGatewayImage() + if err != nil { + return err + } + + runner, err := dockerrun.NewDockerHeadlessEnvironment(ctx, image, s.UniqueWithWorkspace()+"-gateway") + if err != nil { + return s.Wool.Wrapf(err, "cannot create gateway environment") + } + // Own the environment before starting it: a container that is created and + // then exits still has to be removed, and only Shutdown does that. + s.gatewayEnv = runner + runner.WithOutput(os.Stdout) + runner.WithUser(gatewayContainerUser) + runner.WithPortMapping(ctx, hostPort, gatewayContainerPort) + // Consumers running in their own container reach the gateway through + // host.docker.internal (the Container network instance the agent advertises), + // which resolves to the bridge gateway on Linux — unreachable if the port is + // bound only to 127.0.0.1. Publish on all interfaces; what makes that safe is + // SWH_AUTH_TOKEN below, not the binding. + runner.WithPublicPorts() + envs := []*resources.EnvironmentVariable{ + resources.Env("SWH_LISTEN", fmt.Sprintf(":%d", gatewayContainerPort)), + resources.Env("SWH_AUTH_TOKEN", s.gatewayToken), + } + for _, value := range []struct{ name, value string }{ + {"SWH_BACKEND", s.conf.backend}, + {"SWH_DATABASE", s.conf.database}, + {"SWH_DATASET", s.conf.dataset}, + {"SWH_LOCATION", s.conf.location}, + } { + if value.value != "" { + envs = append(envs, resources.Env(value.name, value.value)) + } + } + runner.WithEnvironmentVariables(ctx, envs...) + if err = runner.Init(ctx); err != nil { + return s.Wool.Wrapf(err, "cannot start gateway") + } + return nil +} + +// teardown releases everything the Runtime owns. An environment that fails to +// shut down keeps its reference so a later attempt can retry it. +func (s *Runtime) teardown(ctx context.Context) error { + if s.gatewayEnv == nil { + return nil + } + if err := s.gatewayEnv.Shutdown(ctx); err != nil { + return s.Wool.Wrapf(err, "cannot shut down gateway") + } + s.gatewayEnv = nil + return nil +} + +// rollbackInit releases what Init already owns after a failed startup. The +// startup error is what the caller reports, so a teardown failure is logged +// rather than substituted for it. +func (s *Runtime) rollbackInit(ctx context.Context) { + if err := s.teardown(ctx); err != nil { + s.Wool.Warn("cannot fully roll back after failed init", wool.ErrField(err)) + } +} + +func (s *Runtime) Start(ctx context.Context, req *runtimev0.StartRequest) (*runtimev0.StartResponse, error) { + defer s.Wool.Catch() + ctx = s.Wool.Inject(ctx) + if err := s.WaitForReady(ctx); err != nil { + return s.Runtime.StartError(err) + } + return s.Runtime.StartResponse() +} + +// WaitForReady polls the gateway until its health service reports it serving +// and it has accepted the agent's credential. +// +// This is liveness plus the credential, and says nothing about the warehouse: +// the gateway does not probe its backend and no RPC reports that, so a gateway +// whose warehouse is unreachable is still ready here and fails the first query +// that needs it (README, "Authentication"). Capabilities is the second probe +// because it is the cheapest call that is not exempt from authentication — the +// health service carries no token — so a mismatched token surfaces here as +// itself rather than at a consumer's first call. +// +// The gateway is addressed through the NATIVE network view. The agent is a host +// process whatever runtime context the service itself was given, and a +// container-only name resolves to nothing from here. +func (s *Runtime) WaitForReady(ctx context.Context) error { + instance, err := resources.FindNetworkInstanceInNetworkMappings(ctx, s.NetworkMappings, s.GrpcEndpoint, resources.NewNativeNetworkAccess()) + if err != nil { + return s.Wool.Wrapf(err, "cannot find network instance") + } + address := instance.Address + s.Wool.Debug("waiting for warehouse gateway", wool.Field("address", address)) + + conn, err := grpc.NewClient(address, + grpc.WithTransportCredentials(insecure.NewCredentials()), + auth.DialOption(s.gatewayToken)) + if err != nil { + return s.Wool.Wrapf(err, "cannot create gateway client for %s", address) + } + defer func() { _ = conn.Close() }() + health := healthv1.NewHealthClient(conn) + warehouse := whv0.NewWarehouseClient(conn) + + start := time.Now() + budget, cancel := context.WithTimeout(ctx, readinessBudget) + defer cancel() + ticker := time.NewTicker(readinessPollInterval) + defer ticker.Stop() + + lastErr := "no attempt completed" +poll: + for { + // Stop rather than start a probe the budget cannot see through: a + // truncated attempt reports this deadline and buries the cause the last + // full attempt established. + if deadline, ok := budget.Deadline(); ok && time.Until(deadline) < readinessProbeTimeout { + break + } + + attempt, attemptCancel := context.WithTimeout(budget, readinessProbeTimeout) + lastErr, err = probeGateway(attempt, health, warehouse) + attemptCancel() + if err != nil { + return s.Wool.Wrapf(err, "warehouse gateway rejected the agent's credential") + } + if lastErr == "" { + return nil + } + + select { + case <-budget.Done(): + break poll + case <-ticker.C: + } + } + return s.Wool.NewError("warehouse gateway at %s not ready after %s: %s", + address, time.Since(start).Round(time.Millisecond), lastErr) +} + +// probeGateway makes one readiness attempt. A non-empty reason means the gateway +// is not ready yet and the attempt is worth repeating; a non-nil error is final. +// A rejected credential is not a gateway that is still coming up: retrying +// cannot change the answer, and spending the budget on it reports the failure as +// readiness instead of as the mismatched token it is. +func probeGateway(ctx context.Context, health healthv1.HealthClient, warehouse whv0.WarehouseClient) (reason string, final error) { + res, err := health.Check(ctx, &healthv1.HealthCheckRequest{}) + if err != nil { + return err.Error(), nil + } + if res.GetStatus() != healthv1.HealthCheckResponse_SERVING { + return fmt.Sprintf("health status %s", res.GetStatus()), nil + } + if _, err = warehouse.Capabilities(ctx, &whv0.CapabilitiesRequest{}); err != nil { + if status.Code(err) == codes.Unauthenticated { + return "", err + } + return err.Error(), nil + } + return "", nil +} + +func (s *Runtime) Stop(ctx context.Context, req *runtimev0.StopRequest) (*runtimev0.StopResponse, error) { + defer s.Wool.Catch() + return s.Runtime.StopResponse() +} + +func (s *Runtime) Destroy(ctx context.Context, req *runtimev0.DestroyRequest) (*runtimev0.DestroyResponse, error) { + defer s.Wool.Catch() + ctx = s.Wool.Inject(ctx) + if err := s.teardown(ctx); err != nil { + return s.Runtime.DestroyError(err) + } + return s.Runtime.DestroyResponse() +} + +func (s *Runtime) Test(ctx context.Context, req *runtimev0.TestRequest) (*runtimev0.TestResponse, error) { + return s.Runtime.TestResponse() +} + +// randomSecret returns an unguessable hex secret, so the published port is never +// reachable with a well-known credential. +func randomSecret() (string, error) { + b := make([]byte, 24) + if _, err := rand.Read(b); err != nil { + return "", err + } + return hex.EncodeToString(b), nil +} diff --git a/runtime_e2e_test.go b/runtime_e2e_test.go new file mode 100644 index 0000000..03fd028 --- /dev/null +++ b/runtime_e2e_test.go @@ -0,0 +1,248 @@ +//go:build e2e + +// This file drives the agent Runtime through its real lifecycle — Load → Init → +// Start — standing the gateway container up exactly as `codefly run` does, then +// calls it over the advertised gRPC endpoint with the credential the agent +// handed out. Run with: +// +// docker build -t service-warehouse:e2e . +// SWH_GATEWAY_IMAGE=service-warehouse:e2e go test -tags e2e -run TestRuntimeEndToEnd . +// +// The gateway image is a prerequisite because the published one only exists once +// a release has run; the test points the Runtime at the locally built image +// through the same SWH_GATEWAY_IMAGE override the Runtime already honors. +// +// It asks the gateway for the catalog-only backend: that is the one a container +// can serve with no warehouse, so what is under test is the agent, the image and +// the credential, not a warehouse. +package main + +import ( + "context" + "fmt" + "os" + "os/exec" + "path" + "path/filepath" + "strings" + "testing" + "time" + + basev0 "github.com/codefly-dev/core/generated/go/codefly/base/v0" + builderv0 "github.com/codefly-dev/core/generated/go/codefly/services/builder/v0" + runtimev0 "github.com/codefly-dev/core/generated/go/codefly/services/runtime/v0" + "github.com/codefly-dev/core/network" + "github.com/codefly-dev/core/resources" + "github.com/codefly-dev/core/shared" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/status" + + whv0 "github.com/codefly-dev/service-warehouse/gen/codefly/warehouse/v0" + "github.com/codefly-dev/service-warehouse/internal/auth" +) + +func TestRuntimeEndToEnd(t *testing.T) { + if os.Getenv(gatewayImageOverrideEnv) == "" { + t.Skipf("%s not set; skipping agent-runtime end-to-end test", gatewayImageOverrideEnv) + } + ctx := context.Background() + rt, networkMappings, runtimeContext := loadedRuntime(t, ctx) + + // Register teardown before Init: a failure after the container exists must + // still clean it up. Destroy nil-checks the environment, so it is safe even + // if Init failed before starting anything. The flag keeps this safety net + // from running a second Shutdown after the explicit teardown asserted below. + destroyed := false + defer func() { + if !destroyed { + _, _ = rt.Destroy(context.Background(), &runtimev0.DestroyRequest{}) + } + }() + + init, err := rt.Init(ctx, &runtimev0.InitRequest{ + RuntimeContext: runtimeContext, + ProposedNetworkMappings: networkMappings, + Configuration: configuration(map[string]string{"SWH_BACKEND": "mem"}), + }) + require.NoError(t, err) + // Init reports a lifecycle failure in its status, not as a returned error: + // the contract hands the CLI a diagnostic rather than a transport fault. + require.Equal(t, runtimev0.InitStatus_READY, init.GetStatus().GetState(), init.GetStatus().GetMessage()) + + start, err := rt.Start(ctx, &runtimev0.StartRequest{}) + require.NoError(t, err) + require.Equal(t, runtimev0.StartStatus_STARTED, start.GetStatus().GetState(), start.GetStatus().GetMessage()) + + // A consumer running in its own container reaches the gateway through + // host.docker.internal, which resolves to the bridge gateway on Linux — a + // 127.0.0.1-bound port is unreachable there. It must publish on all + // interfaces; the token checked below is what keeps that from being an open + // door. + gatewayID, err := rt.gatewayEnv.ContainerID() + require.NoError(t, err) + requirePublishedOnAllInterfaces(t, gatewayID, gatewayContainerPort) + + conf, err := resources.ExtractConfiguration(init.RuntimeConfigurations, resources.NewRuntimeContextNative()) + require.NoError(t, err) + endpoint, err := resources.GetConfigurationValue(ctx, conf, configurationGroup, "endpoint") + require.NoError(t, err) + require.NotEmpty(t, endpoint) + connection, err := resources.GetConfigurationValue(ctx, conf, configurationGroup, "connection") + require.NoError(t, err) + token, err := resources.GetConfigurationValue(ctx, conf, configurationGroup, "token") + require.NoError(t, err) + require.NotEmpty(t, token, "the runtime must hand consumers a gateway credential") + require.NotContains(t, endpoint, token, "endpoint must not carry the credential") + require.NotContains(t, connection, token, "connection string must not carry the credential") + requireTokenIsSecret(t, conf, token) + + requireUnauthenticatedPeerIsRefused(t, endpoint) + + // What the gateway was started with is what the agent resolved: the backend + // it was told, and no variable for a value nobody set. + env, err := exec.Command("docker", "inspect", "-f", `{{join .Config.Env "\n"}}`, gatewayID).Output() + require.NoError(t, err) + gatewayEnv := strings.Split(string(env), "\n") + require.Contains(t, gatewayEnv, "SWH_BACKEND=mem") + require.Contains(t, gatewayEnv, fmt.Sprintf("SWH_LISTEN=:%d", gatewayContainerPort)) + for _, variable := range gatewayEnv { + require.False(t, strings.HasPrefix(variable, "SWH_DATABASE="), "a value nobody set must be left to the gateway: %s", variable) + } + + conn, err := grpc.NewClient(endpoint, + grpc.WithTransportCredentials(insecure.NewCredentials()), + auth.DialOption(token)) + require.NoError(t, err) + defer conn.Close() + capabilities, err := whv0.NewWarehouseClient(conn).Capabilities(ctx, &whv0.CapabilitiesRequest{}) + require.NoError(t, err) + require.NotNil(t, capabilities) + + // Teardown owns exactly what it started. + _, err = rt.Destroy(context.Background(), &runtimev0.DestroyRequest{}) + require.NoError(t, err) + destroyed = true + require.Error(t, exec.Command("docker", "inspect", gatewayID).Run(), "container %s survived teardown", gatewayID) +} + +// requireTokenIsSecret asserts the gateway credential is carried as a +// secret-marked configuration value, so nothing downstream logs or renders it. +func requireTokenIsSecret(t *testing.T, conf *basev0.Configuration, token string) { + t.Helper() + for _, info := range conf.GetInfos() { + for _, value := range info.GetConfigurationValues() { + if value.GetKey() == "token" { + require.Equal(t, token, value.GetValue()) + require.True(t, value.GetSecret(), "gateway token must be marked secret") + return + } + } + } + t.Fatal("no token value in the runtime configuration") +} + +// requireUnauthenticatedPeerIsRefused dials the published gateway port the way +// any other host on the network could — no credential at all, then a plausible +// wrong one — and asserts the gateway refuses it. This is the acceptance case +// for the all-interface binding asserted above: reachable, but useless without +// the session token. +func requireUnauthenticatedPeerIsRefused(t *testing.T, endpoint string) { + t.Helper() + peers := map[string][]grpc.DialOption{ + "no credentials": nil, + "wrong token": {auth.DialOption("0000000000000000000000000000000000000000000000000000000000000000")}, + } + for peerName, extra := range peers { + conn, err := grpc.NewClient(endpoint, + append(extra, grpc.WithTransportCredentials(insecure.NewCredentials()))...) + require.NoError(t, err) + t.Run(peerName, func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + _, err := whv0.NewWarehouseClient(conn).Capabilities(ctx, &whv0.CapabilitiesRequest{}) + require.Error(t, err) + require.Equal(t, codes.Unauthenticated, status.Code(err), "got %v", err) + }) + _ = conn.Close() + } +} + +// loadedRuntime scaffolds a service through the Builder and returns a Runtime +// loaded against it, with the network mappings and runtime context Init needs. +// +// The native runtime context puts the test process on the host, so both the +// Runtime's readiness probe and the test reach the gateway at localhost:. +func loadedRuntime(t *testing.T, ctx context.Context) (*Runtime, []*basev0.NetworkMapping, *basev0.RuntimeContext) { + t.Helper() + + // Keep all persistent test data outside the developer's real Codefly home. + codeflyHome, err := filepath.EvalSymlinks(t.TempDir()) + require.NoError(t, err) + t.Setenv(resources.CodeflyHomeEnv, codeflyHome) + workspace := &resources.Workspace{Name: "test"} + tmpDir := t.TempDir() + serviceName := fmt.Sprintf("svc-%v", time.Now().UnixMilli()) + service := resources.Service{Name: serviceName, Version: "test-me"} + require.NoError(t, service.SaveAtDir(ctx, path.Join(tmpDir, "mod", service.Name))) + + identity := &basev0.ServiceIdentity{ + Name: service.Name, + Version: service.Version, + Module: "mod", + Workspace: workspace.Name, + WorkspacePath: tmpDir, + RelativeToWorkspace: fmt.Sprintf("mod/%s", service.Name), + } + + builder := NewBuilder() + _, err = builder.Load(ctx, &builderv0.LoadRequest{ + DisableCatch: true, + Identity: identity, + CreationMode: &builderv0.CreationMode{Communicate: false}, + }) + require.NoError(t, err) + _, err = builder.Create(ctx, &builderv0.CreateRequest{}) + require.NoError(t, err) + + rt := NewRuntime() + networkManager, err := network.NewRuntimeManager(ctx, nil) + require.NoError(t, err) + networkManager.WithTemporaryPorts() + env := resources.LocalEnvironment() + + _, err = rt.Load(ctx, &runtimev0.LoadRequest{ + Identity: identity, + Environment: shared.Must(env.Proto()), + DisableCatch: true, + }) + require.NoError(t, err) + require.Equal(t, 1, len(rt.Endpoints)) + + runtimeContext := resources.NewRuntimeContextNative() + networkMappings, err := networkManager.GenerateNetworkMappings(ctx, env, workspace, rt.Identity, rt.Endpoints, runtimeContext) + require.NoError(t, err) + + return rt, networkMappings, runtimeContext +} + +// requirePublishedOnAllInterfaces asserts the container's port is published on +// 0.0.0.0, not 127.0.0.1 — the difference between reachable and refused from a +// consumer container on a Linux bridge network. +// +// Docker can publish a single port under more than one host binding (an IPv6 +// "::" entry beside the IPv4 "0.0.0.0" one), and their order is not guaranteed, +// so it ranges over every binding rather than trusting index 0. +func requirePublishedOnAllInterfaces(t *testing.T, containerID string, containerPort int) { + t.Helper() + out, err := exec.Command("docker", "inspect", "-f", + fmt.Sprintf(`{{range (index .NetworkSettings.Ports "%d/tcp")}}{{.HostIp}} {{end}}`, containerPort), + containerID).Output() + require.NoError(t, err) + hostIPs := strings.Fields(string(out)) + require.NotEmpty(t, hostIPs, "container port %d has no published host binding", containerPort) + require.NotContains(t, hostIPs, "127.0.0.1", "container port %d must not be published loopback-only", containerPort) + require.Contains(t, hostIPs, "0.0.0.0", "container port %d must publish on all interfaces", containerPort) +} diff --git a/sbom_e2e_test.go b/sbom_e2e_test.go new file mode 100644 index 0000000..d6801b5 --- /dev/null +++ b/sbom_e2e_test.go @@ -0,0 +1,91 @@ +//go:build e2e + +// This file drives the agent's image-SBOM path against a real gateway image, +// inventorying it exactly as a release does and checking the result against the +// shared coverage contract. Run with: +// +// SWH_GATEWAY_IMAGE= go test -tags e2e -run TestImageSBOM . +// +// It needs syft on PATH: an image the local daemon holds and no registry serves +// is only reachable by a scanner with Docker access, which the managed +// containerized scanner deliberately does not have. +// +// Writing the evidence out is the release command's job (cmd/image-sbom), not +// this test's. +package main + +import ( + "context" + "os" + "os/exec" + "strings" + "testing" + + "github.com/codefly-dev/core/agents/services/sbom" + builderv0 "github.com/codefly-dev/core/generated/go/codefly/services/builder/v0" + "github.com/stretchr/testify/require" +) + +func TestImageSBOMInventoriesTheGatewayImage(t *testing.T) { + override := os.Getenv(gatewayImageOverrideEnv) + if override == "" { + t.Skipf("set %s to a locally built gateway image", gatewayImageOverrideEnv) + } + if _, err := exec.LookPath("syft"); err != nil { + t.Skip("syft is required to inventory an image held only by the local daemon") + } + builder := newSBOMBuilder(t) + subjects, err := builder.imageSubjects(context.Background()) + require.NoError(t, err) + + // The daemon resolves the override to its own image ID, so that ID is what a + // scan of it binds evidence to and what the subject has to name. + require.Len(t, subjects, 1, "the single platform the daemon holds") + require.Equal(t, sbom.SourceDockerDaemon, sbom.SourceOf(subjects[0])) + require.Equal(t, override, subjects[0].GetReference()) + require.Empty(t, subjects[0].GetPlatform(), "the platform is read from the image rather than assumed") + require.NoError(t, sbom.RequirePinned(subjects[0])) + + resp, err := builder.SBOM(context.Background(), &builderv0.SBOMRequest{Scope: builderv0.SBOMScope_SBOM_SCOPE_IMAGE}) + require.NoError(t, err) + require.Equal(t, builderv0.SBOMStatus_COMPLETE, resp.GetState().GetState(), resp.GetState().GetMessage()) + require.NoError(t, sbom.ValidateCoverage(testService, subjects, resp)) + + require.Len(t, resp.GetImages(), 1) + evidence := resp.GetImages()[0] + + // The subject named the local image ID before the scan ran; evidence that + // resolved to any other identity describes an image nobody asked for. + require.Equal(t, subjects[0].GetDigest(), evidence.GetDigest()) + require.True(t, strings.HasPrefix(evidence.GetDigest(), "sha256:")) + require.NotEmpty(t, evidence.GetPlatform(), "evidence names no platform") + require.NotEmpty(t, evidence.GetSha256(), "evidence carries no document checksum") + + var goModules, osPackages int + for _, component := range evidence.GetBom().GetComponents() { + switch purl := component.GetPurl(); { + case strings.HasPrefix(purl, "pkg:golang/"): + goModules++ + case isOSPackage(purl): + osPackages++ + } + } + require.NotZero(t, goModules, "inventory found no Go modules: the gateway's application dependencies are missing") + require.NotZero(t, osPackages, "inventory found no OS packages: an image inventory is not satisfied by application dependencies alone") + t.Logf("inventoried %s on %s (%d Go modules, %d OS packages)", evidence.GetDigest(), evidence.GetPlatform(), goModules, osPackages) +} + +// osPackageTypes are the package-URL namespaces an OS inventory reports under. +// Most of what syft finds in an image carries no package URL at all, so counting +// everything that is not a Go module would let an image with no OS inventory +// pass as covered. +var osPackageTypes = []string{"pkg:deb/", "pkg:apk/", "pkg:rpm/"} + +func isOSPackage(purl string) bool { + for _, prefix := range osPackageTypes { + if strings.HasPrefix(purl, prefix) { + return true + } + } + return false +} diff --git a/sbom_test.go b/sbom_test.go new file mode 100644 index 0000000..3624efb --- /dev/null +++ b/sbom_test.go @@ -0,0 +1,541 @@ +package main + +import ( + "context" + "fmt" + "os" + "os/exec" + "path/filepath" + "slices" + "sort" + "strings" + "testing" + + "github.com/codefly-dev/core/agents/services/sbom" + basev0 "github.com/codefly-dev/core/generated/go/codefly/base/v0" + agentv0 "github.com/codefly-dev/core/generated/go/codefly/services/agent/v0" + builderv0 "github.com/codefly-dev/core/generated/go/codefly/services/builder/v0" + "github.com/stretchr/testify/require" + "gopkg.in/yaml.v3" + + "github.com/codefly-dev/service-warehouse/internal/imageevidence" +) + +// testService is the identity newSBOMBuilder loads, and therefore the identity +// every subject it produces carries. +const testService = "module/warehouse" + +// pinnedReference is a digest-pinned image reference of the shape a release +// scans. It stands in for the published image so these tests mean the same thing +// before the first image is published and after. +const pinnedReference = registry + "/service-warehouse@" + digest + +// newSBOMBuilder returns a Builder wired for a headless SBOM call. +func newSBOMBuilder(t *testing.T) *Builder { + t.Helper() + builder := NewBuilder() + identity := &basev0.ServiceIdentity{ + Workspace: "workspace", Module: "module", Name: "warehouse", Version: "1.2.3", + WorkspacePath: t.TempDir(), RelativeToWorkspace: ".", + } + require.NoError(t, builder.Base.HeadlessLoad(context.Background(), identity)) + return builder +} + +func TestImageSubjectsCoverEveryPublishedPlatform(t *testing.T) { + subjects, err := imageevidence.Subjects(context.Background(), testService, pinnedReference, false) + require.NoError(t, err) + + require.Len(t, subjects, len(imageevidence.Platforms), "one subject per shipped platform") + covered := map[string]bool{} + for _, subject := range subjects { + require.Equal(t, sbom.SourceRegistry, sbom.SourceOf(subject)) + require.NoError(t, sbom.RequirePinned(subject)) + require.Equal(t, pinnedReference, subject.GetReference()) + require.Equal(t, imageevidence.Role, subject.GetRole()) + require.Equal(t, testService, subject.GetService()) + covered[subject.GetPlatform()] = true + } + for _, platform := range imageevidence.Platforms { + require.True(t, covered[platform], "no subject covers shipped platform %s", platform) + } +} + +// The registry derivation is core's, and it refuses what the evidence contract +// refuses: a tag serves whatever was pushed to it last, so a subject naming one +// says nothing about the image that was built. +func TestImageSubjectsRefuseAReferenceThatIsNotPinned(t *testing.T) { + for _, reference := range []string{ + registry + "/service-warehouse", + registry + "/service-warehouse:latest", + registry + "/service-warehouse:0.0.1", + } { + _, err := imageevidence.Subjects(context.Background(), testService, reference, false) + require.Error(t, err, reference) + } +} + +// What the Builder enumerates for itself is the pinned image gateway-image.json +// records, however many platforms that is. While the lock records no image there +// is nothing to enumerate, and saying so is the only honest answer. +func TestBuilderEnumeratesThePublishedImageOrRefuses(t *testing.T) { + t.Setenv(gatewayImageOverrideEnv, "") + builder := newSBOMBuilder(t) + + subjects, err := builder.imageSubjects(context.Background()) + if _, lockErr := publishedGatewayImage(); lockErr != nil { + require.ErrorIs(t, err, lockErr) + return + } + require.NoError(t, err) + published, err := publishedGatewayImage() + require.NoError(t, err) + require.Len(t, subjects, len(imageevidence.Platforms)) + for _, subject := range subjects { + require.Equal(t, published.FullName(), subject.GetReference()) + require.Equal(t, testService, subject.GetService()) + } +} + +func TestImageEvidenceSatisfiesTheCoverageContract(t *testing.T) { + subjects, err := imageevidence.Subjects(context.Background(), testService, pinnedReference, false) + require.NoError(t, err) + + // The expectation is built from the platforms the release pipeline actually + // publishes, never from imageevidence: deriving both sides from the code + // under test compares it against itself, and would validate however many + // platforms it dropped. + expected := expectedFromReleasePipeline(t, pinnedReference) + + require.NoError(t, sbom.ValidateCoverage(testService, expected, imageResponse(evidenceFor(subjects))), + "evidence for every published platform must validate as coverage") + require.Error(t, sbom.ValidateCoverage(testService, expected, imageResponse(evidenceFor(subjects[:1]))), + "evidence omitting a published platform must not validate as coverage") +} + +// The daemon resolves whatever reference it is handed to its own image ID, so a +// subject for a local image has to carry that ID: evidence bound to an identity +// the subject never named is not coverage of it. Resolving it can fail — the +// override may name an image the daemon does not hold — and reporting that is +// the only alternative to producing a subject pinned to nothing. +// +// The resolved case needs a real image and is asserted by the e2e inventory. +func TestGatewayOverrideMustBeHeldByTheDaemon(t *testing.T) { + const absent = "swh-gateway-unit-test-absent:none" + t.Setenv(gatewayImageOverrideEnv, absent) + + subjects, err := newSBOMBuilder(t).imageSubjects(context.Background()) + require.Error(t, err, "got %d subjects for an image the daemon does not hold", len(subjects)) + require.Contains(t, err.Error(), absent, "the error must name the override that could not be resolved") +} + +func TestImageScopeReportsFailureRatherThanCoverage(t *testing.T) { + for name, setup := range map[string]func(*testing.T){ + "an override that is not an image": func(t *testing.T) { + t.Setenv(gatewayImageOverrideEnv, "not a valid image reference") + }, + "a lock that records no image": func(t *testing.T) { + if _, err := publishedGatewayImage(); err == nil { + t.Skip("gateway-image.json records a published image") + } + t.Setenv(gatewayImageOverrideEnv, "") + }, + } { + t.Run(name, func(t *testing.T) { + setup(t) + resp, err := newSBOMBuilder(t).SBOM(context.Background(), &builderv0.SBOMRequest{Scope: builderv0.SBOMScope_SBOM_SCOPE_IMAGE}) + require.NoError(t, err) + + require.Equal(t, builderv0.SBOMStatus_ERROR, resp.GetState().GetState()) + require.Equal(t, builderv0.SBOMScope_SBOM_SCOPE_IMAGE, resp.GetScope(), "the failure is attributed to the image request") + require.Empty(t, resp.GetImages(), "a failed scan carries no inventory") + require.Error(t, sbom.ValidateCoverage(testService, nil, resp), "a failed scan must not validate as coverage") + }) + } +} + +func TestSourceScopeKeepsTheExistingInventory(t *testing.T) { + t.Setenv(gatewayImageOverrideEnv, "") + subjects, err := imageevidence.Subjects(context.Background(), testService, pinnedReference, false) + require.NoError(t, err) + + // A cancelled context stops the scan before it reaches the network, which is + // enough to tell the two paths apart: the image path attributes even its + // failures to SBOM_SCOPE_IMAGE, so an unspecified scope that does not come + // back image-scoped was answered by the source inventory. + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + resp, err := newSBOMBuilder(t).SBOM(ctx, &builderv0.SBOMRequest{}) + require.NoError(t, err) + + require.NotEqual(t, builderv0.SBOMScope_SBOM_SCOPE_IMAGE, resp.GetScope(), "an unspecified scope must not be answered with image evidence") + require.Error(t, sbom.ValidateCoverage(testService, subjects, resp), "a source inventory must not validate as image coverage") +} + +// Serving the image scope is what lets the agent answer; advertising +// ValidationCapabilities.image_sbom is what gets the phase scheduled against +// every consumer, and core's docs/sbom.md holds that back until each of them +// runs a core that can represent it. A present Validation is also authoritative +// for every operation it omits, so setting one to advertise this capability +// would declare test unsupported for a service that has no suite to run. Flipping +// this is a rollout decision, which is why a test, not a comment, marks it. +func TestImageSBOMIsServedButNotAdvertised(t *testing.T) { + info, err := NewService().GetAgentInformation(context.Background(), &agentv0.AgentInformationRequest{}) + require.NoError(t, err) + require.Nil(t, info.GetValidation(), "advertising image_sbom is a fleet rollout decision, not this agent's") + + // And it is served: an explicit subject is scanned rather than answered as + // unsupported, so the failure here is the scanner's, not a missing + // implementation. + subject := &builderv0.ImageSubject{Reference: "not-pinned", Service: testService, Role: imageevidence.Role} + resp, err := newSBOMBuilder(t).SBOM(context.Background(), &builderv0.SBOMRequest{ + Scope: builderv0.SBOMScope_SBOM_SCOPE_IMAGE, + Subjects: []*builderv0.ImageSubject{subject}, + }) + require.NoError(t, err) + require.Equal(t, builderv0.SBOMStatus_ERROR, resp.GetState().GetState(), resp.GetState().GetMessage()) + require.NotEqual(t, builderv0.SBOMStatus_UNSUPPORTED, resp.GetState().GetState()) +} + +func TestShippedPlatformsMatchTheReleasePipeline(t *testing.T) { + published := append([]string(nil), publishedPlatforms(t)...) + declared := append([]string(nil), imageevidence.Platforms...) + // Compare as sets: the pipeline and the code have to name the same + // platforms, but neither owns the order the other writes them in. + sort.Strings(published) + sort.Strings(declared) + + require.Equal(t, published, declared, + "the pipeline publishes %v but evidence covers %v; a platform shipped without a subject ships uninventoried", published, declared) +} + +// The release once looped over a platform list written into the workflow and +// called the scanner itself, which is a second list to keep in sync and a +// second scanner to drift from the contract. Evidence comes from the release +// command, which reads the same platform list the agent does. +func TestReleaseEvidenceComesFromTheSharedCommand(t *testing.T) { + body := workflowBody(t, "release.yml") + + require.Contains(t, body, "cmd/image-sbom", "the release workflow does not generate evidence through cmd/image-sbom") + require.NotContains(t, body, "for platform in", "the release workflow loops over its own platform list") + require.NotContains(t, workflowBody(t, "publish-gateway-image.yml"), "for platform in", + "the publish workflow loops over its own platform list; the platforms it builds are read from its build-push step") +} + +// expectedFromReleasePipeline builds the coverage expectation from the publish +// workflow, independently of the code that produces the evidence. +func expectedFromReleasePipeline(t *testing.T, reference string) []*builderv0.ImageSubject { + t.Helper() + platforms := publishedPlatforms(t) + expected := make([]*builderv0.ImageSubject, 0, len(platforms)) + for _, platform := range platforms { + expected = append(expected, &builderv0.ImageSubject{ + Reference: reference, + Platform: platform, + Role: imageevidence.Role, + Service: testService, + }) + } + return expected +} + +// publishedPlatforms reads the platforms the pipeline actually builds, from the +// step that pushes the image rather than from any step that happens to carry a +// platform list. +func publishedPlatforms(t *testing.T) []string { + t.Helper() + var workflow struct { + Jobs map[string]struct { + Steps []struct { + Uses string `yaml:"uses"` + With struct { + Platforms string `yaml:"platforms"` + } `yaml:"with"` + } `yaml:"steps"` + } `yaml:"jobs"` + } + require.NoError(t, yaml.Unmarshal([]byte(workflowBody(t, "publish-gateway-image.yml")), &workflow)) + var published []string + for _, job := range workflow.Jobs { + for _, step := range job.Steps { + if !strings.Contains(step.Uses, "docker/build-push-action") { + continue + } + for _, platform := range strings.Split(step.With.Platforms, ",") { + if platform = strings.TrimSpace(platform); platform != "" { + published = append(published, platform) + } + } + } + } + require.NotEmpty(t, published, "the publish workflow's build-push step publishes no platforms") + return published +} + +func workflowBody(t *testing.T, name string) string { + t.Helper() + data, err := os.ReadFile(filepath.Join(".github", "workflows", name)) + require.NoError(t, err) + return string(data) +} + +// releaseStep is one step of the release workflow, kept in file order so a test +// can assert what happens before what. +type releaseStep struct { + ID string `yaml:"id"` + Name string `yaml:"name"` + Uses string `yaml:"uses"` + Run string `yaml:"run"` + Env map[string]string `yaml:"env"` +} + +type releaseJob struct { + Needs any `yaml:"needs"` + Uses string `yaml:"uses"` + Steps []releaseStep `yaml:"steps"` +} + +func releaseJobs(t *testing.T) map[string]releaseJob { + t.Helper() + var workflow struct { + Jobs map[string]releaseJob `yaml:"jobs"` + } + require.NoError(t, yaml.Unmarshal([]byte(workflowBody(t, "release.yml")), &workflow)) + return workflow.Jobs +} + +// releaseStepRunning returns the one release step whose script contains marker. +func releaseStepRunning(t *testing.T, marker string) releaseStep { + t.Helper() + var found []releaseStep + for _, job := range releaseJobs(t) { + for _, step := range job.Steps { + if strings.Contains(step.Run, marker) { + found = append(found, step) + } + } + } + require.Len(t, found, 1, "want exactly one release step running %q", marker) + return found[0] +} + +// stepIndex is where in the release job order marker first runs, counting the +// jobs in name order so a step added to another job shifts every index the same +// way on each run: Go randomises map iteration. +func stepIndex(t *testing.T, marker string) int { + t.Helper() + jobs := releaseJobs(t) + names := make([]string, 0, len(jobs)) + for name := range jobs { + names = append(names, name) + } + sort.Strings(names) + index := 0 + for _, name := range names { + for _, step := range jobs[name].Steps { + if strings.Contains(step.Run, marker) { + return index + } + index++ + } + } + return -1 +} + +// The release used to push the tags consumers resolve and inventory the image +// afterwards. A failure anywhere in the evidence step then turned the run red +// with a pullable, uninventoried image already published — and a red run reads +// as "nothing shipped", so nobody goes looking for it. The image is now pushed +// by digest before the tag exists, and the tags are created only once evidence +// for that digest exists. +func TestReleasePublishesTagsOnlyAfterEvidence(t *testing.T) { + for jobName, job := range releaseJobs(t) { + for _, step := range job.Steps { + require.NotContains(t, step.Uses, "docker/build-push-action", + "%s builds the gateway image; the digest the agent pins cannot come from the tag that ships it, so the image is published before the tag", jobName) + } + } + evidence, publish := stepIndex(t, "cmd/image-sbom"), stepIndex(t, "imagetools create") + require.GreaterOrEqual(t, evidence, 0, "the release workflow does not generate image evidence") + require.GreaterOrEqual(t, publish, 0, "the release workflow never creates the tags consumers resolve") + require.Greater(t, publish, evidence, "release tags are created before evidence: a failed scan would leave a pullable image with no evidence") +} + +// The gateway image the release tags is the one gateway-image.json records, and +// the agent runs that digest rather than a tag. Reading it from anywhere else — +// resolving :latest, or rebuilding — publishes tags for bytes the released +// agent never names. +func TestReleaseTagsTheRecordedDigest(t *testing.T) { + publish := releaseStepRunning(t, "imagetools create") + require.Contains(t, publish.Run, `"$IMAGE@$DIGEST"`, "the tag-publishing step does not create its tags from a recorded digest") + + locked := releaseStepRunning(t, "gateway-image.json") + require.Contains(t, locked.Run, "org.opencontainers.image.version", + "the release accepts the recorded digest without checking it was built for this version; a lock left unrefreshed publishes the previous release's image under the new tag") +} + +// The release must refuse the tree as it is merged: gateway-image.json records +// no digest until the first image is published, and `jq -e` on its own passes an +// empty string. This runs the workflow's own guard against that file, not a +// description of it. It stops before any registry call, so it needs no network. +func TestReleaseRefusesALockThatRecordsNoImage(t *testing.T) { + if _, err := exec.LookPath("jq"); err != nil { + t.Skip("jq is required to run the release's lock check") + } + if _, err := publishedGatewayImage(); err == nil { + t.Skip("gateway-image.json records a published image; this guards the state before one exists") + } + locked := releaseStepRunning(t, "gateway-image.json") + lock, err := os.ReadFile("gateway-image.json") + require.NoError(t, err) + + dir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(dir, "gateway-image.json"), lock, 0o600)) + command := exec.Command("bash", "-c", locked.Run) + command.Dir = dir + command.Env = append(os.Environ(), "VERSION=0.0.1", "IMAGE="+registry+"/service-warehouse") + out, err := command.CombinedOutput() + require.Error(t, err, "the release accepted a lock that records no image:\n%s", out) + require.Contains(t, string(out), "no gateway image has been published") +} + +// The published image tag is derived from the git tag while the agent reports +// the version of the embedded agent.codefly.yaml. Nothing reconciles them at run +// time, and a tag one release ahead of the file makes a consumer pinned to one +// version run another. This runs the workflow's own guard against the real file +// rather than trusting that it is present. +func TestReleaseGuardRejectsTagThatDisagreesWithAgentVersion(t *testing.T) { + guard := releaseStepRunning(t, "agent.codefly.yaml") + + run := func(version string) error { + command := exec.Command("bash", "-c", guard.Run) + command.Env = append(os.Environ(), "VERSION="+version) + return command.Run() + } + + require.NoError(t, run(agent.Version), "guard rejected the version this repo actually carries") + require.Error(t, run(agent.Version+"9"), "guard accepted a tag that disagrees with agent.codefly.yaml") +} + +// needs is a job's `needs`, which GitHub accepts as one job name or a list. +func needs(job releaseJob) []string { + switch value := job.Needs.(type) { + case string: + return []string{value} + case []any: + names := make([]string, 0, len(value)) + for _, name := range value { + names = append(names, fmt.Sprint(name)) + } + return names + } + return nil +} + +// runsAfter reports whether job cannot start until earlier has succeeded. +func runsAfter(jobs map[string]releaseJob, job, earlier string) bool { + seen := map[string]bool{} + queue := needs(jobs[job]) + for len(queue) > 0 { + next := queue[0] + queue = queue[1:] + if next == earlier { + return true + } + if !seen[next] { + seen[next] = true + queue = append(queue, needs(jobs[next])...) + } + } + return false +} + +// A `v*` tag must start one workflow, because separate workflows cannot order or +// gate each other: either could fail while the other published, and the harmful +// direction is the release becoming `latest` while no registry serves the image +// its agent names. Everything a tag publishes hangs off one chain, ordered by +// the dependency that actually exists. +func TestATagPublishesThroughOneOrderedWorkflow(t *testing.T) { + entries, err := os.ReadDir(filepath.Join(".github", "workflows")) + require.NoError(t, err) + var onTag []string + for _, entry := range entries { + if !entry.Type().IsRegular() || (filepath.Ext(entry.Name()) != ".yml" && filepath.Ext(entry.Name()) != ".yaml") { + continue + } + var workflow struct { + On map[string]any `yaml:"on"` + } + require.NoError(t, yaml.Unmarshal([]byte(workflowBody(t, entry.Name())), &workflow), entry.Name()) + push, ok := workflow.On["push"].(map[string]any) + if !ok { + continue + } + // A push trigger covers branches and tags alike: naming only branches + // drops tags, while naming neither filter leaves both live. + _, tags := push["tags"] + _, tagsIgnore := push["tags-ignore"] + _, branches := push["branches"] + _, branchesIgnore := push["branches-ignore"] + if tags || tagsIgnore || (!branches && !branchesIgnore) { + onTag = append(onTag, entry.Name()) + } + } + require.Equal(t, []string{"release.yml"}, onTag, "a tag push must start one workflow") + + jobs := releaseJobs(t) + var releases, images, gates []string + for name, job := range jobs { + if strings.Contains(job.Uses, "go-service-release.yml") { + releases = append(releases, name) + } + for _, step := range job.Steps { + if strings.Contains(step.Run, "imagetools create") { + images = append(images, name) + } + if strings.Contains(step.Run, "go test") { + gates = append(gates, name) + } + } + } + require.Len(t, releases, 1, "a tag ships one release") + require.NotEmpty(t, images) + require.NotEmpty(t, gates) + // Every image the tag publishes has to exist before the release names it, + // and has to be gated on the suite. Quantifying over the sets keeps a job + // added later from changing which pair happens to be checked. + for _, image := range images { + require.True(t, runsAfter(jobs, releases[0], image), + "job %q does not wait for %q: the release can ship an agent whose gateway image no registry resolves", releases[0], image) + require.True(t, slices.ContainsFunc(gates, func(gate string) bool { return runsAfter(jobs, image, gate) }), + "job %q waits for none of %v: a tag whose tests fail still publishes an image tagged :latest", image, gates) + } +} + +// imageResponse wraps evidence in the complete image-scope response an agent +// returns. +func imageResponse(images []*builderv0.ImageSBOM) *builderv0.SBOMResponse { + return &builderv0.SBOMResponse{ + State: &builderv0.SBOMStatus{State: builderv0.SBOMStatus_COMPLETE}, + Scope: builderv0.SBOMScope_SBOM_SCOPE_IMAGE, + Images: images, + } +} + +// evidenceFor builds one digest-bound inventory per subject, each with a +// distinct digest so coverage is matched per platform. +func evidenceFor(subjects []*builderv0.ImageSubject) []*builderv0.ImageSBOM { + images := make([]*builderv0.ImageSBOM, 0, len(subjects)) + for i, subject := range subjects { + images = append(images, &builderv0.ImageSBOM{ + Digest: fmt.Sprintf("sha256:%064d", i), + Platform: subject.GetPlatform(), + Subjects: []*builderv0.ImageSubject{subject}, + Bom: &agentv0.Bom{Components: []*agentv0.Component{{Name: "libc", Version: "2.36"}}}, + Tool: "syft", + Sha256: fmt.Sprintf("%064d", i), + }) + } + return images +} diff --git a/templates/agent/README.md.tmpl b/templates/agent/README.md.tmpl new file mode 100644 index 0000000..bb0a943 --- /dev/null +++ b/templates/agent/README.md.tmpl @@ -0,0 +1,12 @@ +# warehouse + +The warehouse gateway: a codefly/warehouse/v0 gRPC service in front of a SQL +warehouse. The backend is selected by configuration: the embedded engine +locally, a cloud warehouse when deployed. + +A local run starts the published gateway image with a fresh shared secret and +hands it to consumers as the secret `token` value of the `warehouse` +configuration group. Set `backend` in service.codefly.yaml to choose the engine; +left unset, the gateway's own default applies. See +https://github.com/codefly-dev/service-warehouse for the backends that serve +queries today. diff --git a/templates/deployment/kustomize/base/deployment.yaml.tmpl b/templates/deployment/kustomize/base/deployment.yaml.tmpl new file mode 100644 index 0000000..3bf299b --- /dev/null +++ b/templates/deployment/kustomize/base/deployment.yaml.tmpl @@ -0,0 +1,119 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{.Name}} + namespace: {{.Namespace}} +spec: + replicas: 1 + selector: + matchLabels: + app: {{.Name}} + template: + metadata: + labels: + app: {{.Name}} + spec: + automountServiceAccountToken: false + securityContext: + runAsNonRoot: true + runAsUser: 65532 + runAsGroup: 65532 + seccompProfile: + type: RuntimeDefault + terminationGracePeriodSeconds: 30 + containers: + - name: warehouse + image: {{ .Image }} + imagePullPolicy: IfNotPresent + securityContext: + allowPrivilegeEscalation: false + runAsNonRoot: true + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + seccompProfile: + type: RuntimeDefault + ports: + - name: grpc + containerPort: {{ .Deployment.Parameters.ContainerPort }} + env: + # The service identity the CLI binds the environment's service + # configuration onto: per environment, after this render, it + # replaces the SWH_* literals below by name on the container that + # declares this (and adds the ones not rendered here). + - name: CODEFLY__SERVICE + value: "{{ .Name }}" + - name: SWH_LISTEN + value: ":{{ .Deployment.Parameters.ContainerPort }}" +{{- if .Deployment.Parameters.Backend }} + - name: SWH_BACKEND + value: "{{ .Deployment.Parameters.Backend }}" +{{- end }} +{{- if .Deployment.Parameters.Database }} + - name: SWH_DATABASE + value: "{{ .Deployment.Parameters.Database }}" +{{- end }} +{{- if .Deployment.Parameters.Dataset }} + - name: SWH_DATASET + value: "{{ .Deployment.Parameters.Dataset }}" +{{- end }} +{{- if .Deployment.Parameters.Location }} + - name: SWH_LOCATION + value: "{{ .Deployment.Parameters.Location }}" +{{- end }} + # The gateway refuses to start on an unauthenticated listener unless + # this is set. In the deployed profile caller identity is enforced by + # the cluster — NetworkPolicy plus service-mesh mTLS in front of this + # ClusterIP — not by the gateway. + # + # Do not replace this with SWH_AUTH_TOKEN on its own. The gateway + # would then enforce a credential its consumers do not have, because + # nothing delivers that secret to their warehouse configuration. Both + # halves have to land together; see README "Authentication". + - name: SWH_ALLOW_ANONYMOUS + value: "true" +{{- /* No SWH_CREDENTIALS_FILE: nothing here mounts a credentials file, so a path +would name a file the pod does not have. A deployed cloud warehouse authenticates +through a workload identity bound to the ServiceAccount below, or through a file +an overlay outside this agent mounts and patches the variable for; Deploy rejects +a configured path rather than emit one. Kept as a template comment: rendering it +as YAML would ship that prose into every manifest. */}} +{{- if not .Restricted }} + envFrom: + - secretRef: + name: {{.Name}}-secret +{{- end }} + resources: + requests: + cpu: 50m + memory: 128Mi + limits: + cpu: "1" + memory: 1Gi + # The gateway's gRPC health service reports that the process is serving + # and nothing more: it does not probe the warehouse, so there is no + # service to name here that would attest warehouse access, and all + # three probes check the overall ("") service. A warehouse outage is + # reported per request instead, and does not drain or restart replicas + # that share one warehouse and one credential set. + # + # The grpc probe handler requires Kubernetes >= 1.24 (GA in 1.27); on + # older API servers the field is pruned and the Deployment is rejected + # for specifying no handler. + startupProbe: + grpc: + port: {{ .Deployment.Parameters.ContainerPort }} + periodSeconds: 2 + failureThreshold: 30 + readinessProbe: + grpc: + port: {{ .Deployment.Parameters.ContainerPort }} + periodSeconds: 5 + timeoutSeconds: 3 + livenessProbe: + grpc: + port: {{ .Deployment.Parameters.ContainerPort }} + periodSeconds: 30 + timeoutSeconds: 5 + failureThreshold: 3 diff --git a/templates/deployment/kustomize/base/kustomization.yaml.tmpl b/templates/deployment/kustomize/base/kustomization.yaml.tmpl new file mode 100644 index 0000000..d94a00e --- /dev/null +++ b/templates/deployment/kustomize/base/kustomization.yaml.tmpl @@ -0,0 +1,9 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: +{{- if not .Restricted }} + - namespace.yaml +{{- end }} + - deployment.yaml + - service.yaml diff --git a/templates/deployment/kustomize/base/namespace.yaml.tmpl b/templates/deployment/kustomize/base/namespace.yaml.tmpl new file mode 100644 index 0000000..f24694a --- /dev/null +++ b/templates/deployment/kustomize/base/namespace.yaml.tmpl @@ -0,0 +1,8 @@ +{{- if not .Restricted }} +apiVersion: v1 +kind: Namespace +metadata: + name: {{.Namespace}} + labels: + app.kubernetes.io/managed-by: codefly +{{- end }} diff --git a/templates/deployment/kustomize/base/service.yaml.tmpl b/templates/deployment/kustomize/base/service.yaml.tmpl new file mode 100644 index 0000000..7944a40 --- /dev/null +++ b/templates/deployment/kustomize/base/service.yaml.tmpl @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{.Name}} + namespace: {{.Namespace}} +spec: + selector: + app: {{.Name}} + ports: + - name: grpc + port: {{ .Deployment.Parameters.ServicePort }} + targetPort: {{ .Deployment.Parameters.ContainerPort }} diff --git a/templates/deployment/kustomize/overlays/environment/kustomization.yaml.tmpl b/templates/deployment/kustomize/overlays/environment/kustomization.yaml.tmpl new file mode 100644 index 0000000..904871f --- /dev/null +++ b/templates/deployment/kustomize/overlays/environment/kustomization.yaml.tmpl @@ -0,0 +1,8 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - ../../base +{{- if not .Restricted }} + - secret.yaml +{{- end }} diff --git a/templates/deployment/kustomize/overlays/environment/secret.yaml.tmpl b/templates/deployment/kustomize/overlays/environment/secret.yaml.tmpl new file mode 100644 index 0000000..7ff4522 --- /dev/null +++ b/templates/deployment/kustomize/overlays/environment/secret.yaml.tmpl @@ -0,0 +1,12 @@ +{{- if not .Restricted }} +apiVersion: v1 +kind: Secret +metadata: + name: {{.Name}}-secret + namespace: {{.Namespace}} +type: Opaque +data: +{{- range $key, $value := .SecretMap }} + {{ $key }}: "{{ $value }}" +{{- end }} +{{- end }} diff --git a/templates/factory/GETTING_STARTED.md.tmpl b/templates/factory/GETTING_STARTED.md.tmpl new file mode 100644 index 0000000..c192dd1 --- /dev/null +++ b/templates/factory/GETTING_STARTED.md.tmpl @@ -0,0 +1,5 @@ +# Getting started + +This service runs the warehouse gateway. Locally it needs no credential: the +agent generates the shared secret the gateway enforces and delivers it to +consumers. diff --git a/templates/factory/configurations/local/warehouse.secret.env b/templates/factory/configurations/local/warehouse.secret.env new file mode 100644 index 0000000..f38aff3 --- /dev/null +++ b/templates/factory/configurations/local/warehouse.secret.env @@ -0,0 +1,5 @@ +# A local run needs no secrets. The agent generates the gateway's SWH_AUTH_TOKEN +# per run and hands it to consumers as the secret "token" value of the warehouse +# configuration group — nothing to set here. +# Deployed environments reach a cloud warehouse through a workload identity, not +# a credential file. From 299257f85c3f4b964d6b080592e057eb52303547 Mon Sep 17 00:00:00 2001 From: Antoine Toussaint <312536060+antoinetoussaint-byte@users.noreply.github.com> Date: Sun, 11 Oct 2026 08:54:35 -0400 Subject: [PATCH 05/10] fix(agent): adopt core v0.18.1 gateway primitives --- .github/workflows/agent-ci.yml | 4 +- .github/workflows/publish-gateway-image.yml | 77 ++------ .github/workflows/release.yml | 4 +- AGENTS.md | 17 +- README.md | 39 ++-- builder.go | 14 +- cmd/image-sbom/main.go | 13 +- cmd/service-warehouse/auth_test.go | 25 ++- cmd/service-warehouse/distribution_test.go | 13 +- cmd/service-warehouse/main.go | 8 +- composition_test.go | 4 +- gateway_image.go | 51 +---- gateway_image_test.go | 39 ++-- go.mod | 4 +- go.sum | 8 +- internal/auth/auth.go | 107 ----------- internal/auth/auth_test.go | 132 ------------- internal/config/config.go | 28 +-- internal/config/config_test.go | 4 +- internal/gatewayimage/image.go | 8 + internal/imageevidence/imageevidence.go | 191 ------------------- internal/imageevidence/imageevidence_test.go | 84 -------- readiness_test.go | 8 +- runtime.go | 18 +- runtime_e2e_test.go | 6 +- sbom_e2e_test.go | 7 +- sbom_test.go | 60 +++--- 27 files changed, 194 insertions(+), 779 deletions(-) delete mode 100644 internal/auth/auth.go delete mode 100644 internal/auth/auth_test.go create mode 100644 internal/gatewayimage/image.go delete mode 100644 internal/imageevidence/imageevidence.go delete mode 100644 internal/imageevidence/imageevidence_test.go diff --git a/.github/workflows/agent-ci.yml b/.github/workflows/agent-ci.yml index 8c3f8d3..727f826 100644 --- a/.github/workflows/agent-ci.yml +++ b/.github/workflows/agent-ci.yml @@ -19,8 +19,8 @@ on: jobs: agent-ci: # Pinned to an immutable core commit SHA, not @main (see release.yml): core's - # v0.16.0, the release go.mod requires. Bump the two together. - uses: codefly-dev/core/.github/workflows/agent-ci.yml@e54bc852f3d40cd1aeafc0ced2369a8c13a51bb8 + # v0.18.1, the release go.mod requires. Bump the two together. + uses: codefly-dev/core/.github/workflows/agent-ci.yml@ea49e7a968ef5f2fd325b1c41f823a47eb20141c with: # The CLI release the gate runs with. The go source-packager it installs # is the one the CLI pins, so bumping this is how that pin moves. diff --git a/.github/workflows/publish-gateway-image.yml b/.github/workflows/publish-gateway-image.yml index 37409f2..54f85b9 100644 --- a/.github/workflows/publish-gateway-image.yml +++ b/.github/workflows/publish-gateway-image.yml @@ -1,80 +1,25 @@ name: publish-gateway-image -# Builds the gateway image and pushes it by digest, carrying no tag. Nothing -# resolves it yet: release.yml creates the tags consumers use, from the digest -# gateway-image.json records. -# -# It runs before the release tag rather than from it. The agent pins the digest -# it runs, and a digest is only known once the image is built — so a tag cut -# before the build can never carry it. Publishing here, recording the digest in -# a commit, and tagging afterwards is what lets the released agent name the -# image that was actually built for its version. +# Core builds by digest before the release tag; commit the returned lock before +# tagging. Its publisher refuses commits outside the default branch's history +# and checks out with persist-credentials: false. on: workflow_dispatch: inputs: version: - description: Upcoming stable agent version reported by codefly publish --dry-run (without v) + description: Upcoming stable agent version (without v) required: true type: string -env: - # The repository this runs in, never a typed name: the agent derives the same - # name from core's image registry and gatewayImageName, and - # TestGatewayImageNameIsThisRepository holds the two together. - IMAGE: ghcr.io/${{ github.repository }} - jobs: publish: - runs-on: ubuntu-latest permissions: contents: read packages: write - steps: - - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 - - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3 - - uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: Validate the upcoming agent version - id: version - env: - VERSION: ${{ inputs.version }} - run: | - set -euo pipefail - if [[ ! "$VERSION" =~ ^(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)$ ]]; then - echo "version must be a stable semantic version without a v prefix" >&2 - exit 1 - fi - echo "version=$VERSION" >> "$GITHUB_OUTPUT" - # The version label is how the release tells this image apart from the one - # published for an earlier version: both are digests in the same - # repository, and a lock left unrefreshed would otherwise publish the - # previous release's bytes under the new tag. - - uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6 - id: build - with: - context: . - platforms: linux/amd64,linux/arm64 - labels: org.opencontainers.image.version=${{ steps.version.outputs.version }} - outputs: type=image,name=${{ env.IMAGE }},push-by-digest=true,name-canonical=true,push=true - - name: Record the digest - env: - DIGEST: ${{ steps.build.outputs.digest }} - run: | - set -euo pipefail - mkdir -p release-evidence - jq -n --arg name "$IMAGE" --arg digest "$DIGEST" '{name: $name, digest: $digest}' > release-evidence/gateway-image.json - { - echo 'Commit this as `gateway-image.json`, then tag the release:' - echo - echo '```json' - cat release-evidence/gateway-image.json - echo '```' - } >> "$GITHUB_STEP_SUMMARY" - - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: gateway-image-lock-${{ steps.version.outputs.version }} - path: release-evidence/gateway-image.json - if-no-files-found: error + uses: codefly-dev/core/.github/workflows/publish-service-image.yml@ea49e7a968ef5f2fd325b1c41f823a47eb20141c # v0.18.1 + with: + version: ${{ inputs.version }} + lock-file: gateway-image.json + platforms: linux/amd64,linux/arm64 + secrets: + registry-token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a2ff2bd..848f9d5 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -153,9 +153,9 @@ jobs: needs: gateway-image # Pinned to an immutable core commit SHA, not @main: this call carries # secrets: inherit, so a mutable ref would hand an upstream change this - # repo's GH_PAT with no commit here. It is core's v0.16.0, the release go.mod + # repo's GH_PAT with no commit here. It is core's v0.18.1, the release go.mod # requires; bump the two together. This version of the workflow keeps the # release PAT out of jobs that run unreviewed code and refuses a tag not # reachable from the default branch. - uses: codefly-dev/core/.github/workflows/go-service-release.yml@e54bc852f3d40cd1aeafc0ced2369a8c13a51bb8 + uses: codefly-dev/core/.github/workflows/go-service-release.yml@ea49e7a968ef5f2fd325b1c41f823a47eb20141c secrets: inherit diff --git a/AGENTS.md b/AGENTS.md index 44826e7..5cbd4db 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -19,14 +19,14 @@ this file needs a fix in the same PR. tenant, a permission, a product noun, a business rule, or which application is asking. The server is bound to one database and authorizes nothing; the caller's authority is settled before the request arrives. -- **Depends on:** the warehouse drivers and the gRPC contract, nothing else. - The SBOM scanner and the image-SBOM contract live in `codefly-dev/core` and - are not reimplemented here. +- **Depends on:** the warehouse drivers, the gRPC contract, and core's lightweight + `callertoken` package. Caller authentication, image lock parsing, the SBOM + scanner and evidence writing live in `codefly-dev/core` and are not copied here. - **The agent is a second program in this module.** The repo-root `main` package (`main.go`, `runtime.go`, `builder.go`) is the codefly agent that - lets a workspace compose the gateway; it links `codefly-dev/core`, and - `cmd/service-warehouse` must not. The agent links no backend, so no cgo - driver reaches its `CGO_ENABLED=0` release build + lets a workspace compose the gateway; it links core's agent runtime, which + `cmd/service-warehouse` must not (it uses only core's `callertoken`). The agent + links no backend, so no cgo driver reaches its `CGO_ENABLED=0` release build (`TestAgentLinksNoBackendDriver`). Like the gateway, it authorizes nothing: it starts the gateway, hands out the one token it enforces, and renders the manifest. @@ -156,8 +156,9 @@ rather than run `:latest`. The tag must match `version:` in `agent.codefly.yaml` or the job fails by design. Core is pinned in two places that move together: the `codefly-dev/core` version -in `go.mod` and the commit SHA of core's reusable workflows in `release.yml` and -`agent-ci.yml` (never `@main`: `release.yml` passes `secrets: inherit`). +in `go.mod` and the commit SHA of core's reusable workflows in `release.yml`, +`agent-ci.yml` and `publish-gateway-image.yml` (never `@main`: `release.yml` +passes `secrets: inherit`). ## Where the depth is diff --git a/README.md b/README.md index 8feb675..29bba5d 100644 --- a/README.md +++ b/README.md @@ -109,10 +109,11 @@ With `SWH_AUTH_TOKEN` set, every RPC — unary and streaming, `Capabilities` included — must carry the secret as the `x-codefly-token` gRPC metadata header, the key the Codefly host already uses to authenticate agent plugins. Anything else is rejected with `UNAUTHENTICATED` before it reaches a backend. The token -is compared in constant time and never logged. The standard gRPC health service -(`grpc.health.v1.Health`) is the one exemption: a readiness probe carries no -token. It reports that the process is serving, not that a warehouse is -reachable. +is compared in constant time and never logged by core's `callertoken` package. +Only `grpc.health.v1.Health/Check` and `/Watch` are exempt; `/List` still needs +the token. These probes report that the process is serving, not that a warehouse +is reachable. An interceptor configured with an empty server token fails closed; +the explicit anonymous mode installs no token interceptor. This is authentication only — whether the caller holds the deployment's token. The server still authorizes nothing: which caller may run which statement is @@ -131,8 +132,8 @@ image, pinned by digest in [`gateway-image.json`](gateway-image.json). **What a consumer receives.** The configuration group `warehouse`, with `connection` (`grpc://host:port`), `endpoint` (`host:port`) and `token`, a secret value to send as the `x-codefly-token` metadata header. The agent mints a -fresh token for every local run unless an operator pins one, so a run revokes the -previous run's. +fresh 64-character lowercase hex token (256 random bits) for every local run +unless an operator pins one, so a run revokes the previous run's. **Locally** (`codefly run`) the agent starts the gateway container, publishes its port, waits until the gateway reports itself serving *and* accepts the agent's @@ -174,11 +175,12 @@ that was scanned and attributed to the service. The scanner is core's (`sbom.Image`, through `BuilderWrapper.SBOMImages`), and the response is checked against core's `sbom.ValidateCoverage`. -- [`internal/imageevidence`](internal/imageevidence/imageevidence.go) holds the - one list of shipped platforms and the subjects derived from it; the agent, the - release command and the tests all read it. -- [`cmd/image-sbom`](cmd/image-sbom/main.go) writes the documents. A release runs - it against the digest [`gateway-image.json`](gateway-image.json) records and +- [`internal/gatewayimage`](internal/gatewayimage/image.go) declares the shipped + platforms and role. Core's `sbom.ImageSubjects` derives one subject per declared + platform; local images get their digest and platform from the Docker daemon. +- [`cmd/image-sbom`](cmd/image-sbom/main.go) is a thin entry point to core's + `CollectImageEvidence` and `WriteImageEvidenceIndex`. A release runs it + against the digest [`gateway-image.json`](gateway-image.json) records and attaches them as the `gateway-image-sbom-` workflow artifact, before the tags consumers resolve are created, so an image whose scan failed is never pullable by the version it was built for. @@ -211,7 +213,10 @@ The digest is only known once the image is built, so the order is fixed, and 1. Find the upcoming version with `codefly publish --dry-run`. 2. Run the `publish-gateway-image` workflow with that version (no `v`). It - pushes the image by digest, carrying no tag, and prints `gateway-image.json`. + calls core's pinned `publish-service-image.yml`, pushes the image by digest, + carrying no tag, and prints `gateway-image.json`. The publisher uses + `persist-credentials: false` and refuses a commit outside the default branch's + history. 3. Commit that file in a reviewed change. Until it is committed the lock records no digest, and the agent, the deployment and the release all refuse to name an image. @@ -222,7 +227,12 @@ The digest is only known once the image is built, so the order is fixed, and The release call needs a `GH_PAT` secret in the repository (`secrets: inherit`). -[sbom]: https://github.com/codefly-dev/core/blob/v0.16.0/docs/sbom.md +The lock is parsed by core's `resources.ParseImageLock`: it must name +`ghcr.io/codefly-dev/service-warehouse`, contain only `name` and `digest`, and +use `sha256:` followed by 64 lowercase hex characters. An empty digest records +an unpublished image and is refused by the agent and release. + +[sbom]: https://github.com/codefly-dev/core/blob/v0.18.1/docs/sbom.md ## Develop @@ -269,12 +279,11 @@ proto/buf.{yaml,gen.yaml} buf config, read by the proto companion gen/ generated gRPC stubs internal/backend/ Backend interface + mem, duckdb (+ cloud backends) internal/server/ gRPC Warehouse implementation + proto↔backend mapping -internal/auth/ caller authentication (the x-codefly-token interceptors) internal/config/ env configuration internal/serr/ normalized error model cmd/service-warehouse/ the server binary cmd/image-sbom/ writes the image's SBOM evidence (run by the release) -internal/imageevidence/ the shipped platforms and the SBOM subjects derived from them +internal/gatewayimage/ the shipped platforms and runtime role; core derives evidence main.go runtime.go builder.go the codefly agent (codefly.dev/warehouse) that runs the gateway agent.codefly.yaml the agent's manifest and version gateway-image.json the gateway image the agent pins, by digest diff --git a/builder.go b/builder.go index f73dc39..3718871 100644 --- a/builder.go +++ b/builder.go @@ -8,13 +8,14 @@ import ( "github.com/codefly-dev/core/agents/communicate" "github.com/codefly-dev/core/agents/services" + "github.com/codefly-dev/core/agents/services/sbom" v0 "github.com/codefly-dev/core/generated/go/codefly/base/v0" builderv0 "github.com/codefly-dev/core/generated/go/codefly/services/builder/v0" "github.com/codefly-dev/core/resources" "github.com/codefly-dev/core/standards" "github.com/codefly-dev/core/wool" - "github.com/codefly-dev/service-warehouse/internal/imageevidence" + "github.com/codefly-dev/service-warehouse/internal/gatewayimage" ) type Builder struct { @@ -111,15 +112,18 @@ func (s *Builder) SBOM(ctx context.Context, req *builderv0.SBOMRequest) (*builde // them. The gateway is the only one: the service has no init, migration or // sidecar image, and the warehouse behind it is not run by this service. // -// The subjects themselves come from imageevidence, which the release command -// reads too, so the evidence a release publishes describes the same images the -// agent reports. +// Core derives the subjects from the same gateway metadata the release command +// uses, so release evidence describes the images the agent reports. func (s *Builder) imageSubjects(ctx context.Context) ([]*builderv0.ImageSubject, error) { image, overridden, err := effectiveGatewayImage() if err != nil { return nil, err } - return imageevidence.Subjects(ctx, s.Base.Unique(), image.FullName(), overridden) + published := sbom.PublishedImage{Service: s.Base.Unique(), Role: gatewayimage.Role, Reference: image.FullName()} + if !overridden { + published.Platforms = gatewayimage.Platforms + } + return sbom.ImageSubjects(ctx, published, overridden) } func (s *Builder) Deploy(ctx context.Context, req *builderv0.DeploymentRequest) (*builderv0.DeploymentResponse, error) { diff --git a/cmd/image-sbom/main.go b/cmd/image-sbom/main.go index be50af2..fccf60c 100644 --- a/cmd/image-sbom/main.go +++ b/cmd/image-sbom/main.go @@ -14,10 +14,11 @@ import ( "fmt" "os" + "github.com/codefly-dev/core/agents/services/sbom" "github.com/codefly-dev/core/resources" "github.com/codefly-dev/core/shared" - "github.com/codefly-dev/service-warehouse/internal/imageevidence" + "github.com/codefly-dev/service-warehouse/internal/gatewayimage" ) func main() { @@ -41,17 +42,21 @@ func main() { } ctx := context.Background() - subjects, err := imageevidence.Subjects(ctx, *service, *image, *local) + published := sbom.PublishedImage{Service: *service, Role: gatewayimage.Role, Reference: *image} + if !*local { + published.Platforms = gatewayimage.Platforms + } + subjects, err := sbom.ImageSubjects(ctx, published, *local) if err != nil { fmt.Fprintf(os.Stderr, "image-sbom: %v\n", err) os.Exit(1) } - documents, err := imageevidence.Collect(ctx, *out, subjects) + documents, err := sbom.CollectImageEvidence(ctx, *out, subjects) if err != nil { fmt.Fprintf(os.Stderr, "image-sbom: %v\n", err) os.Exit(1) } - if err := imageevidence.WriteIndex(*out, documents); err != nil { + if err := sbom.WriteImageEvidenceIndex(*out, documents); err != nil { fmt.Fprintf(os.Stderr, "image-sbom: %v\n", err) os.Exit(1) } diff --git a/cmd/service-warehouse/auth_test.go b/cmd/service-warehouse/auth_test.go index 78355e1..937d4a4 100644 --- a/cmd/service-warehouse/auth_test.go +++ b/cmd/service-warehouse/auth_test.go @@ -9,6 +9,7 @@ import ( "testing" "time" + "github.com/codefly-dev/core/callertoken" "github.com/stretchr/testify/require" "google.golang.org/grpc" "google.golang.org/grpc/codes" @@ -20,7 +21,6 @@ import ( "google.golang.org/protobuf/types/known/emptypb" whv0 "github.com/codefly-dev/service-warehouse/gen/codefly/warehouse/v0" - "github.com/codefly-dev/service-warehouse/internal/auth" "github.com/codefly-dev/service-warehouse/internal/backend" "github.com/codefly-dev/service-warehouse/internal/backend/mem" ) @@ -56,7 +56,7 @@ type rpc struct { } // registeredRPCs lists every method of every service the server registers -// except the health service, read from the server rather than from a list kept +// except health Check/Watch, read from the server rather than from a list kept // beside it: an RPC added to the proto, or a service added to the server, is // covered by the refusal tests the moment it exists, without anyone remembering // to extend them. @@ -70,7 +70,7 @@ func registeredRPCs(t *testing.T, srv *grpc.Server) []rpc { for service, info := range srv.GetServiceInfo() { for _, m := range info.Methods { full := "/" + service + "/" + m.Name - if auth.IsHealthMethod(full) { + if callertoken.IsHealthMethod(full) { continue } rpcs = append(rpcs, rpc{name: full, call: caller(full, m)}) @@ -140,10 +140,10 @@ func TestUnauthenticatedPeerIsRefused(t *testing.T) { peers := map[string]*grpc.ClientConn{ "no credentials": dial(), - "wrong token": dial(auth.DialOption("00000000000000000000000000000000")), - "prefix of token": dial(auth.DialOption(serverToken[:len(serverToken)-1])), - "token plus extra": dial(auth.DialOption(serverToken + "x")), - "empty token": dial(auth.DialOption("")), + "wrong token": dial(callertoken.DialOption("00000000000000000000000000000000")), + "prefix of token": dial(callertoken.DialOption(serverToken[:len(serverToken)-1])), + "token plus extra": dial(callertoken.DialOption(serverToken + "x")), + "empty token": dial(callertoken.DialOption("")), } for peerName, conn := range peers { @@ -165,7 +165,7 @@ func TestUnauthenticatedPeerIsRefused(t *testing.T) { // Unimplemented) still proves the interceptor let the call through. func TestAuthorizedPeerReachesTheHandler(t *testing.T) { srv, dial := startGateway(t, serverToken) - conn := dial(auth.DialOption(serverToken)) + conn := dial(callertoken.DialOption(serverToken)) for _, r := range registeredRPCs(t, srv) { t.Run(r.name, func(t *testing.T) { @@ -181,7 +181,7 @@ func TestAuthorizedPeerReachesTheHandler(t *testing.T) { // the same metadata key the codefly host uses for agent plugins, so a caller // built against that convention interoperates. func TestTokenTravelsUnderTheEcosystemKey(t *testing.T) { - require.Equal(t, "x-codefly-token", auth.MetadataKey) + require.Equal(t, "x-codefly-token", callertoken.MetadataKey) _, dial := startGateway(t, serverToken) conn := dial() @@ -207,7 +207,7 @@ func TestTokenUnderAnotherHeaderIsRefused(t *testing.T) { } // TestHealthNeedsNoToken pins the one exemption: the standard gRPC health -// service answers a peer that presents no token. A readiness probe — the Codefly +// Check and Watch answer a peer that presents no token. A readiness probe — the Codefly // CLI's, a load balancer's, a mesh's — carries none, and a guarded health service // makes the gateway look permanently unready to all of them. Every data RPC stays // refused; TestUnauthenticatedPeerIsRefused holds that side. @@ -226,6 +226,11 @@ func TestHealthNeedsNoToken(t *testing.T) { first, err := stream.Recv() require.NoError(t, err) require.Equal(t, healthv1.HealthCheckResponse_SERVING, first.GetStatus()) + + _, err = health.List(ctx, &healthv1.HealthListRequest{}) + require.Equal(t, codes.Unauthenticated, status.Code(err), "List enumerates services and is not exempt") + _, err = healthv1.NewHealthClient(dial(callertoken.DialOption(serverToken))).List(ctx, &healthv1.HealthListRequest{}) + require.NoError(t, err) } // TestWithoutATokenTheListenerIsAnonymous is the other side of the config guard: diff --git a/cmd/service-warehouse/distribution_test.go b/cmd/service-warehouse/distribution_test.go index a39d77f..ea2d49e 100644 --- a/cmd/service-warehouse/distribution_test.go +++ b/cmd/service-warehouse/distribution_test.go @@ -24,6 +24,7 @@ var imageBuildCommands = []string{ "docker build", "docker buildx", "docker/build-push-action", + "codefly-dev/core/.github/workflows/publish-service-image.yml@", } // manifestExtensions can name an image this service deploys. Evidence binds to @@ -134,7 +135,7 @@ func contentSignal(path, rel string) (string, error) { // to be re-checked against it. var shippedImage = []string{ ".github/workflows/ci.yml: workflow runs docker build", - ".github/workflows/publish-gateway-image.yml: workflow runs docker/build-push-action", + ".github/workflows/publish-gateway-image.yml: workflow runs codefly-dev/core/.github/workflows/publish-service-image.yml@", ".github/workflows/release.yml: workflow runs docker build", "Dockerfile: container image build definition", "agent.codefly.yaml: Codefly agent manifest, whose Builder ships image build recipes", @@ -237,7 +238,7 @@ func TestImageSignalsFindsAWorkflowImageBuild(t *testing.T) { // the production list cannot fail when a command is dropped from it, which // would silently narrow the gate. commands := []string{"docker build", "docker buildx", "docker/build-push-action"} - if len(commands) != len(imageBuildCommands) { + if len(commands)+1 != len(imageBuildCommands) { t.Fatalf("imageBuildCommands is %v; update this test to match", imageBuildCommands) } for _, command := range commands { @@ -251,6 +252,14 @@ func TestImageSignalsFindsAWorkflowImageBuild(t *testing.T) { } } +// Shared image publishers are image-producing paths even without local steps. +func TestImageSignalsFindsTheCoreImagePublisher(t *testing.T) { + root := t.TempDir() + path := filepath.Join(".github", "workflows", "publish.yml") + write(t, root, path, "jobs:\n publish:\n uses: codefly-dev/core/.github/workflows/publish-service-image.yml@revision\n") + requireSignal(t, root, path) +} + // TestImageSignalsFindsADeployedImage covers an image this repository deploys // without building: the evidence contract binds to the digest built *or // selected for deployment*, so a manifest naming an image is image coverage diff --git a/cmd/service-warehouse/main.go b/cmd/service-warehouse/main.go index abbb011..76abd0f 100644 --- a/cmd/service-warehouse/main.go +++ b/cmd/service-warehouse/main.go @@ -14,12 +14,12 @@ import ( "syscall" "time" + "github.com/codefly-dev/core/callertoken" "google.golang.org/grpc" grpchealth "google.golang.org/grpc/health" healthv1 "google.golang.org/grpc/health/grpc_health_v1" whv0 "github.com/codefly-dev/service-warehouse/gen/codefly/warehouse/v0" - "github.com/codefly-dev/service-warehouse/internal/auth" "github.com/codefly-dev/service-warehouse/internal/backend" "github.com/codefly-dev/service-warehouse/internal/config" "github.com/codefly-dev/service-warehouse/internal/server" @@ -83,7 +83,7 @@ func run() error { // newGRPCServer assembles the gateway's gRPC server: the Warehouse service over // be, the standard health service, and — when token is set — the interceptors // that refuse any call that does not present it. The health service is exempt -// from them (see auth.HealthServicePrefix) so a readiness probe, which carries +// from them (only Check and Watch) so a readiness probe, which carries // no token, can still reach it. // // The health service reports the process as serving and nothing more. It is @@ -93,8 +93,8 @@ func newGRPCServer(be backend.Backend, token string) *grpc.Server { var options []grpc.ServerOption if token != "" { options = append(options, - grpc.ChainUnaryInterceptor(auth.UnaryInterceptor(token)), - grpc.ChainStreamInterceptor(auth.StreamInterceptor(token)), + grpc.ChainUnaryInterceptor(callertoken.UnaryServerInterceptor(token)), + grpc.ChainStreamInterceptor(callertoken.StreamServerInterceptor(token)), ) } srv := grpc.NewServer(options...) diff --git a/composition_test.go b/composition_test.go index 7a108e6..23b55f9 100644 --- a/composition_test.go +++ b/composition_test.go @@ -100,7 +100,7 @@ func TestResolveGatewayToken(t *testing.T) { require.NoError(t, err) second, err := rt.resolveGatewayToken() require.NoError(t, err) - require.Len(t, first, 48, "24 random bytes, hex encoded") + require.Regexp(t, `^[0-9a-f]{64}$`, first, "32 random bytes, hex encoded") require.NotEqual(t, first, second, "each run mints a fresh token, which is what revokes the last run's") } @@ -412,7 +412,7 @@ func TestDeployRefusesWhileNoGatewayImageIsPublished(t *testing.T) { resp, err := builder.Deploy(ctx, deployRequest(destination, nil, restricted)) require.NoError(t, err) require.Equal(t, builderv0.DeploymentStatus_ERROR, resp.GetState().GetState()) - require.Contains(t, resp.GetState().GetMessage(), "no gateway image has been published") + require.Contains(t, resp.GetState().GetMessage(), resources.ErrImageUnpublished.Error()) } // TestDeployRestrictedBindsTheWorkloadIdentity pins what infra has to grant diff --git a/gateway_image.go b/gateway_image.go index 823b964..01d9274 100644 --- a/gateway_image.go +++ b/gateway_image.go @@ -1,12 +1,9 @@ package main import ( - "encoding/hex" - "encoding/json" "errors" "fmt" "os" - "strings" "github.com/codefly-dev/core/resources" ) @@ -17,50 +14,14 @@ import ( // TestGatewayImageNameIsThisRepository holds the two together. const gatewayImageName = "service-warehouse" -// errGatewayImageUnpublished means gateway-image.json records no digest: the -// image has not been published for any version yet. It is a state the tree -// legitimately sits in between merging the agent and publishing its first -// image, and the only safe reading of it is a refusal. A reference with no -// digest and no tag is the registry's :latest, which is whatever was pushed -// last and which the evidence contract refuses to inventory. -var errGatewayImageUnpublished = errors.New("no gateway image has been published: gateway-image.json records no digest. " + - "Run publish-gateway-image.yml for the upcoming version and commit the digest it prints") - -// gatewayImageLock is the recorded identity of the published gateway image. It -// carries no tag: the tag is the agent version, and a second copy of it is one -// more thing to disagree with agent.codefly.yaml. -type gatewayImageLock struct { - Name string `json:"name"` - Digest string `json:"digest"` -} - -// parseGatewayImageLock returns the image the lock pins. The registry comes from -// core, never from the lock: a lock naming another registry is the lock being -// wrong, not an instruction to run an image from there. +// parseGatewayImageLock delegates lock validation to core, adding the service's +// recovery instruction when no image has been published yet. func parseGatewayImageLock(content []byte) (*resources.DockerImage, error) { - var lock gatewayImageLock - if err := json.Unmarshal(content, &lock); err != nil { - return nil, fmt.Errorf("parse gateway image lock: %w", err) - } - image := resources.PublishedImage(gatewayImageName, "") - if want := image.Repository + "/" + image.Name; lock.Name != want { - return nil, fmt.Errorf("gateway image lock names %q, but this service's gateway image is %q", lock.Name, want) + image, err := resources.ParseImageLock(content, gatewayImageName) + if errors.Is(err, resources.ErrImageUnpublished) { + return nil, fmt.Errorf("%w: run publish-gateway-image.yml for the upcoming version and commit the digest it prints", err) } - if lock.Digest == "" { - return nil, errGatewayImageUnpublished - } - if !isSHA256Digest(lock.Digest) { - return nil, fmt.Errorf("gateway image digest %q must be a sha256 digest", lock.Digest) - } - image.Digest = lock.Digest - return &image, nil -} - -// isSHA256Digest reports whether digest is a complete sha256 image digest. -func isSHA256Digest(digest string) bool { - algorithm, encoded, found := strings.Cut(digest, ":") - decoded, err := hex.DecodeString(encoded) - return found && algorithm == "sha256" && err == nil && len(decoded) == 32 + return image, err } // gatewayImageOverrideEnv names a locally built gateway image to run instead of diff --git a/gateway_image_test.go b/gateway_image_test.go index 01abe0e..9c74a6a 100644 --- a/gateway_image_test.go +++ b/gateway_image_test.go @@ -8,11 +8,9 @@ import ( "strings" "testing" - "github.com/codefly-dev/core/agents" "github.com/codefly-dev/core/resources" "github.com/stretchr/testify/require" - "github.com/codefly-dev/service-warehouse/internal/auth" "github.com/codefly-dev/service-warehouse/internal/config" // The mem backend is registered here, in a test, so config.FromEnv can @@ -50,11 +48,15 @@ func TestParseGatewayImageLockRejectsAnythingButASHA256Digest(t *testing.T) { "truncated": `{"name": "` + name + `", "digest": "sha256:1205da07"}`, "not hex": `{"name": "` + name + `", "digest": "sha256:zzzzda0743e56b68c12c754438476e8fdd0f05cdb08ce1c9687ddb8d8f0e1669"}`, "not json": `sha256:1205da07`, + "uppercase digest": `{"name": "` + name + `", "digest": "` + strings.ToUpper(digest) + `"}`, + "uppercase hex": `{"name": "` + name + `", "digest": "sha256:` + strings.ToUpper(strings.TrimPrefix(digest, "sha256:")) + `"}`, + "unknown key": `{"name": "` + name + `", "digest": "` + digest + `", "tag": "latest"}`, + "trailing json": `{"name": "` + name + `", "digest": "` + digest + `"} {}`, } { t.Run(label, func(t *testing.T) { _, err := parseGatewayImageLock([]byte(lock)) require.Error(t, err, "accepted a lock that pins no image") - require.NotErrorIs(t, err, errGatewayImageUnpublished) + require.NotErrorIs(t, err, resources.ErrImageUnpublished) }) } } @@ -64,7 +66,7 @@ func TestParseGatewayImageLockRejectsAnythingButASHA256Digest(t *testing.T) { // safe reading is a refusal that says what to do. func TestAnUnpublishedLockIsRefusedNeverRunAsLatest(t *testing.T) { image, err := parseGatewayImageLock([]byte(`{"name": "` + registry + `/service-warehouse", "digest": ""}`)) - require.ErrorIs(t, err, errGatewayImageUnpublished) + require.ErrorIs(t, err, resources.ErrImageUnpublished) require.Nil(t, image) require.Contains(t, err.Error(), "publish-gateway-image.yml") } @@ -99,7 +101,7 @@ func TestTheOverrideDecidesTheRunButNotThePublishedImage(t *testing.T) { if err == nil { require.NotEqual(t, "service-warehouse:local", published.FullName()) } else { - require.ErrorIs(t, err, errGatewayImageUnpublished) + require.ErrorIs(t, err, resources.ErrImageUnpublished) } t.Setenv(gatewayImageOverrideEnv, "") @@ -127,9 +129,9 @@ func TestGatewayImageNameIsThisRepository(t *testing.T) { require.NoError(t, err) require.Contains(t, string(lock), `"name": "ghcr.io/`+repository+`"`) - for _, workflow := range []string{"publish-gateway-image.yml", "release.yml"} { - require.Contains(t, workflowBody(t, workflow), "IMAGE: ghcr.io/${{ github.repository }}", workflow) - } + require.Contains(t, workflowBody(t, "release.yml"), "IMAGE: ghcr.io/${{ github.repository }}") + // The shared publisher derives the image name from the caller repository. + require.NotContains(t, workflowBody(t, "publish-gateway-image.yml"), "image-name:") } // The container port is stated once in the agent and handed to the gateway as @@ -151,13 +153,6 @@ func TestGatewayContainerPortIsTheGatewaysDefault(t *testing.T) { require.Regexp(t, regexp.MustCompile(fmt.Sprintf(`(?m)^EXPOSE %d$`, gatewayContainerPort)), string(dockerfile)) } -// The agent presents the token under the key the gateway enforces. internal/auth -// redeclares the key because the gateway must not link core's agent runtime; the -// agent does link it, so this is where a divergence is caught. -func TestTheTokenKeyIsTheOneCoreAuthenticatesAgentsWith(t *testing.T) { - require.Equal(t, agents.AuthMetadataKey, auth.MetadataKey) -} - // The agent is released without cgo (.goreleaser.yaml) and must stay free of the // gateway's backends: the embedded DuckDB engine is a cgo driver, and linking it // here would turn the release's CGO_ENABLED=0 build into one that fails the day @@ -171,3 +166,17 @@ func TestAgentLinksNoBackendDriver(t *testing.T) { "the agent links %s, a backend the gateway carries", dependency) } } + +// The gateway may use core's standalone caller token, but must not acquire the +// agent runtime or any of its resource orchestration dependencies. +func TestGatewayLinksOnlyCoreCallerToken(t *testing.T) { + out, err := exec.Command("go", "list", "-deps", "-f", "{{.ImportPath}}", "./cmd/service-warehouse").Output() + require.NoError(t, err) + var linked []string + for _, dependency := range strings.Fields(string(out)) { + if strings.HasPrefix(dependency, "github.com/codefly-dev/core/") { + linked = append(linked, dependency) + } + } + require.Equal(t, []string{"github.com/codefly-dev/core/callertoken"}, linked) +} diff --git a/go.mod b/go.mod index 0d39d12..8b6cc9a 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/codefly-dev/service-warehouse go 1.27.0 require ( - github.com/codefly-dev/core v0.16.0 + github.com/codefly-dev/core v0.18.1 github.com/stretchr/testify v1.12.1 google.golang.org/grpc v1.83.2 google.golang.org/protobuf v1.36.12 @@ -99,7 +99,7 @@ require ( go.yaml.in/yaml/v2 v2.4.2 // indirect go.yaml.in/yaml/v3 v3.0.5 // indirect golang.org/x/exp v0.0.0-20260820142414-ca536658362e // indirect - golang.org/x/net v0.59.0 // indirect + golang.org/x/net v0.60.0 // indirect golang.org/x/sys v0.48.0 // indirect golang.org/x/term v0.46.0 // indirect golang.org/x/text v0.42.0 // indirect diff --git a/go.sum b/go.sum index 800ce44..a221a4e 100644 --- a/go.sum +++ b/go.sum @@ -38,8 +38,8 @@ github.com/clipperhouse/uax29/v2 v2.7.0 h1:+gs4oBZ2gPfVrKPthwbMzWZDaAFPGYK72F0NJ github.com/clipperhouse/uax29/v2 v2.7.0/go.mod h1:EFJ2TJMRUaplDxHKj1qAEhCtQPW2tJSwu5BF98AuoVM= github.com/cloudflare/circl v1.6.3 h1:9GPOhQGF9MCYUeXyMYlqTR6a5gTrgR/fBLXvUgtVcg8= github.com/cloudflare/circl v1.6.3/go.mod h1:2eXP6Qfat4O/Yhh8BznvKnJ+uzEoTQ6jVKJRn81BiS4= -github.com/codefly-dev/core v0.16.0 h1:iJQ9SteSBRaGs+jeD/DqgKMsukJfTKvnqcYTjsOBjDs= -github.com/codefly-dev/core v0.16.0/go.mod h1:rp4YZkCJEOufliWQAgK3sv8YlO7Xcva7+cN7hIlAR0I= +github.com/codefly-dev/core v0.18.1 h1:W1RtH/Hjly+rh+BQlWB2HYYO3jdQmAe+bkhERro85bU= +github.com/codefly-dev/core v0.18.1/go.mod h1:FKBqn6GMGIsx9b3LNY6kKCRxCEjjeswvqt2CxSZWq20= github.com/codefly-dev/gortk v0.2.0 h1:7bOlS5valYz2zil+fZctQNcPCYBcPj86abcw9N8h1hQ= github.com/codefly-dev/gortk v0.2.0/go.mod h1:dDWUMFgAP063OCGhTEpV7FFUj9+zTcxxO/nV80VKEwg= github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI= @@ -278,8 +278,8 @@ golang.org/x/exp v0.0.0-20260820142414-ca536658362e h1:01Ju2A/fZKkci4zqx0eZxw//D golang.org/x/exp v0.0.0-20260820142414-ca536658362e/go.mod h1:zeBbvyFKDaLwa7CH/zI8KXt7gTl14SF7sO08Pl5jBCM= golang.org/x/mod v0.41.0 h1:qJmnOUb4YB+FsEuM3HcWucdZASCPGhsX6uljO6pog0c= golang.org/x/mod v0.41.0/go.mod h1:Ek9pY8RKWXwsWvd3rQiHYtMqkjSUV+s1Rj7j4H5Ur6o= -golang.org/x/net v0.59.0 h1:5zfYln+w5XCxwrnMMJPufRgNoXEaGxl0wo5GqPXyues= -golang.org/x/net v0.59.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg= +golang.org/x/net v0.60.0 h1:79p50tfZlm0J9YfoDsSi639qSXNGVwEzOPLCxM2FsYU= +golang.org/x/net v0.60.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/internal/auth/auth.go b/internal/auth/auth.go deleted file mode 100644 index 193fcdd..0000000 --- a/internal/auth/auth.go +++ /dev/null @@ -1,107 +0,0 @@ -// Package auth authenticates callers of the gateway's gRPC surface: does the -// caller present the one token this deployment was started with. It decides -// nothing else — which caller may run which statement is settled before the -// request arrives (see AGENTS.md), so there is no identity here, no -// per-method rule, and no statement inspection. -// -// It is the same mechanism codefly-dev/service-object-storage uses -// (internal/auth) and the one the Codefly host already uses to talk to agent -// plugins: a shared secret carried as the "x-codefly-token" metadata header, -// compared in constant time. No exported equivalent exists to import: -// codefly-dev/core exports only the key (agents.AuthMetadataKey, in the -// package that links the whole agent runtime) and keeps its interceptors -// unexported, and codefly-dev/sdk-go has none. The key is therefore redeclared -// here, and this package is a copy of service-object-storage's; a fix to one -// belongs in the other. -package auth - -import ( - "context" - "crypto/subtle" - "strings" - - "google.golang.org/grpc" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/metadata" - "google.golang.org/grpc/status" -) - -// MetadataKey is the gRPC metadata key carrying the token. Lowercase per gRPC -// convention — metadata keys are case-insensitive but the wire form is -// lowercase. -const MetadataKey = "x-codefly-token" - -// HealthServicePrefix is the standard gRPC health service. Its Check and Watch -// carry no data and no authority: a readiness probe — the Codefly CLI's, a -// load balancer's, a mesh's — presents no token, so a guarded health service -// makes the gateway look permanently unready to everything that would route to -// it. Every other method stays behind the token. -const HealthServicePrefix = "/grpc.health.v1.Health/" - -// IsHealthMethod reports whether fullMethod belongs to the gRPC health service. -func IsHealthMethod(fullMethod string) bool { - return strings.HasPrefix(fullMethod, HealthServicePrefix) -} - -// UnaryInterceptor rejects every unary call that does not present token. -func UnaryInterceptor(token string) grpc.UnaryServerInterceptor { - return func(ctx context.Context, req any, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (any, error) { - if !IsHealthMethod(info.FullMethod) { - if err := verify(ctx, token); err != nil { - return nil, err - } - } - return handler(ctx, req) - } -} - -// StreamInterceptor rejects every streaming call that does not present token. -// gRPC runs it before the handler observes a single frame, so an unauthorized -// InsertRows is refused at stream open rather than after its first batch is -// accepted. -func StreamInterceptor(token string) grpc.StreamServerInterceptor { - return func(srv any, ss grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) error { - if !IsHealthMethod(info.FullMethod) { - if err := verify(ss.Context(), token); err != nil { - return err - } - } - return handler(srv, ss) - } -} - -// DialOption is how a client presents token on every call it makes to the -// gateway. -func DialOption(token string) grpc.DialOption { - return grpc.WithPerRPCCredentials(bearer(token)) -} - -func verify(ctx context.Context, expected string) error { - md, ok := metadata.FromIncomingContext(ctx) - if !ok { - return status.Error(codes.Unauthenticated, "missing "+MetadataKey) - } - presented := md.Get(MetadataKey) - // An empty presented value would compare equal to an empty expected one, so - // it is refused as absent rather than reaching the comparison. - if len(presented) == 0 || presented[0] == "" { - return status.Error(codes.Unauthenticated, "missing "+MetadataKey) - } - if subtle.ConstantTimeCompare([]byte(presented[0]), []byte(expected)) != 1 { - return status.Error(codes.Unauthenticated, "invalid "+MetadataKey) - } - return nil -} - -// bearer carries the token as per-RPC credentials. -type bearer string - -func (b bearer) GetRequestMetadata(context.Context, ...string) (map[string]string, error) { - return map[string]string{MetadataKey: string(b)}, nil -} - -// RequireTransportSecurity is false: the local profile reaches the gateway over -// the Docker host bridge without TLS, and the deployed profile terminates -// transport security at the mesh. Demanding it here would make the credential -// unusable on both. -func (b bearer) RequireTransportSecurity() bool { return false } diff --git a/internal/auth/auth_test.go b/internal/auth/auth_test.go deleted file mode 100644 index dab3f3a..0000000 --- a/internal/auth/auth_test.go +++ /dev/null @@ -1,132 +0,0 @@ -package auth_test - -import ( - "context" - "testing" - - "github.com/stretchr/testify/require" - "google.golang.org/grpc" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/metadata" - "google.golang.org/grpc/status" - - "github.com/codefly-dev/service-warehouse/internal/auth" -) - -const token = "89d0d1f6d0e2f5a4c3b2a1908f7e6d5c" - -// peerContext is the context a handler sees for a call whose metadata carries -// the given header values; none means the call carried no metadata at all. -func peerContext(values ...string) context.Context { - if len(values) == 0 { - return context.Background() - } - return metadata.NewIncomingContext(context.Background(), metadata.Pairs(auth.MetadataKey, values[0])) -} - -// serverStream is the least a stream interceptor needs of a ServerStream: the -// context the call arrived with. -type serverStream struct { - grpc.ServerStream - ctx context.Context -} - -func (s serverStream) Context() context.Context { return s.ctx } - -// callUnary runs one unary call through the interceptor and reports whether -// the handler was reached and what the caller was told. -func callUnary(ctx context.Context, method string) (reached bool, err error) { - _, err = auth.UnaryInterceptor(token)(ctx, nil, &grpc.UnaryServerInfo{FullMethod: method}, - func(context.Context, any) (any, error) { - reached = true - return nil, nil - }) - return reached, err -} - -func callStream(ctx context.Context, method string) (reached bool, err error) { - err = auth.StreamInterceptor(token)(nil, serverStream{ctx: ctx}, &grpc.StreamServerInfo{FullMethod: method}, - func(any, grpc.ServerStream) error { - reached = true - return nil - }) - return reached, err -} - -const dataMethod = "/codefly.warehouse.v0.Warehouse/Query" - -// Every way of not presenting the token is refused with Unauthenticated before -// the handler runs, on both interceptors. The near misses matter most: a -// comparison that stopped at the shorter length, or that accepted an empty -// value against an empty expectation, would pass an obvious wrong-token test. -func TestOnlyTheExactTokenReachesTheHandler(t *testing.T) { - refused := map[string][]string{ - "no metadata": nil, - "empty value": {""}, - "wrong token": {"00000000000000000000000000000000"}, - "prefix of token": {token[:len(token)-1]}, - "token plus extra": {token + "x"}, - "different case": {"89D0D1F6D0E2F5A4C3B2A1908F7E6D5C"}, - } - for name, values := range refused { - t.Run(name, func(t *testing.T) { - reached, err := callUnary(peerContext(values...), dataMethod) - require.False(t, reached, "the handler must not run for a refused unary call") - require.Equal(t, codes.Unauthenticated, status.Code(err), "got %v", err) - - reached, err = callStream(peerContext(values...), dataMethod) - require.False(t, reached, "the handler must not run for a refused stream") - require.Equal(t, codes.Unauthenticated, status.Code(err), "got %v", err) - }) - } - - reached, err := callUnary(peerContext(token), dataMethod) - require.NoError(t, err) - require.True(t, reached) - - reached, err = callStream(peerContext(token), dataMethod) - require.NoError(t, err) - require.True(t, reached) -} - -// The refusal says which header was wrong, not what the right one is. -func TestRefusalNeverEchoesTheExpectedToken(t *testing.T) { - _, err := callUnary(peerContext("wrong"), dataMethod) - require.Error(t, err) - require.NotContains(t, err.Error(), token) -} - -// The health service is the one exemption: a readiness probe carries no token. -// Anything that merely looks like it does not inherit it. -func TestOnlyTheHealthServiceIsExempt(t *testing.T) { - for _, method := range []string{"/grpc.health.v1.Health/Check", "/grpc.health.v1.Health/Watch"} { - t.Run(method, func(t *testing.T) { - require.True(t, auth.IsHealthMethod(method)) - - reached, err := callUnary(peerContext(), method) - require.NoError(t, err) - require.True(t, reached) - - reached, err = callStream(peerContext(), method) - require.NoError(t, err) - require.True(t, reached) - }) - } - - for _, method := range []string{ - "/grpc.health.v1.HealthX/Check", - "/grpc.health.v1.Healthy/Check", - "/other.Service/grpc.health.v1.Health/Check", - "/codefly.warehouse.v0.Warehouse/Health", - "grpc.health.v1.Health/Check", - "", - } { - t.Run("not "+method, func(t *testing.T) { - require.False(t, auth.IsHealthMethod(method)) - - reached, err := callUnary(peerContext(), method) - require.False(t, reached) - require.Equal(t, codes.Unauthenticated, status.Code(err), "got %v", err) - }) - } -} diff --git a/internal/config/config.go b/internal/config/config.go index 1a9c8a2..6597b6d 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -12,7 +12,7 @@ import ( "strings" "time" - "github.com/codefly-dev/service-warehouse/internal/auth" + "github.com/codefly-dev/core/callertoken" "github.com/codefly-dev/service-warehouse/internal/backend" ) @@ -21,7 +21,7 @@ type Config struct { ListenAddr string Backend backend.Config // AuthToken is the shared secret every caller must present as the - // auth.MetadataKey metadata header, whitespace-trimmed: a secret delivered + // callertoken.MetadataKey metadata header, whitespace-trimmed: a secret delivered // through a file or a block scalar arrives with a trailing newline that no // client would send. Empty means the listener is anonymous, which FromEnv // resolves only when the operator asked for it with SWH_ALLOW_ANONYMOUS. @@ -101,25 +101,11 @@ func checkListenerIsAuthenticated(token string) error { if err != nil { return err } - if token != "" { - // Both set is a contradiction, not a preference: "here is a credential" - // and "accept callers with no credential" cannot both be the intent, and - // silently picking one leaves the operator believing the other. It is - // also the shape a half-finished migration takes — a token added to the - // secret while the manifest still carries the opt-out. - if anonymous { - return fmt.Errorf("SWH_AUTH_TOKEN and SWH_ALLOW_ANONYMOUS=true are mutually exclusive: " + - "unset SWH_ALLOW_ANONYMOUS to enforce the token, or unset SWH_AUTH_TOKEN to accept anonymous callers") - } - return nil - } - if anonymous { - return nil - } - return fmt.Errorf("SWH_AUTH_TOKEN is required: without it every caller that can reach the listener can run SQL against the bound database. "+ - "Set SWH_AUTH_TOKEN to a shared secret and have clients send it as the %q gRPC metadata header, "+ - "or set SWH_ALLOW_ANONYMOUS=true if the listener is confined to a private boundary enforced elsewhere (cluster NetworkPolicy / service-mesh mTLS)", - auth.MetadataKey) + return callertoken.CheckListener(token, anonymous, callertoken.Listener{ + TokenVar: "SWH_AUTH_TOKEN", + AnonymousVar: "SWH_ALLOW_ANONYMOUS", + Exposure: "can run SQL against the bound database", + }) } func env(key, def string) string { diff --git a/internal/config/config_test.go b/internal/config/config_test.go index b99af96..552e952 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -146,14 +146,14 @@ func TestUnsetLimitsFallBackToDefaults(t *testing.T) { // TestFromEnvRefusesAnUnauthenticatedListener is the startup guard: with // neither a token nor an explicit opt-out the server does not resolve, and the -// message names both ways out and the header a client must send. +// message names both ways out and the exposed capability. func TestFromEnvRefusesAnUnauthenticatedListener(t *testing.T) { clearEnv(t) t.Setenv("SWH_AUTH_TOKEN", "") _, err := FromEnv() require.Error(t, err) - for _, want := range []string{"SWH_AUTH_TOKEN", "x-codefly-token", "SWH_ALLOW_ANONYMOUS"} { + for _, want := range []string{"SWH_AUTH_TOKEN", "can run SQL", "SWH_ALLOW_ANONYMOUS"} { require.ErrorContains(t, err, want) } } diff --git a/internal/gatewayimage/image.go b/internal/gatewayimage/image.go new file mode 100644 index 0000000..5f58cb9 --- /dev/null +++ b/internal/gatewayimage/image.go @@ -0,0 +1,8 @@ +// Package gatewayimage declares the gateway's shipped platforms and role. +// Core owns subject derivation, scanning, and evidence writing. +package gatewayimage + +// Platforms is held to the publish workflow's explicit platform input by tests. +var Platforms = []string{"linux/amd64", "linux/arm64"} + +const Role = "runtime" diff --git a/internal/imageevidence/imageevidence.go b/internal/imageevidence/imageevidence.go deleted file mode 100644 index 2981f7b..0000000 --- a/internal/imageevidence/imageevidence.go +++ /dev/null @@ -1,191 +0,0 @@ -// Package imageevidence holds the one description of the images this service -// ships: the platforms the release pipeline publishes, the subjects that -// describe them, and the CycloneDX documents inventorying them. -// -// The agent's Builder.SBOM, the release command (cmd/image-sbom), and the drift -// test all read it from here. A second copy of the platform list is how a -// platform gets added to the pipeline and silently shipped with no evidence, so -// there is only one. -// -// What core exports is imported, not redeclared: the scanner (sbom.Image), the -// subject derivation for a pushed image (sbom.ExpectedFromImageReference), the -// scanner source selector and the CycloneDX encoder. Core exports no helper -// that writes a scanned image's documents to disk or resolves a locally built -// image's ID, so Collect, WriteIndex and localImageID below are this -// package's, adapted from codefly-dev/service-object-storage's -// internal/imageevidence (the same three functions); a fix to one belongs in -// the other until core grows an exported equivalent. -package imageevidence - -import ( - "context" - "fmt" - "os" - "os/exec" - "path/filepath" - "strings" - - "github.com/codefly-dev/core/agents/services/sbom" - builderv0 "github.com/codefly-dev/core/generated/go/codefly/services/builder/v0" -) - -// Platforms are the platforms the release pipeline publishes the gateway image -// for, and therefore the platforms evidence has to cover: a multi-architecture -// image is not inventoried by scanning one of its children. It is asserted -// against .github/workflows/publish-gateway-image.yml. -var Platforms = []string{"linux/amd64", "linux/arm64"} - -// Role is the gateway's purpose within the service, carried on every subject. -const Role = "runtime" - -// Subjects describes the images the service ships. A published image is -// multi-architecture and contributes one subject per shipped platform; core -// derives them from the digest-pinned reference, which is what binds each scan -// to the image that was built and which core refuses to accept as a bare tag. A -// local image was built and never pushed, so the daemon holds exactly one -// platform of it and the subject names none: the platform is read back from the -// image rather than asserted by the caller. -// -// A local subject carries the image ID instead, because the daemon resolves -// whatever reference it is handed to that ID: a pin sitting only in the -// reference is not the identity such a scan binds evidence to. -func Subjects(ctx context.Context, service, reference string, local bool) ([]*builderv0.ImageSubject, error) { - if !local { - return sbom.ExpectedFromImageReference(sbom.PublishedImage{ - Service: service, - Role: Role, - Reference: reference, - Platforms: Platforms, - }) - } - id, err := localImageID(ctx, reference) - if err != nil { - return nil, err - } - return []*builderv0.ImageSubject{{ - Reference: reference, - Digest: id, - Role: Role, - Service: service, - Source: builderv0.ImageSourceKind_IMAGE_SOURCE_KIND_DOCKER_DAEMON, - }}, nil -} - -// localImageID reports the identity the Docker daemon holds an image under. -func localImageID(ctx context.Context, reference string) (string, error) { - command := exec.CommandContext(ctx, "docker", "image", "inspect", reference, "--format", "{{.Id}}") - var stderr strings.Builder - command.Stderr = &stderr - out, err := command.Output() - if err != nil { - // An absent image, an absent daemon and an absent docker are different - // things to go fix, and only docker's own message tells them apart. - return "", fmt.Errorf("resolve local image %s: %w: %s", reference, err, strings.TrimSpace(stderr.String())) - } - id := strings.TrimSpace(string(out)) - if !strings.HasPrefix(id, "sha256:") { - return "", fmt.Errorf("local image %s reports id %q, which is not a sha256 digest", reference, id) - } - return id, nil -} - -// Document is one platform's inventory, written to disk and named by the digest -// it is bound to. -type Document struct { - Reference string - Digest string - Platform string - Path string - SHA256 string -} - -// Collect inventories every subject through the shared scanner and writes one -// CycloneDX document each. The document carries the scanned digest in its own -// root component, so an exported artifact still names the image it describes -// rather than relying on the filename to say so. -// -// A single failed scan fails the whole call: publishing evidence for some -// platforms while silently omitting others is the false coverage claim this -// evidence exists to prevent. -func Collect(ctx context.Context, dir string, subjects []*builderv0.ImageSubject) ([]Document, error) { - if len(subjects) == 0 { - return nil, fmt.Errorf("no image subjects to inventory") - } - if err := os.MkdirAll(dir, 0o755); err != nil { - return nil, fmt.Errorf("create evidence directory: %w", err) - } - documents := make([]Document, 0, len(subjects)) - for _, subject := range subjects { - result, err := sbom.Image(ctx, sbom.ImageRequest{ - Reference: subject.GetReference(), - Platform: subject.GetPlatform(), - Source: sbom.SourceOf(subject), - }) - if err != nil { - return nil, fmt.Errorf("inventory %s: %w", describe(subject), err) - } - encoded, err := sbom.MarshalCycloneDXJSON(result.Bom) - if err != nil { - return nil, fmt.Errorf("encode %s: %w", describe(subject), err) - } - path := filepath.Join(dir, documentName(subject.GetReference(), result.Platform, result.Digest)) - if err := os.WriteFile(path, encoded, 0o644); err != nil { - return nil, fmt.Errorf("write %s: %w", path, err) - } - documents = append(documents, Document{ - Reference: subject.GetReference(), - Digest: result.Digest, - Platform: result.Platform, - Path: path, - SHA256: result.SHA256, - }) - } - return documents, nil -} - -// WriteIndex records which image and platform each document covers, so a -// release asset is readable without opening every document. -func WriteIndex(dir string, documents []Document) error { - var index strings.Builder - for _, document := range documents { - fmt.Fprintf(&index, "%s %s@%s %s %s\n", - document.Platform, referenceName(document.Reference), document.Digest, - filepath.Base(document.Path), document.SHA256) - } - return os.WriteFile(filepath.Join(dir, "index.txt"), []byte(index.String()), 0o644) -} - -// documentName names a document after the repository it inventories, the -// platform and the digest it is bound to, none of which is typed here. -func documentName(reference, platform, digest string) string { - repository := referenceName(reference) - return fmt.Sprintf("%s-%s-%s.cdx.json", - repository[strings.LastIndex(repository, "/")+1:], - strings.ReplaceAll(platform, "/", "-"), - strings.TrimPrefix(digest, "sha256:")) -} - -func describe(subject *builderv0.ImageSubject) string { - if platform := subject.GetPlatform(); platform != "" { - return subject.GetReference() + " on " + platform - } - return subject.GetReference() -} - -// referenceName strips any tag or digest so the index names the repository and -// the digest actually scanned, rather than the tag that was asked for. -func referenceName(reference string) string { - if base, _, found := strings.Cut(reference, "@"); found { - reference = base - } - if slash := strings.LastIndex(reference, "/"); slash >= 0 { - if colon := strings.LastIndex(reference[slash:], ":"); colon >= 0 { - return reference[:slash+colon] - } - return reference - } - if colon := strings.LastIndex(reference, ":"); colon >= 0 { - return reference[:colon] - } - return reference -} diff --git a/internal/imageevidence/imageevidence_test.go b/internal/imageevidence/imageevidence_test.go deleted file mode 100644 index c1f191f..0000000 --- a/internal/imageevidence/imageevidence_test.go +++ /dev/null @@ -1,84 +0,0 @@ -package imageevidence - -import ( - "context" - "os" - "path/filepath" - "testing" - - "github.com/codefly-dev/core/agents/services/sbom" - "github.com/stretchr/testify/require" -) - -const pinned = "ghcr.io/codefly-dev/service-warehouse@sha256:1205da0743e56b68c12c754438476e8fdd0f05cdb08ce1c9687ddb8d8f0e1669" - -func TestSubjectsAreOnePerShippedPlatformAndPinned(t *testing.T) { - subjects, err := Subjects(context.Background(), "codefly.dev/warehouse", pinned, false) - require.NoError(t, err) - - var platforms []string - for _, subject := range subjects { - platforms = append(platforms, subject.GetPlatform()) - require.Equal(t, pinned, subject.GetReference()) - require.Equal(t, Role, subject.GetRole()) - require.Equal(t, "codefly.dev/warehouse", subject.GetService()) - require.Equal(t, sbom.SourceRegistry, sbom.SourceOf(subject)) - require.NoError(t, sbom.RequirePinned(subject)) - } - require.ElementsMatch(t, Platforms, platforms) -} - -// An evidence request that names no service would match evidence belonging to -// any service, so core refuses it; this package does not paper over that. -func TestSubjectsRefuseAnUnnamedServiceAndAFloatingReference(t *testing.T) { - _, err := Subjects(context.Background(), "", pinned, false) - require.Error(t, err) - _, err = Subjects(context.Background(), "codefly.dev/warehouse", "ghcr.io/codefly-dev/service-warehouse:latest", false) - require.Error(t, err) -} - -// A local subject carries the image ID the daemon holds, so an image the daemon -// does not hold can only be reported, never turned into a subject pinned to -// nothing. The resolved case needs a real image and is asserted by the e2e -// inventory. -func TestALocalSubjectNeedsAnImageTheDaemonHolds(t *testing.T) { - const absent = "swh-imageevidence-unit-test-absent:none" - subjects, err := Subjects(context.Background(), "codefly.dev/warehouse", absent, true) - require.Error(t, err, "got %d subjects", len(subjects)) - require.Contains(t, err.Error(), absent) -} - -func TestCollectRefusesToReportCoverageOfNothing(t *testing.T) { - _, err := Collect(context.Background(), t.TempDir(), nil) - require.Error(t, err) -} - -func TestDocumentsAreNamedAfterTheImageThePlatformAndTheDigest(t *testing.T) { - require.Equal(t, "service-warehouse-linux-arm64-1205da.cdx.json", - documentName(pinned, "linux/arm64", "sha256:1205da")) - require.Equal(t, "service-warehouse-linux-amd64-abc.cdx.json", - documentName("service-warehouse:e2e", "linux/amd64", "sha256:abc")) -} - -func TestReferenceNameDropsTheTagAndTheDigest(t *testing.T) { - for reference, want := range map[string]string{ - pinned: "ghcr.io/codefly-dev/service-warehouse", - "ghcr.io/codefly-dev/service-warehouse:0.0.1": "ghcr.io/codefly-dev/service-warehouse", - "localhost:5000/service-warehouse:e2e": "localhost:5000/service-warehouse", - "service-warehouse": "service-warehouse", - "service-warehouse:e2e": "service-warehouse", - } { - require.Equal(t, want, referenceName(reference), reference) - } -} - -func TestTheIndexNamesEveryDocumentWithItsPlatformAndDigest(t *testing.T) { - dir := t.TempDir() - require.NoError(t, WriteIndex(dir, []Document{{ - Reference: pinned, Digest: "sha256:aa", Platform: "linux/amd64", - Path: filepath.Join(dir, "a.cdx.json"), SHA256: "bb", - }})) - index, err := os.ReadFile(filepath.Join(dir, "index.txt")) - require.NoError(t, err) - require.Equal(t, "linux/amd64 ghcr.io/codefly-dev/service-warehouse@sha256:aa a.cdx.json bb\n", string(index)) -} diff --git a/readiness_test.go b/readiness_test.go index b7918e6..2d6c557 100644 --- a/readiness_test.go +++ b/readiness_test.go @@ -6,6 +6,7 @@ import ( "testing" "time" + "github.com/codefly-dev/core/callertoken" basev0 "github.com/codefly-dev/core/generated/go/codefly/base/v0" "github.com/codefly-dev/core/resources" "github.com/stretchr/testify/require" @@ -16,7 +17,6 @@ import ( healthv1 "google.golang.org/grpc/health/grpc_health_v1" whv0 "github.com/codefly-dev/service-warehouse/gen/codefly/warehouse/v0" - "github.com/codefly-dev/service-warehouse/internal/auth" "github.com/codefly-dev/service-warehouse/internal/backend" "github.com/codefly-dev/service-warehouse/internal/backend/mem" "github.com/codefly-dev/service-warehouse/internal/server" @@ -33,8 +33,8 @@ func startGateway(t *testing.T, token string) (string, *grpchealth.Server) { require.NoError(t, err) health := grpchealth.NewServer() srv := grpc.NewServer( - grpc.ChainUnaryInterceptor(auth.UnaryInterceptor(token)), - grpc.ChainStreamInterceptor(auth.StreamInterceptor(token)), + grpc.ChainUnaryInterceptor(callertoken.UnaryServerInterceptor(token)), + grpc.ChainStreamInterceptor(callertoken.StreamServerInterceptor(token)), ) healthv1.RegisterHealthServer(srv, health) whv0.RegisterWarehouseServer(srv, server.New(mem.New(backend.Config{}))) @@ -96,7 +96,7 @@ func TestWaitForReadyWaitsForTheGatewayToServe(t *testing.T) { func TestProbeGatewayDistinguishesNotYetFromNever(t *testing.T) { address, health := startGateway(t, "server-token") - conn, err := grpc.NewClient(address, grpc.WithTransportCredentials(insecureCredentials()), auth.DialOption("server-token")) + conn, err := grpc.NewClient(address, grpc.WithTransportCredentials(insecureCredentials()), callertoken.DialOption("server-token")) require.NoError(t, err) defer conn.Close() healthClient, warehouse := healthv1.NewHealthClient(conn), whv0.NewWarehouseClient(conn) diff --git a/runtime.go b/runtime.go index c101d47..7078c89 100644 --- a/runtime.go +++ b/runtime.go @@ -2,8 +2,6 @@ package main import ( "context" - "crypto/rand" - "encoding/hex" "fmt" "os" "time" @@ -18,12 +16,12 @@ import ( runtimev0 "github.com/codefly-dev/core/generated/go/codefly/services/runtime/v0" "github.com/codefly-dev/core/agents/services" + "github.com/codefly-dev/core/callertoken" "github.com/codefly-dev/core/resources" dockerrun "github.com/codefly-dev/core/runners/dockerrun" "github.com/codefly-dev/core/wool" whv0 "github.com/codefly-dev/service-warehouse/gen/codefly/warehouse/v0" - "github.com/codefly-dev/service-warehouse/internal/auth" ) // gatewayContainerPort is the port the gateway listens on inside its container. @@ -157,7 +155,7 @@ func (s *Runtime) resolveGatewayToken() (string, error) { if s.conf.authToken != "" { return s.conf.authToken, nil } - return randomSecret() + return callertoken.Generate() } // startGateway runs the gateway container, mapping the assigned host port onto @@ -261,7 +259,7 @@ func (s *Runtime) WaitForReady(ctx context.Context) error { conn, err := grpc.NewClient(address, grpc.WithTransportCredentials(insecure.NewCredentials()), - auth.DialOption(s.gatewayToken)) + callertoken.DialOption(s.gatewayToken)) if err != nil { return s.Wool.Wrapf(err, "cannot create gateway client for %s", address) } @@ -344,13 +342,3 @@ func (s *Runtime) Destroy(ctx context.Context, req *runtimev0.DestroyRequest) (* func (s *Runtime) Test(ctx context.Context, req *runtimev0.TestRequest) (*runtimev0.TestResponse, error) { return s.Runtime.TestResponse() } - -// randomSecret returns an unguessable hex secret, so the published port is never -// reachable with a well-known credential. -func randomSecret() (string, error) { - b := make([]byte, 24) - if _, err := rand.Read(b); err != nil { - return "", err - } - return hex.EncodeToString(b), nil -} diff --git a/runtime_e2e_test.go b/runtime_e2e_test.go index 03fd028..8effe8c 100644 --- a/runtime_e2e_test.go +++ b/runtime_e2e_test.go @@ -28,6 +28,7 @@ import ( "testing" "time" + "github.com/codefly-dev/core/callertoken" basev0 "github.com/codefly-dev/core/generated/go/codefly/base/v0" builderv0 "github.com/codefly-dev/core/generated/go/codefly/services/builder/v0" runtimev0 "github.com/codefly-dev/core/generated/go/codefly/services/runtime/v0" @@ -41,7 +42,6 @@ import ( "google.golang.org/grpc/status" whv0 "github.com/codefly-dev/service-warehouse/gen/codefly/warehouse/v0" - "github.com/codefly-dev/service-warehouse/internal/auth" ) func TestRuntimeEndToEnd(t *testing.T) { @@ -114,7 +114,7 @@ func TestRuntimeEndToEnd(t *testing.T) { conn, err := grpc.NewClient(endpoint, grpc.WithTransportCredentials(insecure.NewCredentials()), - auth.DialOption(token)) + callertoken.DialOption(token)) require.NoError(t, err) defer conn.Close() capabilities, err := whv0.NewWarehouseClient(conn).Capabilities(ctx, &whv0.CapabilitiesRequest{}) @@ -153,7 +153,7 @@ func requireUnauthenticatedPeerIsRefused(t *testing.T, endpoint string) { t.Helper() peers := map[string][]grpc.DialOption{ "no credentials": nil, - "wrong token": {auth.DialOption("0000000000000000000000000000000000000000000000000000000000000000")}, + "wrong token": {callertoken.DialOption("0000000000000000000000000000000000000000000000000000000000000000")}, } for peerName, extra := range peers { conn, err := grpc.NewClient(endpoint, diff --git a/sbom_e2e_test.go b/sbom_e2e_test.go index d6801b5..e0079f7 100644 --- a/sbom_e2e_test.go +++ b/sbom_e2e_test.go @@ -43,7 +43,10 @@ func TestImageSBOMInventoriesTheGatewayImage(t *testing.T) { require.Len(t, subjects, 1, "the single platform the daemon holds") require.Equal(t, sbom.SourceDockerDaemon, sbom.SourceOf(subjects[0])) require.Equal(t, override, subjects[0].GetReference()) - require.Empty(t, subjects[0].GetPlatform(), "the platform is read from the image rather than assumed") + require.NotEmpty(t, subjects[0].GetPlatform(), "core reads the platform from the daemon image") + id, err := sbom.LocalImageID(context.Background(), override) + require.NoError(t, err) + require.Equal(t, id, subjects[0].GetDigest()) require.NoError(t, sbom.RequirePinned(subjects[0])) resp, err := builder.SBOM(context.Background(), &builderv0.SBOMRequest{Scope: builderv0.SBOMScope_SBOM_SCOPE_IMAGE}) @@ -58,7 +61,7 @@ func TestImageSBOMInventoriesTheGatewayImage(t *testing.T) { // resolved to any other identity describes an image nobody asked for. require.Equal(t, subjects[0].GetDigest(), evidence.GetDigest()) require.True(t, strings.HasPrefix(evidence.GetDigest(), "sha256:")) - require.NotEmpty(t, evidence.GetPlatform(), "evidence names no platform") + require.Equal(t, subjects[0].GetPlatform(), evidence.GetPlatform()) require.NotEmpty(t, evidence.GetSha256(), "evidence carries no document checksum") var goModules, osPackages int diff --git a/sbom_test.go b/sbom_test.go index 3624efb..c697179 100644 --- a/sbom_test.go +++ b/sbom_test.go @@ -15,10 +15,11 @@ import ( basev0 "github.com/codefly-dev/core/generated/go/codefly/base/v0" agentv0 "github.com/codefly-dev/core/generated/go/codefly/services/agent/v0" builderv0 "github.com/codefly-dev/core/generated/go/codefly/services/builder/v0" + "github.com/codefly-dev/core/resources" "github.com/stretchr/testify/require" "gopkg.in/yaml.v3" - "github.com/codefly-dev/service-warehouse/internal/imageevidence" + "github.com/codefly-dev/service-warehouse/internal/gatewayimage" ) // testService is the identity newSBOMBuilder loads, and therefore the identity @@ -43,20 +44,20 @@ func newSBOMBuilder(t *testing.T) *Builder { } func TestImageSubjectsCoverEveryPublishedPlatform(t *testing.T) { - subjects, err := imageevidence.Subjects(context.Background(), testService, pinnedReference, false) + subjects, err := sbom.ImageSubjects(context.Background(), sbom.PublishedImage{Service: testService, Role: gatewayimage.Role, Reference: pinnedReference, Platforms: gatewayimage.Platforms}, false) require.NoError(t, err) - require.Len(t, subjects, len(imageevidence.Platforms), "one subject per shipped platform") + require.Len(t, subjects, len(gatewayimage.Platforms), "one subject per shipped platform") covered := map[string]bool{} for _, subject := range subjects { require.Equal(t, sbom.SourceRegistry, sbom.SourceOf(subject)) require.NoError(t, sbom.RequirePinned(subject)) require.Equal(t, pinnedReference, subject.GetReference()) - require.Equal(t, imageevidence.Role, subject.GetRole()) + require.Equal(t, gatewayimage.Role, subject.GetRole()) require.Equal(t, testService, subject.GetService()) covered[subject.GetPlatform()] = true } - for _, platform := range imageevidence.Platforms { + for _, platform := range gatewayimage.Platforms { require.True(t, covered[platform], "no subject covers shipped platform %s", platform) } } @@ -70,7 +71,7 @@ func TestImageSubjectsRefuseAReferenceThatIsNotPinned(t *testing.T) { registry + "/service-warehouse:latest", registry + "/service-warehouse:0.0.1", } { - _, err := imageevidence.Subjects(context.Background(), testService, reference, false) + _, err := sbom.ImageSubjects(context.Background(), sbom.PublishedImage{Service: testService, Role: gatewayimage.Role, Reference: reference, Platforms: gatewayimage.Platforms}, false) require.Error(t, err, reference) } } @@ -84,13 +85,14 @@ func TestBuilderEnumeratesThePublishedImageOrRefuses(t *testing.T) { subjects, err := builder.imageSubjects(context.Background()) if _, lockErr := publishedGatewayImage(); lockErr != nil { - require.ErrorIs(t, err, lockErr) + require.ErrorIs(t, lockErr, resources.ErrImageUnpublished) + require.ErrorIs(t, err, resources.ErrImageUnpublished) return } require.NoError(t, err) published, err := publishedGatewayImage() require.NoError(t, err) - require.Len(t, subjects, len(imageevidence.Platforms)) + require.Len(t, subjects, len(gatewayimage.Platforms)) for _, subject := range subjects { require.Equal(t, published.FullName(), subject.GetReference()) require.Equal(t, testService, subject.GetService()) @@ -98,11 +100,11 @@ func TestBuilderEnumeratesThePublishedImageOrRefuses(t *testing.T) { } func TestImageEvidenceSatisfiesTheCoverageContract(t *testing.T) { - subjects, err := imageevidence.Subjects(context.Background(), testService, pinnedReference, false) + subjects, err := sbom.ImageSubjects(context.Background(), sbom.PublishedImage{Service: testService, Role: gatewayimage.Role, Reference: pinnedReference, Platforms: gatewayimage.Platforms}, false) require.NoError(t, err) // The expectation is built from the platforms the release pipeline actually - // publishes, never from imageevidence: deriving both sides from the code + // publishes, never from gatewayimage: deriving both sides from the code // under test compares it against itself, and would validate however many // platforms it dropped. expected := expectedFromReleasePipeline(t, pinnedReference) @@ -156,7 +158,7 @@ func TestImageScopeReportsFailureRatherThanCoverage(t *testing.T) { func TestSourceScopeKeepsTheExistingInventory(t *testing.T) { t.Setenv(gatewayImageOverrideEnv, "") - subjects, err := imageevidence.Subjects(context.Background(), testService, pinnedReference, false) + subjects, err := sbom.ImageSubjects(context.Background(), sbom.PublishedImage{Service: testService, Role: gatewayimage.Role, Reference: pinnedReference, Platforms: gatewayimage.Platforms}, false) require.NoError(t, err) // A cancelled context stops the scan before it reaches the network, which is @@ -188,7 +190,7 @@ func TestImageSBOMIsServedButNotAdvertised(t *testing.T) { // And it is served: an explicit subject is scanned rather than answered as // unsupported, so the failure here is the scanner's, not a missing // implementation. - subject := &builderv0.ImageSubject{Reference: "not-pinned", Service: testService, Role: imageevidence.Role} + subject := &builderv0.ImageSubject{Reference: "not-pinned", Service: testService, Role: gatewayimage.Role} resp, err := newSBOMBuilder(t).SBOM(context.Background(), &builderv0.SBOMRequest{ Scope: builderv0.SBOMScope_SBOM_SCOPE_IMAGE, Subjects: []*builderv0.ImageSubject{subject}, @@ -200,7 +202,7 @@ func TestImageSBOMIsServedButNotAdvertised(t *testing.T) { func TestShippedPlatformsMatchTheReleasePipeline(t *testing.T) { published := append([]string(nil), publishedPlatforms(t)...) - declared := append([]string(nil), imageevidence.Platforms...) + declared := append([]string(nil), gatewayimage.Platforms...) // Compare as sets: the pipeline and the code have to name the same // platforms, but neither owns the order the other writes them in. sort.Strings(published) @@ -233,43 +235,37 @@ func expectedFromReleasePipeline(t *testing.T, reference string) []*builderv0.Im expected = append(expected, &builderv0.ImageSubject{ Reference: reference, Platform: platform, - Role: imageevidence.Role, + Role: gatewayimage.Role, Service: testService, }) } return expected } -// publishedPlatforms reads the platforms the pipeline actually builds, from the -// step that pushes the image rather than from any step that happens to carry a -// platform list. +// publishedPlatforms reads the explicit inputs supplied to core's publisher. func publishedPlatforms(t *testing.T) []string { t.Helper() var workflow struct { Jobs map[string]struct { - Steps []struct { - Uses string `yaml:"uses"` - With struct { - Platforms string `yaml:"platforms"` - } `yaml:"with"` - } `yaml:"steps"` + Uses string `yaml:"uses"` + With struct { + Platforms string `yaml:"platforms"` + } `yaml:"with"` } `yaml:"jobs"` } require.NoError(t, yaml.Unmarshal([]byte(workflowBody(t, "publish-gateway-image.yml")), &workflow)) var published []string for _, job := range workflow.Jobs { - for _, step := range job.Steps { - if !strings.Contains(step.Uses, "docker/build-push-action") { - continue - } - for _, platform := range strings.Split(step.With.Platforms, ",") { - if platform = strings.TrimSpace(platform); platform != "" { - published = append(published, platform) - } + if !strings.HasPrefix(job.Uses, "codefly-dev/core/.github/workflows/publish-service-image.yml@") { + continue + } + for _, platform := range strings.Split(job.With.Platforms, ",") { + if platform = strings.TrimSpace(platform); platform != "" { + published = append(published, platform) } } } - require.NotEmpty(t, published, "the publish workflow's build-push step publishes no platforms") + require.NotEmpty(t, published, "the publisher must declare the platforms evidence covers") return published } From 53d95eb210df1a4a6133c7a7fefd66e21dd7b90b Mon Sep 17 00:00:00 2001 From: Antoine Toussaint <312536060+antoinetoussaint-byte@users.noreply.github.com> Date: Sun, 11 Oct 2026 09:00:58 -0400 Subject: [PATCH 06/10] ci: show gateway end-to-end test results --- .github/workflows/ci.yml | 4 ++-- AGENTS.md | 4 ++-- README.md | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d4d9cee..f87d871 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -35,7 +35,7 @@ jobs: - name: end-to-end run through the agent runtime env: SWH_GATEWAY_IMAGE: service-warehouse:e2e - run: go test -tags e2e -count=1 -timeout 300s -run TestRuntimeEndToEnd . + run: go test -v -tags e2e -count=1 -timeout 300s -run TestRuntimeEndToEnd . # Inventories a real gateway image through the agent's own image-SBOM path and # checks it against the shared coverage contract, so a change that would ship @@ -56,7 +56,7 @@ jobs: - name: assert the image contract against a real image env: SWH_GATEWAY_IMAGE: service-warehouse:e2e - run: go test -tags e2e -count=1 -timeout 300s -run TestImageSBOM . + run: go test -v -tags e2e -count=1 -timeout 300s -run TestImageSBOM . - name: write evidence through the release command run: go run ./cmd/image-sbom -image service-warehouse:e2e -local -out sbom # The local image is single-platform and reached through the daemon. The diff --git a/AGENTS.md b/AGENTS.md index 5cbd4db..a7a7018 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -105,8 +105,8 @@ Two more CI jobs need Docker and are not part of that gate. They build the image the `e2e` build tag: ```sh -SWH_GATEWAY_IMAGE=service-warehouse:e2e go test -tags e2e -count=1 -run TestRuntimeEndToEnd . -SWH_GATEWAY_IMAGE=service-warehouse:e2e go test -tags e2e -count=1 -run TestImageSBOM . # needs syft on PATH +SWH_GATEWAY_IMAGE=service-warehouse:e2e go test -v -tags e2e -count=1 -run TestRuntimeEndToEnd . +SWH_GATEWAY_IMAGE=service-warehouse:e2e go test -v -tags e2e -count=1 -run TestImageSBOM . # needs syft on PATH ``` **A green e2e run is not evidence that anything ran.** With diff --git a/README.md b/README.md index 29bba5d..003b227 100644 --- a/README.md +++ b/README.md @@ -250,7 +250,7 @@ one also needs `syft` on `PATH`: ```bash docker build -t service-warehouse:e2e . -SWH_GATEWAY_IMAGE=service-warehouse:e2e go test -tags e2e -count=1 . +SWH_GATEWAY_IMAGE=service-warehouse:e2e go test -v -tags e2e -count=1 . ``` With `SWH_GATEWAY_IMAGE` unset they skip, and a skipped run still prints `ok`: From 2bc9e68a086c5aaf5271b3bd2f0e30a640d77dd3 Mon Sep 17 00:00:00 2001 From: Antoine Toussaint <312536060+antoinetoussaint-byte@users.noreply.github.com> Date: Sun, 11 Oct 2026 09:06:28 -0400 Subject: [PATCH 07/10] test(agent): verify published platform wiring through the builder --- .github/workflows/agent-ci.yml | 13 ++++--------- AGENTS.md | 9 ++++----- sbom_test.go | 4 +++- 3 files changed, 11 insertions(+), 15 deletions(-) diff --git a/.github/workflows/agent-ci.yml b/.github/workflows/agent-ci.yml index 727f826..ce93a7f 100644 --- a/.github/workflows/agent-ci.yml +++ b/.github/workflows/agent-ci.yml @@ -2,15 +2,10 @@ # and generated-service conformance gates against this agent, through the shared # reusable workflow every codefly service agent uses. # -# Manual (workflow_dispatch) until a run of it has been seen green on this -# repository. service-object-storage parks the same workflow because the CLI's -# pinned go source-packager predated Go 1.27, and the release/build stage failed -# with `go.mod requires go >= 1.27.0 (GOTOOLCHAIN=local)` -# (codefly-dev/cli#499, since closed). Run locally with a CLI whose packager -# still carries Go 1.26.4, `codefly agent ci` passes its manifest and source -# stages here and fails that same build stage; the CLI release named below has -# not been run against this repository. Switch the trigger to `pull_request` once the first -# dispatch passes. +# Manual until a full run has been seen green on this repository. Packaging +# needs Docker and source-packager/toolchain downloads; source validation alone +# does not prove the release and generated-service stages. Switch to +# pull_request once the first full run passes. name: agent-ci on: diff --git a/AGENTS.md b/AGENTS.md index a7a7018..a3ec32c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -115,11 +115,10 @@ SWH_GATEWAY_IMAGE=service-warehouse:e2e go test -v -tags e2e -count=1 -run TestI output for `--- PASS`, never `ok` alone. `codefly agent ci` is the full agent conformance gate. Its workflow -(`agent-ci.yml`) is `workflow_dispatch` only until a run of it has been seen -green here. Run locally from the repo root, it needs a CLI whose go -source-packager carries Go 1.27: with one that carries 1.26.4 the manifest and -source stages pass and the build stage fails on `go.mod requires go >= 1.27.0`, -which is the CLI's gap, not this repo's. +(`agent-ci.yml`) is `workflow_dispatch` only until a full run has been seen green +here. Run it from the repo root; packaging needs Docker and access to the source +packager's toolchain downloads. `codefly ci run` instead operates on a consuming +workspace: this agent repository has no workspace manifest. Run the server locally against the zero-dependency backend — it needs no database, no warehouse credential, and no network. It still refuses an diff --git a/sbom_test.go b/sbom_test.go index c697179..d6b4e83 100644 --- a/sbom_test.go +++ b/sbom_test.go @@ -44,7 +44,9 @@ func newSBOMBuilder(t *testing.T) *Builder { } func TestImageSubjectsCoverEveryPublishedPlatform(t *testing.T) { - subjects, err := sbom.ImageSubjects(context.Background(), sbom.PublishedImage{Service: testService, Role: gatewayimage.Role, Reference: pinnedReference, Platforms: gatewayimage.Platforms}, false) + withPublishedLock(t) + t.Setenv(gatewayImageOverrideEnv, "") + subjects, err := newSBOMBuilder(t).imageSubjects(context.Background()) require.NoError(t, err) require.Len(t, subjects, len(gatewayimage.Platforms), "one subject per shipped platform") From fd2dab805d0c4fb3df7604ade6f06c750eb128e3 Mon Sep 17 00:00:00 2001 From: Antoine Toussaint <312536060+antoinetoussaint-byte@users.noreply.github.com> Date: Sun, 11 Oct 2026 10:13:02 -0400 Subject: [PATCH 08/10] fix(agent): address gateway readiness, deployment auth and release review --- .github/scripts/allow-unpublished-image.sh | 22 +++ .github/workflows/agent-ci.yml | 4 +- .github/workflows/ci.yml | 25 ++-- .github/workflows/release.yml | 20 +++ AGENTS.md | 24 +++- Dockerfile | 3 +- README.md | 132 +++++++++++++----- agent.codefly.yaml | 2 +- builder.go | 22 +-- composition_test.go | 68 ++++++++- go.mod | 2 +- main.go | 6 +- release_gate_test.go | 59 ++++++++ runtime.go | 15 +- runtime_e2e_test.go | 35 ++++- sbom_e2e_test.go | 3 + templates/agent/README.md.tmpl | 23 +-- .../kustomize/base/deployment.yaml.tmpl | 18 ++- .../kustomize/base/kustomization.yaml.tmpl | 1 + .../kustomize/base/networkpolicy.yaml.tmpl | 17 +++ templates/factory/GETTING_STARTED.md.tmpl | 13 +- .../configurations/local/warehouse.secret.env | 2 + 22 files changed, 412 insertions(+), 104 deletions(-) create mode 100644 .github/scripts/allow-unpublished-image.sh create mode 100644 release_gate_test.go create mode 100644 templates/deployment/kustomize/base/networkpolicy.yaml.tmpl diff --git a/.github/scripts/allow-unpublished-image.sh b/.github/scripts/allow-unpublished-image.sh new file mode 100644 index 0000000..c161f82 --- /dev/null +++ b/.github/scripts/allow-unpublished-image.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +# CI's sole empty-lock exception: a full pre-release history with no prior lock. +set -euo pipefail +if [ "$(git rev-parse --is-shallow-repository)" != false ]; then + echo "cannot allow an unpublished image with incomplete git history" >&2 + exit 1 +fi +if [ -n "$(git tag --list 'v*')" ]; then + echo "gateway-image.json cannot be empty after a release tag exists" >&2 + exit 1 +fi +# Follow every fetched branch and both parents of merges. Deleting a digest +# before the first tag must not reset the pre-release exception. +for commit in $(git rev-list --all -- gateway-image.json); do + if git cat-file -e "$commit:gateway-image.json" 2>/dev/null; then + digest=$(git show "$commit:gateway-image.json" | jq -er '.digest | strings') + if [ -n "$digest" ]; then + echo "gateway-image.json cannot be empty: a lock already exists in history at $commit" >&2 + exit 1 + fi + fi +done diff --git a/.github/workflows/agent-ci.yml b/.github/workflows/agent-ci.yml index ce93a7f..5513fe2 100644 --- a/.github/workflows/agent-ci.yml +++ b/.github/workflows/agent-ci.yml @@ -17,6 +17,6 @@ jobs: # v0.18.1, the release go.mod requires. Bump the two together. uses: codefly-dev/core/.github/workflows/agent-ci.yml@ea49e7a968ef5f2fd325b1c41f823a47eb20141c with: - # The CLI release the gate runs with. The go source-packager it installs - # is the one the CLI pins, so bumping this is how that pin moves. + # The CLI release the gate runs with. agent.codefly.yaml selects the + # source-packager version (0.0.69); go.mod selects Go 1.27.2. codefly-cli-version: "0.1.181" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f87d871..adcd336 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,14 +5,19 @@ on: branches: [main] pull_request: +env: + GOTOOLCHAIN: go1.27.2 + jobs: build-test: runs-on: ubuntu-latest steps: - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + fetch-depth: 0 - uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5 with: - go-version: "1.27.0" + go-version: "1.27.2" cache: true - run: go build ./... - run: go vet ./... @@ -26,9 +31,11 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + fetch-depth: 0 - uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5 with: - go-version: "1.27.0" + go-version: "1.27.2" cache: true - name: build gateway image run: docker build -t service-warehouse:e2e . @@ -44,9 +51,11 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + fetch-depth: 0 - uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5 with: - go-version: "1.27.0" + go-version: "1.27.2" cache: true - name: build gateway image run: docker build -t service-warehouse:e2e . @@ -66,16 +75,16 @@ jobs: # digest gateway-image.json records — the image the agent actually names — # so a lock pointing at something that does not resolve, or that ships # fewer platforms than evidence has to cover, fails here rather than at - # release time. A lock with no digest is the tree before its first image - # was published: there is nothing to inventory, release.yml refuses to tag - # it, and the agent refuses to run it, so this is the one place it is - # allowed to pass, and says so. + # release time. The empty-lock exception ends at the first nonempty lock + # in fetched history or any v* tag. Full history is required above, so + # deleting an established digest cannot quietly skip the published scan. - name: inventory the published multi-architecture image run: | set -euo pipefail digest=$(jq -er '.digest' gateway-image.json) if [ -z "$digest" ]; then - echo "::notice::gateway-image.json records no digest: no gateway image has been published yet, so there is no published image to inventory" + bash .github/scripts/allow-unpublished-image.sh + echo "::notice::pre-release tree with no prior image lock: no published image to inventory" exit 0 fi go run ./cmd/image-sbom -image "$(jq -er '.name + "@" + .digest' gateway-image.json)" -out sbom-published diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 848f9d5..f921fe5 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -159,3 +159,23 @@ jobs: # reachable from the default branch. uses: codefly-dev/core/.github/workflows/go-service-release.yml@ea49e7a968ef5f2fd325b1c41f823a47eb20141c secrets: inherit + + # Workflow artifacts expire. Copy the digest-bound documents and index onto + # the release after GoReleaser has created it; retries replace the same assets. + attach-image-evidence: + needs: [gateway-image, release] + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + pattern: gateway-image-sbom-* + merge-multiple: true + path: sbom + - name: Attach image SBOMs and index to the GitHub release + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + set -euo pipefail + gh release upload "$GITHUB_REF_NAME" sbom/* --repo "$GITHUB_REPOSITORY" --clobber diff --git a/AGENTS.md b/AGENTS.md index a3ec32c..253f958 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -31,6 +31,11 @@ this file needs a fix in the same PR. starts the gateway, hands out the one token it enforces, and renders the manifest. +Deployments require an environment-managed `SWH_AUTH_TOKEN` secret reference; +consumers bind the same key to `warehouse.token`. The templates render a +same-namespace ingress NetworkPolicy. Anonymous access is only an explicit local +gateway opt-in. README owns the secret binding procedure and policy limits. + Three rules follow from that: 1. **The API is the honest intersection.** An operation goes into the uniform @@ -91,7 +96,7 @@ preferences; each is a rule an agent broke at real cost. Derived from [`.github/workflows/ci.yml`](.github/workflows/ci.yml): the `build-test` job is those three commands, so a green local run of all three is the definition of done. CI pins the Go version as a literal in that workflow -(`go-version: "1.27.0"`) rather than reading `go.mod`, so bumping `go.mod` +(`go-version: "1.27.2"`) rather than reading `go.mod`, so bumping `go.mod` alone moves your toolchain and not CI's — change both. ```sh @@ -116,9 +121,15 @@ output for `--- PASS`, never `ok` alone. `codefly agent ci` is the full agent conformance gate. Its workflow (`agent-ci.yml`) is `workflow_dispatch` only until a full run has been seen green -here. Run it from the repo root; packaging needs Docker and access to the source -packager's toolchain downloads. `codefly ci run` instead operates on a consuming -workspace: this agent repository has no workspace manifest. +here. Run `GOTOOLCHAIN=go1.27.2 codefly agent ci` from the repo root. The manifest +pins source packager `codefly.dev/go:0.0.69`, whose image recipes use immutable +digests. The gateway build pins Go 1.27.2 by tag and image-index digest and fixes +`GOTOOLCHAIN`; keep `go.mod`, CI and the image toolchain in sync. A fresh service +declares `backend: mem`; Runtime.Test probes health and authenticated Capabilities +even when Start was not called. Before the first published lock, full conformance +still fails closed on the absent image; a passing binary audit is not full +conformance. `codefly ci run` operates on a consuming workspace: this agent +repository has no workspace manifest. Run the server locally against the zero-dependency backend — it needs no database, no warehouse credential, and no network. It still refuses an @@ -148,8 +159,9 @@ it — you changed your toolchain, not the contract. A `v*` tag publishes the gateway image's tags first and the agent binary second, from one workflow, in that order. The image itself is published *before* the tag, because the agent pins its digest and a digest cannot be recorded in a -commit the tag was already cut from; `README.md` ("Releasing") has the four -steps. `gateway-image.json` records no digest until the first image exists, and +commit the tag was already cut from; `README.md` ("Releasing") has the nine +first-release steps. The manifest seed is 0.0.1; publish bumps the first release +to v0.0.2, so dispatch the image publisher with 0.0.2. `gateway-image.json` records no digest until the first image exists, and every consumer of it — the agent, `Deploy`, the release — refuses an empty one rather than run `:latest`. The tag must match `version:` in `agent.codefly.yaml` or the job fails by design. diff --git a/Dockerfile b/Dockerfile index 84825ec..4819382 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,7 +9,8 @@ # driver lands and then stop. Both stages name Debian 13 explicitly: the binary # links against the build stage's glibc and needs one at least as new in the # image it runs in, which "golang:1.27" moving to the next Debian would break. -FROM golang:1.27-trixie AS build +FROM golang:1.27.2-trixie@sha256:e58d6f83b3416618d8bcac2b3dde1b7f7e3c4a77d25e88637f8bbae81536c48d AS build +ENV GOTOOLCHAIN=go1.27.2 WORKDIR /src COPY go.mod go.sum ./ RUN go mod download diff --git a/README.md b/README.md index 003b227..5a15e44 100644 --- a/README.md +++ b/README.md @@ -102,8 +102,8 @@ The server holds the warehouse credentials and runs SQL against the database it is bound to, and the listen address says nothing about who can reach it. So it **refuses to start on an unauthenticated listener** unless the operator says otherwise: `SWH_AUTH_TOKEN` must be set, or `SWH_ALLOW_ANONYMOUS` must be `true` -(for a listener confined to a private boundary enforced elsewhere, such as a -cluster NetworkPolicy or service-mesh mTLS). +as an explicit local-development opt-in. The generated deployment requires +authentication and never enables anonymous access. With `SWH_AUTH_TOKEN` set, every RPC — unary and streaming, `Capabilities` included — must carry the secret as the `x-codefly-token` gRPC metadata header, @@ -140,24 +140,52 @@ port, waits until the gateway reports itself serving *and* accepts the agent's token, and hands the endpoint and token to consumers. That wait attests the process and the credential, not the warehouse: the gateway does not probe its backend and no RPC reports that. Set `backend:` in `service.codefly.yaml` to -choose the engine; left unset, the gateway's own default applies, which today is -`duckdb` and refuses to start, so a local run presently needs `backend: mem` -(the catalog-only backend). The agent delivers `SWH_BACKEND`, `SWH_DATABASE`, +choose the engine. Newly generated services declare `backend: mem` (the +catalog-only backend, with no query execution). Removing that setting falls back +to the gateway's unsupported `duckdb` default and startup fails. The agent's +`Test` step performs the same health and authenticated `Capabilities` check as +`Start`, so conformance's Init → Test path rejects a dead gateway. The agent delivers `SWH_BACKEND`, `SWH_DATABASE`, `SWH_DATASET` and `SWH_LOCATION`, and refuses `SWH_CREDENTIALS_FILE`, which nothing could project into the container. `SWH_GATEWAY_IMAGE` makes the agent run a locally built image instead of the pinned one (`docker build -t service-warehouse:local .`); it configures the agent, not the gateway. -**Deployed** the agent renders a Deployment, Service and its own ServiceAccount -(so a cloud warehouse can grant exactly this workload through a workload -identity), with gRPC health probes on the gateway's overall service. The -deployed listener is `SWH_ALLOW_ANONYMOUS=true`: caller identity is enforced by -the cluster (NetworkPolicy and service-mesh mTLS), because nothing delivers a -secret to the consumers' `warehouse` configuration for a deployment. A configured -`SWH_AUTH_TOKEN` or `SWH_CREDENTIALS_FILE` is therefore refused rather than -rendered, and so is a value the manifest cannot carry. The rest of the `SWH_*` -surface reaches the gateway by the environment's service configuration being -bound onto the container by name, after the render. +**Deployed** the agent renders a Deployment, Service, dedicated ServiceAccount +and NetworkPolicy, with gRPC health probes on the gateway's overall service. +The listener requires a token. Supply `SWH_AUTH_TOKEN` through the deployment's +Kubernetes `secret_references`, referencing a secret provisioned by the +environment's secret manager; plaintext tokens in deployment requests are +refused. Both ephemeral and restricted output profiles use `secretKeyRef`: + +```yaml +secret_references: + SWH_AUTH_TOKEN: + name: warehouse-auth + key: token +``` + +Provision a random 64-character hex token once per environment and bind **the +same secret key** to each consumer's `warehouse.token` secret configuration. +For a consumer that uses Codefly's configuration carriers, its secret-reference +map names `CODEFLY__SERVICE_SECRET_CONFIGURATION______WAREHOUSE__TOKEN` +(the warehouse producer's module and service, uppercased) and points to that +same `warehouse-auth` / `token` key. No token is embedded in an endpoint or +manifest. The agent does not create or rotate deployed secrets, or automatically +copy secret references between services: the environment owns both bindings. +An absent secret prevents pod startup; an absent or wrong consumer token is +rejected before any SQL reaches the backend. + +The NetworkPolicy allows TCP ingress to the gateway port from pods in its own +namespace only. Enforcement requires a CNI that implements NetworkPolicy. This +is a namespace boundary, not a declared-consumer allowlist; same-namespace pods +still need the token. Cross-namespace consumers require an explicit policy +overlay. No service mesh or mTLS is installed by this agent. + +`SWH_ALLOW_ANONYMOUS=true` is refused in deployment configuration. A configured +`SWH_CREDENTIALS_FILE` is also refused because nothing mounts that file: grant +the dedicated ServiceAccount a workload identity, or supply a mount and variable +through an external overlay. Other `SWH_*` values are bound onto the container +by name from the environment's service configuration after the render. ## Distribution and SBOM evidence @@ -181,9 +209,11 @@ against core's `sbom.ValidateCoverage`. - [`cmd/image-sbom`](cmd/image-sbom/main.go) is a thin entry point to core's `CollectImageEvidence` and `WriteImageEvidenceIndex`. A release runs it against the digest [`gateway-image.json`](gateway-image.json) records and - attaches them as the `gateway-image-sbom-` workflow artifact, before - the tags consumers resolve are created, so an image whose scan failed is never - pullable by the version it was built for. + saves them as the `gateway-image-sbom-` workflow artifact before + the image tags are created, then attaches every CycloneDX document and the + index to the GitHub release. Release assets retain the evidence after workflow + artifacts expire; `attach-image-evidence` must finish green before release + verification is complete. - CI builds the image and checks the same contract against it (`image-sbom`), and runs the agent against it (`agent-runtime-e2e`). - `ValidationCapabilities.image_sbom` is **served but not advertised**: core @@ -208,29 +238,48 @@ image is built. ### Releasing -The digest is only known once the image is built, so the order is fixed, and -`release.yml` refuses to tag a tree that skips a step: - -1. Find the upcoming version with `codefly publish --dry-run`. -2. Run the `publish-gateway-image` workflow with that version (no `v`). It - calls core's pinned `publish-service-image.yml`, pushes the image by digest, - carrying no tag, and prints `gateway-image.json`. The publisher uses - `persist-credentials: false` and refuses a commit outside the default branch's - history. -3. Commit that file in a reviewed change. Until it is committed the lock records - no digest, and the agent, the deployment and the release all refuse to name an - image. -4. `codefly publish` tags the release. The tag workflow tests, checks that the - tag matches `agent.codefly.yaml`, that the recorded digest was built for this - version, inventories it, and only then creates the `:` and `:latest` - tags and the GitHub release carrying the agent. - -The release call needs a `GH_PAT` secret in the repository (`secrets: inherit`). +The coordinator performs the following first-release sequence. The manifest's +seed is `0.0.1`; `codefly publish` increments it, so the **first release is +`v0.0.2`** and the image publisher must receive **`0.0.2`**. + +1. Merge the reviewed gateway PR after its CI is green. +2. On clean, synced `main`, run `codefly publish --dry-run` and confirm + `0.0.1` → `0.0.2`. +3. Dispatch `publish-gateway-image` on `main` with `version=0.0.2` (no `v`). + Core's pinned publisher pushes by digest and prints `gateway-image.json`. + It uses `persist-credentials: false` and refuses a commit outside the default + branch's history. +4. Make the new `ghcr.io/codefly-dev/service-warehouse` package **public** in + GitHub package settings. Verify an anonymous pull by the returned digest + using a fresh Docker configuration with no registry login. +5. Commit the returned lock in a reviewed PR. Let CI inventory both declared + platforms (`linux/amd64`, `linux/arm64`), then merge that PR. +6. Run the full `codefly agent ci` on synced `main`, including its audit and + generated-service `codefly ci run`. After it has passed, enable the PR trigger + in `agent-ci.yml`. +7. Confirm the repository's `GH_PAT` release secret is configured, and retain + source packager `codefly.dev/go:0.0.69` and the patched Go 1.27.2 toolchain. +8. From clean, synced `main` with Docker available, run `codefly publish`. + Verify that the tag matches `agent.codefly.yaml`, the locked image's version + labels match `0.0.2`, image tags resolve to the locked digest, and the workflow + attaches the image SBOM documents and index alongside the agent archives. +9. Verify the release and the production loader's archive resolution. Close + [#12](https://github.com/codefly-dev/service-warehouse/issues/12) only then, + with the release URL and the per-row answer below. + +**Per-row error answer at this revision:** `RowError` contains `row_index` and +free-form `string error`; it has **no permanent/retryable code**. Consumers cannot +branch on retryability without parsing text. The separate API work in +[#14](https://github.com/codefly-dev/service-warehouse/pull/14) must land before +claiming otherwise. This agent PR references #12; it does not close it. The lock is parsed by core's `resources.ParseImageLock`: it must name `ghcr.io/codefly-dev/service-warehouse`, contain only `name` and `digest`, and use `sha256:` followed by 64 lowercase hex characters. An empty digest records -an unpublished image and is refused by the agent and release. +an unpublished image and is refused by the agent and release. CI permits the +initial empty lock only with complete history, no `v*` tag, and no prior nonempty +lock in fetched history. Clearing an established lock fails CI even before the +first tag. [sbom]: https://github.com/codefly-dev/core/blob/v0.18.1/docs/sbom.md @@ -245,6 +294,15 @@ go vet ./... go test -race ./... # unit tests (mem backend + bufconn server) and the agent's ``` +The release-gate regression tests use `git` and `jq` (also present in CI). + +The source packager is pinned to `codefly.dev/go:0.0.69`, whose image recipes use +immutable digests. `go.mod` and CI require Go 1.27.2; the gateway Dockerfile pins +the Go 1.27.2 image index and sets `GOTOOLCHAIN=go1.27.2`. For local conformance, +run `GOTOOLCHAIN=go1.27.2 codefly agent ci` so host toolchain selection is explicit. +Before the first image lock, the agent-binary audit can pass but full generated +service conformance still fails closed on the unpublished image. + The agent's end-to-end tests need Docker and are behind the `e2e` tag; the SBOM one also needs `syft` on `PATH`: diff --git a/agent.codefly.yaml b/agent.codefly.yaml index 49b4ab1..5545fe6 100644 --- a/agent.codefly.yaml +++ b/agent.codefly.yaml @@ -4,7 +4,7 @@ name: warehouse version: 0.0.1 source: directory: . - agent: codefly.dev/go:0.0.56 + agent: codefly.dev/go:0.0.69 release: owner: workflow workflow: release.yml diff --git a/builder.go b/builder.go index 3718871..20582e1 100644 --- a/builder.go +++ b/builder.go @@ -186,16 +186,17 @@ func (s *Builder) Deploy(ctx context.Context, req *builderv0.DeploymentRequest) "or mount the file from an overlay of your own and patch SWH_CREDENTIALS_FILE onto the container there", s.conf.credentialsFile)) } - // A configured gateway token cannot be honored by this deployment: the - // manifest carries no secret values, and CreateConnectionConfiguration only - // emits a token the Runtime generated, so consumers would receive none. - // Accepting it would render a manifest that either crash-loops the gateway or, - // worse, starts a gateway enforcing a credential every consumer lacks. Reject - // it where the manifest is owned. + // Deployment credentials are provided by the environment's secret store, + // never embedded in manifests. Consumers bind the same key to warehouse.token. if s.conf.authToken != "" { - return s.Builder.DeployError(fmt.Errorf("SWH_AUTH_TOKEN cannot be delivered by this deployment: " + - "the emitted Secret carries no configuration values and consumers would receive no credential; " + - "caller identity in the deployed profile is enforced by the cluster (NetworkPolicy and service-mesh mTLS)")) + return s.Builder.DeployError(fmt.Errorf("SWH_AUTH_TOKEN must be delivered as a Kubernetes secret reference, not a literal configuration value")) + } + ref := req.GetDeployment().GetKubernetes().GetSecretReferences()["SWH_AUTH_TOKEN"] + if ref.GetName() == "" || ref.GetKey() == "" { + return s.Builder.DeployError(fmt.Errorf("deployment requires an SWH_AUTH_TOKEN Kubernetes secret reference; bind the same secret key to consumers' warehouse.token configuration")) + } + if value, _ := resources.GetConfigurationValue(ctx, req.GetConfiguration(), configurationGroup, "SWH_ALLOW_ANONYMOUS"); value != "" && value != "false" { + return s.Builder.DeployError(fmt.Errorf("SWH_ALLOW_ANONYMOUS is only an explicit local gateway opt-in; deployments require authentication")) } return s.Builder.DeployKustomize(ctx, req, services.KustomizeDeployment{ EnvironmentVariables: s.EnvironmentVariables, @@ -253,6 +254,9 @@ func workloadPodOverlay() *services.PodTemplateOverlay { func (s *Builder) Create(ctx context.Context, req *builderv0.CreateRequest) (*builderv0.CreateResponse, error) { defer s.Wool.Catch() + if s.Settings.Backend == "" { + s.Settings.Backend = "mem" + } if err := s.Templates(ctx, s.Information, services.WithFactory(factoryFS)); err != nil { return s.Builder.CreateError(err) } diff --git a/composition_test.go b/composition_test.go index 23b55f9..dc6689d 100644 --- a/composition_test.go +++ b/composition_test.go @@ -213,11 +213,7 @@ func TestDeploymentTemplates(t *testing.T) { require.Equal(t, want, env[key]) } require.Equal(t, ":9465", env["SWH_LISTEN"]) - // The gateway refuses an unauthenticated listener unless the manifest - // says so, so a deployment that stops emitting this opt-out without - // also delivering SWH_AUTH_TOKEN would crash-loop. - require.Equal(t, "true", env["SWH_ALLOW_ANONYMOUS"], "the deployed profile must declare its unauthenticated listener") - require.NotContains(t, env, "SWH_AUTH_TOKEN", "the manifest must not carry the gateway credential") + require.Equal(t, "false", env["SWH_ALLOW_ANONYMOUS"]) // No rendering names a credentials path: nothing in this manifest // mounts one, so an emitted path would point at a file the pod does // not have. Deploy rejects a configured path instead. Matched on the @@ -312,6 +308,9 @@ func deployRequest(destination string, values map[string]string, profile builder Namespace: "codefly", Destination: destination, Profile: profile, + SecretReferences: map[string]*builderv0.KubernetesSecretKeyReference{ + "SWH_AUTH_TOKEN": {Name: "warehouse-auth", Key: "token"}, + }, }, }}, Configuration: configuration(values), @@ -380,11 +379,12 @@ func TestDeployRefusesWhatItHasNoWayToDeliver(t *testing.T) { want string }{ "a credentials file nothing mounts": {map[string]string{"SWH_CREDENTIALS_FILE": "/keys/sa.json"}, "SWH_CREDENTIALS_FILE cannot be delivered"}, - "a token the consumers would lack": {map[string]string{"SWH_AUTH_TOKEN": "pinned"}, "SWH_AUTH_TOKEN cannot be delivered"}, + "a literal token": {map[string]string{"SWH_AUTH_TOKEN": "pinned"}, "SWH_AUTH_TOKEN must be delivered as a Kubernetes secret reference"}, "a value that would write the manifest": { map[string]string{"SWH_DATASET": "x\"\n - name: SWH_AUTH_TOKEN"}, "SWH_DATASET", }, - "a templating action": {map[string]string{"SWH_DATABASE": "{{ .Image }}"}, "SWH_DATABASE"}, + "anonymous deployment": {map[string]string{"SWH_ALLOW_ANONYMOUS": "true"}, "only an explicit local"}, + "a templating action": {map[string]string{"SWH_DATABASE": "{{ .Image }}"}, "SWH_DATABASE"}, } { t.Run(name, func(t *testing.T) { builder := newDeployBuilder(t, ctx) @@ -544,3 +544,57 @@ func TestCreateEndpointsDeclaresAnInternalGRPCEndpoint(t *testing.T) { require.Equal(t, resources.VisibilityInternal, endpoint.GetVisibility()) require.True(t, strings.HasPrefix(endpoint.GetService(), "warehouse"), endpoint.GetService()) } + +// Both output profiles use the same external secret, so a portable rendering +// never needs a plaintext credential. Consumers reference that key as well. +func TestDeployRequiresAuthenticationAndRendersNetworkBoundary(t *testing.T) { + for _, profile := range []builderv0.KubernetesOutputProfile{ephemeral, restricted} { + t.Run(profile.String(), func(t *testing.T) { + ctx := context.Background() + builder := newDeployBuilder(t, ctx) + destination := t.TempDir() + req := deployRequest(destination, nil, profile) + req.GetDeployment().GetKubernetes().SecretReferences = nil + resp, err := builder.Deploy(ctx, req) + require.NoError(t, err) + require.Equal(t, builderv0.DeploymentStatus_ERROR, resp.GetState().GetState()) + require.Contains(t, resp.GetState().GetMessage(), "requires an SWH_AUTH_TOKEN Kubernetes secret reference") + entries, err := os.ReadDir(destination) + require.NoError(t, err) + require.Empty(t, entries) + + req = deployRequest(destination, nil, profile) + resp, err = builder.Deploy(ctx, req) + require.NoError(t, err) + require.Equal(t, builderv0.DeploymentStatus_SUCCESS, resp.GetState().GetState(), resp.GetState().GetMessage()) + var deployment map[string]any + require.NoError(t, yaml.Unmarshal([]byte(renderedDeployment(t, destination)), &deployment)) + container := deployment["spec"].(map[string]any)["template"].(map[string]any)["spec"].(map[string]any)["containers"].([]any)[0].(map[string]any) + found := false + for _, raw := range container["env"].([]any) { + env := raw.(map[string]any) + if env["name"] == "SWH_AUTH_TOKEN" { + found = true + require.NotContains(t, env, "value") + require.Equal(t, map[string]any{"secretKeyRef": map[string]any{"name": "warehouse-auth", "key": "token"}}, env["valueFrom"]) + } + } + require.True(t, found) + body, err := os.ReadFile(filepath.Join(destination, "base", "networkpolicy.yaml")) + require.NoError(t, err) + var policy map[string]any + require.NoError(t, yaml.Unmarshal(body, &policy)) + require.Equal(t, "NetworkPolicy", policy["kind"]) + spec := policy["spec"].(map[string]any) + require.Equal(t, map[string]any{"matchLabels": map[string]any{"app": "warehouse"}}, spec["podSelector"]) + require.Equal(t, []any{"Ingress"}, spec["policyTypes"]) + ingress := spec["ingress"].([]any) + require.Len(t, ingress, 1) + require.Equal(t, []any{map[string]any{"podSelector": map[string]any{}}}, ingress[0].(map[string]any)["from"]) + require.Equal(t, []any{map[string]any{"protocol": "TCP", "port": gatewayContainerPort}}, ingress[0].(map[string]any)["ports"]) + kustomization, err := os.ReadFile(filepath.Join(destination, "base", "kustomization.yaml")) + require.NoError(t, err) + require.Contains(t, string(kustomization), "networkpolicy.yaml") + }) + } +} diff --git a/go.mod b/go.mod index 8b6cc9a..4faa5c2 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/codefly-dev/service-warehouse -go 1.27.0 +go 1.27.2 require ( github.com/codefly-dev/core v0.18.1 diff --git a/main.go b/main.go index 99e9b6c..72258d5 100644 --- a/main.go +++ b/main.go @@ -48,9 +48,9 @@ const configurationGroup = "warehouse" // runtime configuration (an environment's values) takes precedence. Credentials // are never settings: they arrive as runtime configuration, never here. type Settings struct { - // Backend selects the warehouse. Empty leaves the choice to the gateway, - // whose default is the embedded engine. A deployed environment names a cloud - // warehouse. + // Backend selects the warehouse. Create defaults new services to mem, the + // catalog-only backend. An explicitly empty setting uses the gateway's + // default, currently the unsupported DuckDB skeleton. Backend string `yaml:"backend"` // Database is the database the gateway is bound to (a BigQuery project, a // Snowflake or Redshift database, an embedded engine's file). diff --git a/release_gate_test.go b/release_gate_test.go new file mode 100644 index 0000000..8ed82fe --- /dev/null +++ b/release_gate_test.go @@ -0,0 +1,59 @@ +package main + +import ( + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestUnpublishedImageExceptionEndsAtFirstLockOrRelease(t *testing.T) { + script, err := filepath.Abs(".github/scripts/allow-unpublished-image.sh") + require.NoError(t, err) + for _, scenario := range []string{"unpublished", "previous lock", "release tag", "shallow history"} { + t.Run(scenario, func(t *testing.T) { + dir := t.TempDir() + git := func(args ...string) string { + t.Helper() + command := exec.Command("git", args...) + command.Dir = dir + output, err := command.CombinedOutput() + require.NoError(t, err, "%s", output) + return string(output) + } + git("init", "--quiet") + git("config", "commit.gpgsign", "false") + git("config", "tag.gpgsign", "false") + git("config", "core.hooksPath", "/dev/null") + git("config", "user.name", "gate fixture") + git("config", "user.email", "fixture@example.invalid") + commit := func(digest string) { + t.Helper() + require.NoError(t, os.WriteFile(filepath.Join(dir, "gateway-image.json"), []byte(`{"name":"registry.example/warehouse","digest":"`+digest+`"}`), 0600)) + git("add", "gateway-image.json") + git("commit", "--quiet", "-m", "fixture") + } + commit("") + switch scenario { + case "previous lock": + commit("sha256:" + strings.Repeat("a", 64)) + commit("") + case "release tag": + git("tag", "v0.0.2") // This tag exists only in the disposable test repository. + case "shallow history": + require.NoError(t, os.WriteFile(filepath.Join(dir, ".git", "shallow"), []byte(git("rev-parse", "HEAD")), 0600)) + } + command := exec.Command("bash", script) + command.Dir = dir + output, err := command.CombinedOutput() + if scenario == "unpublished" { + require.NoError(t, err, "%s", output) + } else { + require.Error(t, err, "%s", output) + } + }) + } +} diff --git a/runtime.go b/runtime.go index 7078c89..653c77d 100644 --- a/runtime.go +++ b/runtime.go @@ -36,7 +36,7 @@ const gatewayContainerPort = 9465 // is run. const gatewayContainerUser = "65532:65532" -// readinessBudget bounds the whole wait for the gateway in Start: one overall +// readinessBudget bounds the whole wait for the gateway in Start and Test: one overall // deadline, so a slow attempt spends the budget instead of extending it. const readinessBudget = 60 * time.Second @@ -276,10 +276,7 @@ func (s *Runtime) WaitForReady(ctx context.Context) error { lastErr := "no attempt completed" poll: for { - // Stop rather than start a probe the budget cannot see through: a - // truncated attempt reports this deadline and buries the cause the last - // full attempt established. - if deadline, ok := budget.Deadline(); ok && time.Until(deadline) < readinessProbeTimeout { + if budget.Err() != nil { break } @@ -340,5 +337,11 @@ func (s *Runtime) Destroy(ctx context.Context, req *runtimev0.DestroyRequest) (* } func (s *Runtime) Test(ctx context.Context, req *runtimev0.TestRequest) (*runtimev0.TestResponse, error) { - return s.Runtime.TestResponse() + defer s.Wool.Catch() + // Conformance calls Init then Test without Start. Creating a container is + // not proof that it stayed alive or accepts the credential we handed out. + if err := s.WaitForReady(s.Wool.Inject(ctx)); err != nil { + return s.Runtime.TestError(err) + } + return s.Runtime.TestResponseWithResults(1, 1, 0, 0, 0, nil, nil) } diff --git a/runtime_e2e_test.go b/runtime_e2e_test.go index 8effe8c..1fdd573 100644 --- a/runtime_e2e_test.go +++ b/runtime_e2e_test.go @@ -65,13 +65,20 @@ func TestRuntimeEndToEnd(t *testing.T) { init, err := rt.Init(ctx, &runtimev0.InitRequest{ RuntimeContext: runtimeContext, ProposedNetworkMappings: networkMappings, - Configuration: configuration(map[string]string{"SWH_BACKEND": "mem"}), }) require.NoError(t, err) // Init reports a lifecycle failure in its status, not as a returned error: // the contract hands the CLI a diagnostic rather than a transport fault. require.Equal(t, runtimev0.InitStatus_READY, init.GetStatus().GetState(), init.GetStatus().GetMessage()) + // This is the conformance lifecycle: Create → Load → Init → Test, no Start + // and no backend override. It must exercise the generated service defaults. + require.Equal(t, "mem", rt.Settings.Backend) + tested, err := rt.Test(ctx, &runtimev0.TestRequest{}) + require.NoError(t, err) + require.Equal(t, runtimev0.TestStatus_SUCCESS, tested.GetStatus().GetState(), tested.GetStatus().GetMessage()) + require.EqualValues(t, 1, tested.GetTestsPassed()) + start, err := rt.Start(ctx, &runtimev0.StartRequest{}) require.NoError(t, err) require.Equal(t, runtimev0.StartStatus_STARTED, start.GetStatus().GetState(), start.GetStatus().GetMessage()) @@ -246,3 +253,29 @@ func requirePublishedOnAllInterfaces(t *testing.T, containerID string, container require.NotContains(t, hostIPs, "127.0.0.1", "container port %d must not be published loopback-only", containerPort) require.Contains(t, hostIPs, "0.0.0.0", "container port %d must publish on all interfaces", containerPort) } + +// A gateway can die after Init returns. Test must reject that dead gateway +// even though container creation succeeded and Start was never called. +func TestRuntimeEndToEndRejectsDeadGateway(t *testing.T) { + if os.Getenv(gatewayImageOverrideEnv) == "" { + t.Skipf("set %s to a locally built gateway image", gatewayImageOverrideEnv) + } + ctx := context.Background() + rt, mappings, runtimeContext := loadedRuntime(t, ctx) + t.Cleanup(func() { _, _ = rt.Destroy(context.Background(), &runtimev0.DestroyRequest{}) }) + init, err := rt.Init(ctx, &runtimev0.InitRequest{ + RuntimeContext: runtimeContext, ProposedNetworkMappings: mappings, + }) + require.NoError(t, err) + require.Equal(t, runtimev0.InitStatus_READY, init.GetStatus().GetState(), init.GetStatus().GetMessage()) + id, err := rt.gatewayEnv.ContainerID() + require.NoError(t, err) + require.NoError(t, exec.Command("docker", "stop", id).Run()) + probe, cancel := context.WithTimeout(ctx, 3*time.Second) + defer cancel() + result, err := rt.Test(probe, &runtimev0.TestRequest{}) + require.NoError(t, err) + require.Equal(t, runtimev0.TestStatus_ERROR, result.GetStatus().GetState()) + require.Contains(t, result.GetStatus().GetMessage(), "not ready") + require.NotContains(t, result.GetStatus().GetMessage(), "no attempt completed") +} diff --git a/sbom_e2e_test.go b/sbom_e2e_test.go index e0079f7..8cd8503 100644 --- a/sbom_e2e_test.go +++ b/sbom_e2e_test.go @@ -65,14 +65,17 @@ func TestImageSBOMInventoriesTheGatewayImage(t *testing.T) { require.NotEmpty(t, evidence.GetSha256(), "evidence carries no document checksum") var goModules, osPackages int + var libc bool for _, component := range evidence.GetBom().GetComponents() { switch purl := component.GetPurl(); { case strings.HasPrefix(purl, "pkg:golang/"): goModules++ case isOSPackage(purl): osPackages++ + libc = libc || strings.HasPrefix(purl, "pkg:deb/debian/libc6@") } } + require.True(t, libc, "inventory must contain Debian libc6: the gateway dynamically links glibc") require.NotZero(t, goModules, "inventory found no Go modules: the gateway's application dependencies are missing") require.NotZero(t, osPackages, "inventory found no OS packages: an image inventory is not satisfied by application dependencies alone") t.Logf("inventoried %s on %s (%d Go modules, %d OS packages)", evidence.GetDigest(), evidence.GetPlatform(), goModules, osPackages) diff --git a/templates/agent/README.md.tmpl b/templates/agent/README.md.tmpl index bb0a943..77a7fa8 100644 --- a/templates/agent/README.md.tmpl +++ b/templates/agent/README.md.tmpl @@ -1,12 +1,17 @@ # warehouse -The warehouse gateway: a codefly/warehouse/v0 gRPC service in front of a SQL -warehouse. The backend is selected by configuration: the embedded engine -locally, a cloud warehouse when deployed. +The warehouse gateway is a codefly/warehouse/v0 gRPC service in front of a SQL +warehouse. A generated service declares `backend: mem`, the catalog-only backend +that needs no warehouse credentials. Set `backend` in service.codefly.yaml to +choose another engine; removing it falls back to the unsupported DuckDB skeleton. +See https://github.com/codefly-dev/service-warehouse for current backend support. -A local run starts the published gateway image with a fresh shared secret and -hands it to consumers as the secret `token` value of the `warehouse` -configuration group. Set `backend` in service.codefly.yaml to choose the engine; -left unset, the gateway's own default applies. See -https://github.com/codefly-dev/service-warehouse for the backends that serve -queries today. +A local run starts the gateway image with a fresh shared secret and hands it to +consumers as the secret `token` value of the `warehouse` configuration group. +Both Start and Test wait for health and authenticated Capabilities. + +Deployment requires an environment-managed Kubernetes secret reference for +SWH_AUTH_TOKEN. Bind the same secret key to each consumer's warehouse.token +configuration. The agent renders a same-namespace ingress NetworkPolicy and a +dedicated ServiceAccount; a policy-capable CNI is required. See the repository +README for both secret bindings and workload-identity setup. diff --git a/templates/deployment/kustomize/base/deployment.yaml.tmpl b/templates/deployment/kustomize/base/deployment.yaml.tmpl index 3bf299b..35607dc 100644 --- a/templates/deployment/kustomize/base/deployment.yaml.tmpl +++ b/templates/deployment/kustomize/base/deployment.yaml.tmpl @@ -62,17 +62,15 @@ spec: - name: SWH_LOCATION value: "{{ .Deployment.Parameters.Location }}" {{- end }} - # The gateway refuses to start on an unauthenticated listener unless - # this is set. In the deployed profile caller identity is enforced by - # the cluster — NetworkPolicy plus service-mesh mTLS in front of this - # ClusterIP — not by the gateway. - # - # Do not replace this with SWH_AUTH_TOKEN on its own. The gateway - # would then enforce a credential its consumers do not have, because - # nothing delivers that secret to their warehouse configuration. Both - # halves have to land together; see README "Authentication". - name: SWH_ALLOW_ANONYMOUS - value: "true" + value: "false" +{{- range $key, $ref := .Deployment.SecretReferences }} + - name: {{ $key }} + valueFrom: + secretKeyRef: + name: {{ $ref.Name }} + key: {{ $ref.Key }} +{{- end }} {{- /* No SWH_CREDENTIALS_FILE: nothing here mounts a credentials file, so a path would name a file the pod does not have. A deployed cloud warehouse authenticates through a workload identity bound to the ServiceAccount below, or through a file diff --git a/templates/deployment/kustomize/base/kustomization.yaml.tmpl b/templates/deployment/kustomize/base/kustomization.yaml.tmpl index d94a00e..fab5a9c 100644 --- a/templates/deployment/kustomize/base/kustomization.yaml.tmpl +++ b/templates/deployment/kustomize/base/kustomization.yaml.tmpl @@ -7,3 +7,4 @@ resources: {{- end }} - deployment.yaml - service.yaml + - networkpolicy.yaml diff --git a/templates/deployment/kustomize/base/networkpolicy.yaml.tmpl b/templates/deployment/kustomize/base/networkpolicy.yaml.tmpl new file mode 100644 index 0000000..98adccc --- /dev/null +++ b/templates/deployment/kustomize/base/networkpolicy.yaml.tmpl @@ -0,0 +1,17 @@ +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{.Name}} + namespace: {{.Namespace}} +spec: + podSelector: + matchLabels: + app: {{.Name}} + policyTypes: + - Ingress + ingress: + - from: + - podSelector: {} + ports: + - protocol: TCP + port: {{ .Deployment.Parameters.ContainerPort }} diff --git a/templates/factory/GETTING_STARTED.md.tmpl b/templates/factory/GETTING_STARTED.md.tmpl index c192dd1..03486ad 100644 --- a/templates/factory/GETTING_STARTED.md.tmpl +++ b/templates/factory/GETTING_STARTED.md.tmpl @@ -1,5 +1,12 @@ # Getting started -This service runs the warehouse gateway. Locally it needs no credential: the -agent generates the shared secret the gateway enforces and delivers it to -consumers. +This service runs the warehouse gateway with `backend: mem`: the catalog-only +backend needs no warehouse credentials and does not execute queries. The agent +generates the caller token locally and delivers it to consumers. Its test step +checks that the gateway is serving and accepts that token. + +Deployment requires an environment-managed Kubernetes secret reference for +`SWH_AUTH_TOKEN`. Bind the same secret key to every consumer's `warehouse.token` +configuration. The rendered NetworkPolicy allows only same-namespace ingress; +a policy-capable CNI is required. Anonymous access is only an explicit opt-in +when running the gateway directly for local development. diff --git a/templates/factory/configurations/local/warehouse.secret.env b/templates/factory/configurations/local/warehouse.secret.env index f38aff3..75fb6b8 100644 --- a/templates/factory/configurations/local/warehouse.secret.env +++ b/templates/factory/configurations/local/warehouse.secret.env @@ -1,5 +1,7 @@ # A local run needs no secrets. The agent generates the gateway's SWH_AUTH_TOKEN # per run and hands it to consumers as the secret "token" value of the warehouse # configuration group — nothing to set here. +# For deployment, bind SWH_AUTH_TOKEN and each consumer's warehouse.token to +# the same environment-managed Kubernetes secret key (see agent README). # Deployed environments reach a cloud warehouse through a workload identity, not # a credential file. From fc6a6eae65f6bbde08adae9c0c4622fbf0a38494 Mon Sep 17 00:00:00 2001 From: Antoine Toussaint <312536060+antoinetoussaint-byte@users.noreply.github.com> Date: Sun, 11 Oct 2026 12:52:29 -0400 Subject: [PATCH 09/10] fix(release): keep gateway publishing in the service repository --- .github/workflows/publish-gateway-image.yml | 118 +++++++++++-- AGENTS.md | 9 +- README.md | 17 +- cmd/service-warehouse/distribution_test.go | 2 +- gateway_image_test.go | 4 +- publisher_test.go | 179 ++++++++++++++++++++ sbom_test.go | 26 +-- 7 files changed, 324 insertions(+), 31 deletions(-) create mode 100644 publisher_test.go diff --git a/.github/workflows/publish-gateway-image.yml b/.github/workflows/publish-gateway-image.yml index 54f85b9..bd454d5 100644 --- a/.github/workflows/publish-gateway-image.yml +++ b/.github/workflows/publish-gateway-image.yml @@ -1,25 +1,121 @@ name: publish-gateway-image -# Core builds by digest before the release tag; commit the returned lock before -# tagging. Its publisher refuses commits outside the default branch's history -# and checks out with persist-credentials: false. +# STOPGAP pending codefly-dev/core#770 (the composite publisher in core 0.18.2). +# GHCR judges the workflow's defining repository: core's reusable workflow was +# denied "installation not allowed to Write organization package" when pushing +# a service package. Keep this job and its GITHUB_TOKEN in the service repo until +# the shared composite action can run inside it. The first dispatch is 0.0.2. on: workflow_dispatch: inputs: version: - description: Upcoming stable agent version (without v) + description: Upcoming stable agent version (without v; first release 0.0.2) required: true type: string +permissions: + contents: read + jobs: publish: + if: github.event_name == 'workflow_dispatch' + runs-on: ubuntu-latest + timeout-minutes: 30 permissions: contents: read packages: write - uses: codefly-dev/core/.github/workflows/publish-service-image.yml@ea49e7a968ef5f2fd325b1c41f823a47eb20141c # v0.18.1 - with: - version: ${{ inputs.version }} - lock-file: gateway-image.json - platforms: linux/amd64,linux/arm64 - secrets: - registry-token: ${{ secrets.GITHUB_TOKEN }} + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + fetch-depth: 0 + persist-credentials: false + + - name: Require the commit to be on the default branch, or refuse + id: default-branch + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + # Authentication exists for these remote reads only; no persisted + # credential-bearing git config can enter the Docker build context. + git_remote() { + local authorization + authorization=$(printf 'x-access-token:%s' "$GH_TOKEN" | base64 | tr -d '\n') + git -c "http.https://github.com/.extraheader=AUTHORIZATION: basic ${authorization}" "$@" + } + authority=$(git_remote ls-remote --symref origin HEAD \ + | awk '$1 == "ref:" && $3 == "HEAD" { sub("refs/heads/", "", $2); print $2 }') + if [ -z "$authority" ]; then + echo "::error::cannot read this repository's default branch from origin; refusing to publish." >&2 + exit 1 + fi + git_remote fetch --quiet --no-tags --force origin \ + "+refs/heads/${authority}:refs/remotes/origin/${authority}" + if ! git merge-base --is-ancestor "$GITHUB_SHA" "refs/remotes/origin/${authority}"; then + echo "::error::${GITHUB_REF_NAME} points at ${GITHUB_SHA}, which is not reachable from refs/remotes/origin/${authority}; refusing to publish." >&2 + exit 1 + fi + + - name: Validate the version and derive the image name + id: inputs + env: + REPOSITORY: ${{ github.repository }} + VERSION: ${{ inputs.version }} + run: | + set -euo pipefail + if [[ ! "$VERSION" =~ ^(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)$ ]]; then + echo "version must be a stable semantic version without a v prefix" >&2 + exit 1 + fi + image=$(printf 'ghcr.io/%s' "$REPOSITORY" | tr '[:upper:]' '[:lower:]') + { + echo "image=$image" + echo "version=$VERSION" + } >> "$GITHUB_OUTPUT" + + # The gateway uses cgo; cross-platform compilation needs an emulator for + # the non-native build stage rather than just a GOARCH override. + - uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 # v3 + - uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1 + - uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0 + id: build + with: + context: . + platforms: linux/amd64,linux/arm64 + labels: | + org.opencontainers.image.version=${{ steps.inputs.outputs.version }} + org.opencontainers.image.source=https://github.com/${{ github.repository }} + org.opencontainers.image.revision=${{ github.sha }} + outputs: type=image,name=${{ steps.inputs.outputs.image }},push-by-digest=true,name-canonical=true,push=true + + - name: Record the digest lock + id: lock + env: + IMAGE: ${{ steps.inputs.outputs.image }} + DIGEST: ${{ steps.build.outputs.digest }} + run: | + set -euo pipefail + if [[ ! "$DIGEST" =~ ^sha256:[0-9a-f]{64}$ ]]; then + echo "build returned no valid lowercase sha256 image-index digest" >&2 + exit 1 + fi + mkdir -p release-evidence + jq -n --arg name "$IMAGE" --arg digest "$DIGEST" '{name: $name, digest: $digest}' > release-evidence/gateway-image.json + { + echo 'Commit the artifact as gateway-image.json in a reviewed PR before publishing the agent:' + echo + echo '```json' + cat release-evidence/gateway-image.json + echo '```' + } >> "$GITHUB_STEP_SUMMARY" + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: gateway-image-lock-${{ steps.inputs.outputs.version }} + path: release-evidence/gateway-image.json + if-no-files-found: error diff --git a/AGENTS.md b/AGENTS.md index 253f958..956cd08 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -167,9 +167,12 @@ rather than run `:latest`. The tag must match `version:` in `agent.codefly.yaml` or the job fails by design. Core is pinned in two places that move together: the `codefly-dev/core` version -in `go.mod` and the commit SHA of core's reusable workflows in `release.yml`, -`agent-ci.yml` and `publish-gateway-image.yml` (never `@main`: `release.yml` -passes `secrets: inherit`). +in `go.mod` and the commit SHA of core's reusable workflows in `release.yml` and +`agent-ci.yml` (never `@main`: `release.yml` passes `secrets: inherit`). The image +publisher is an explicitly labelled stopgap pending +[core#770](https://github.com/codefly-dev/core/issues/770), the core 0.18.2 composite +action: its job must stay in this repository so GHCR accepts its own job token. +Do not restore the reusable publisher call; README owns the transition procedure. ## Where the depth is diff --git a/README.md b/README.md index 5a15e44..c25c385 100644 --- a/README.md +++ b/README.md @@ -238,6 +238,16 @@ image is built. ### Releasing +**Publisher stopgap pending [core#770](https://github.com/codefly-dev/core/issues/770):** +GHCR checks the workflow's defining repository. Core's reusable publisher was +denied `installation not allowed to Write organization package` when publishing +a service package. `publish-gateway-image.yml` therefore defines the job here, +logs in with its own `GITHUB_TOKEN` and `packages: write`, refuses unmerged +commits before login, checks out with `persist-credentials: false`, and pushes +by digest with no tags. Replace these local publishing steps with core 0.18.2's +composite action once core#770 is available and adopted; keep the job in this +repository. The other core integrations remain pinned to v0.18.1. + The coordinator performs the following first-release sequence. The manifest's seed is `0.0.1`; `codefly publish` increments it, so the **first release is `v0.0.2`** and the image publisher must receive **`0.0.2`**. @@ -246,9 +256,10 @@ seed is `0.0.1`; `codefly publish` increments it, so the **first release is 2. On clean, synced `main`, run `codefly publish --dry-run` and confirm `0.0.1` → `0.0.2`. 3. Dispatch `publish-gateway-image` on `main` with `version=0.0.2` (no `v`). - Core's pinned publisher pushes by digest and prints `gateway-image.json`. - It uses `persist-credentials: false` and refuses a commit outside the default - branch's history. + The in-repository publisher pushes by digest and records `gateway-image.json` + in the `gateway-image-lock-0.0.2` artifact and run summary. It uses + `persist-credentials: false` and refuses a commit outside the default branch's + history. 4. Make the new `ghcr.io/codefly-dev/service-warehouse` package **public** in GitHub package settings. Verify an anonymous pull by the returned digest using a fresh Docker configuration with no registry login. diff --git a/cmd/service-warehouse/distribution_test.go b/cmd/service-warehouse/distribution_test.go index ea2d49e..a6b8320 100644 --- a/cmd/service-warehouse/distribution_test.go +++ b/cmd/service-warehouse/distribution_test.go @@ -135,7 +135,7 @@ func contentSignal(path, rel string) (string, error) { // to be re-checked against it. var shippedImage = []string{ ".github/workflows/ci.yml: workflow runs docker build", - ".github/workflows/publish-gateway-image.yml: workflow runs codefly-dev/core/.github/workflows/publish-service-image.yml@", + ".github/workflows/publish-gateway-image.yml: workflow runs docker/build-push-action", ".github/workflows/release.yml: workflow runs docker build", "Dockerfile: container image build definition", "agent.codefly.yaml: Codefly agent manifest, whose Builder ships image build recipes", diff --git a/gateway_image_test.go b/gateway_image_test.go index 9c74a6a..33908f4 100644 --- a/gateway_image_test.go +++ b/gateway_image_test.go @@ -130,8 +130,8 @@ func TestGatewayImageNameIsThisRepository(t *testing.T) { require.Contains(t, string(lock), `"name": "ghcr.io/`+repository+`"`) require.Contains(t, workflowBody(t, "release.yml"), "IMAGE: ghcr.io/${{ github.repository }}") - // The shared publisher derives the image name from the caller repository. - require.NotContains(t, workflowBody(t, "publish-gateway-image.yml"), "image-name:") + // The in-repository publisher derives the image name from this repository. + require.Contains(t, workflowBody(t, "publish-gateway-image.yml"), "REPOSITORY: ${{ github.repository }}") } // The container port is stated once in the agent and handed to the gateway as diff --git a/publisher_test.go b/publisher_test.go new file mode 100644 index 0000000..3081a86 --- /dev/null +++ b/publisher_test.go @@ -0,0 +1,179 @@ +package main + +import ( + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/codefly-dev/core/resources" + "github.com/stretchr/testify/require" + "gopkg.in/yaml.v3" +) + +type publisherStep struct { + ID string `yaml:"id"` + Uses string `yaml:"uses"` + Run string `yaml:"run"` + With map[string]any `yaml:"with"` +} + +type publisherJob struct { + Uses string `yaml:"uses"` + If string `yaml:"if"` + Permissions map[string]string `yaml:"permissions"` + Steps []publisherStep `yaml:"steps"` +} + +func loadPublisher(t *testing.T) publisherJob { + t.Helper() + var workflow struct { + Jobs map[string]publisherJob `yaml:"jobs"` + } + require.NoError(t, yaml.Unmarshal([]byte(workflowBody(t, "publish-gateway-image.yml")), &workflow)) + require.Len(t, workflow.Jobs, 1) + return workflow.Jobs["publish"] +} + +func publisherStepByID(t *testing.T, id string) publisherStep { + t.Helper() + for _, step := range loadPublisher(t).Steps { + if step.ID == id { + return step + } + } + t.Fatalf("publisher has no %s step", id) + return publisherStep{} +} + +// GHCR checks the defining repo. A job-level reusable workflow call broke the +// live object-storage release even with a caller-supplied registry token. +func TestPublisherKeepsRegistryAuthorityInServiceRepository(t *testing.T) { + job := loadPublisher(t) + require.Empty(t, job.Uses, "the publisher job must be defined in this repository") + require.Equal(t, "github.event_name == 'workflow_dispatch'", job.If) + require.Equal(t, "write", job.Permissions["packages"]) + require.Equal(t, "read", job.Permissions["contents"]) + guard, login, build, artifact := -1, -1, -1, -1 + for i, step := range job.Steps { + if strings.HasPrefix(step.Uses, "actions/checkout@") { + require.Equal(t, false, step.With["persist-credentials"]) + } + if step.ID == "default-branch" { + guard = i + } + if strings.HasPrefix(step.Uses, "docker/login-action@") { + login = i + require.Equal(t, "${{ secrets.GITHUB_TOKEN }}", step.With["password"]) + require.Equal(t, "ghcr.io", step.With["registry"]) + } + if strings.HasPrefix(step.Uses, "docker/build-push-action@") { + build = i + require.NotContains(t, step.With, "tags") + require.Contains(t, step.With["outputs"], "push-by-digest=true") + require.Contains(t, step.With["outputs"], "push=true") + require.Contains(t, step.With["labels"], "org.opencontainers.image.version=${{ steps.inputs.outputs.version }}") + } + if strings.HasPrefix(step.Uses, "actions/upload-artifact@") { + artifact = i + require.Equal(t, "release-evidence/gateway-image.json", step.With["path"]) + require.Equal(t, "error", step.With["if-no-files-found"]) + } + } + require.GreaterOrEqual(t, guard, 0) + require.Greater(t, login, guard, "refuse unmerged code before registry login") + require.Greater(t, build, login) + require.Greater(t, artifact, build) +} + +// Execute the workflow's actual guard against a local origin. The remote's +// default branch is deliberately not called main, and the supplied ref name +// lies: authority must come from the remote's HEAD and the commit's ancestry. +func TestPublisherRefusesCommitOutsideDefaultBranch(t *testing.T) { + dir := t.TempDir() + git := func(at string, args ...string) string { + t.Helper() + command := exec.Command("git", args...) + command.Dir = at + out, err := command.CombinedOutput() + require.NoError(t, err, "%s", out) + return strings.TrimSpace(string(out)) + } + git(dir, "init", "--quiet", "--initial-branch=trunk") + git(dir, "config", "commit.gpgsign", "false") + git(dir, "config", "core.hooksPath", "/dev/null") + git(dir, "config", "user.name", "publisher fixture") + git(dir, "config", "user.email", "fixture@example.invalid") + git(dir, "commit", "--quiet", "--allow-empty", "-m", "reviewed") + reviewed := git(dir, "rev-parse", "HEAD") + remote := t.TempDir() + git(remote, "init", "--bare", "--quiet", "--initial-branch=trunk") + git(dir, "remote", "add", "origin", remote) + git(dir, "push", "--quiet", "origin", "trunk") + git(dir, "switch", "--quiet", "-c", "unmerged") + git(dir, "commit", "--quiet", "--allow-empty", "-m", "unreviewed") + unmerged := git(dir, "rev-parse", "HEAD") + + guard := publisherStepByID(t, "default-branch") + run := func(sha string) (string, error) { + command := exec.Command("bash", "-c", guard.Run) + command.Dir = dir + command.Env = append(os.Environ(), "GH_TOKEN=fixture-only", "GITHUB_SHA="+sha, "GITHUB_REF_NAME=trunk") + out, err := command.CombinedOutput() + return string(out), err + } + out, err := run(reviewed) + require.NoError(t, err, "%s", out) + out, err = run(unmerged) + require.Error(t, err, "%s", out) + require.Contains(t, out, "not reachable from refs/remotes/origin/trunk") + git(remote, "symbolic-ref", "HEAD", "refs/heads/missing") + out, err = run(reviewed) + require.Error(t, err, "%s", out) + require.Contains(t, out, "cannot read this repository's default branch") + config, err := os.ReadFile(filepath.Join(dir, ".git", "config")) + require.NoError(t, err) + require.NotContains(t, string(config), "extraheader", "authentication must not persist in the checkout") +} + +func TestPublisherValidatesVersionAndRecordsStrictLock(t *testing.T) { + run := func(step publisherStep, env ...string) (string, error) { + t.Helper() + dir := t.TempDir() + command := exec.Command("bash", "-c", step.Run) + command.Dir = dir + command.Env = append(os.Environ(), append(env, + "GITHUB_OUTPUT="+filepath.Join(dir, "outputs"), + "GITHUB_STEP_SUMMARY="+filepath.Join(dir, "summary"))...) + out, err := command.CombinedOutput() + if err != nil { + return dir, err + } + require.NotContains(t, string(out), "error") + return dir, nil + } + inputs := publisherStepByID(t, "inputs") + dir, err := run(inputs, "REPOSITORY=codefly-dev/service-warehouse", "VERSION=0.0.2") + require.NoError(t, err) + output, err := os.ReadFile(filepath.Join(dir, "outputs")) + require.NoError(t, err) + require.Contains(t, string(output), "image=ghcr.io/codefly-dev/service-warehouse\nversion=0.0.2\n") + for _, version := range []string{"", "v0.0.2", "0.0.2-rc1", "0.0.2\nimage=other"} { + _, err = run(inputs, "REPOSITORY=codefly-dev/service-warehouse", "VERSION="+version) + require.Error(t, err, "version %q must be rejected", version) + } + lock := publisherStepByID(t, "lock") + valid := "sha256:" + strings.Repeat("a", 64) + dir, err = run(lock, "IMAGE=ghcr.io/codefly-dev/service-warehouse", "DIGEST="+valid) + require.NoError(t, err) + content, err := os.ReadFile(filepath.Join(dir, "release-evidence", "gateway-image.json")) + require.NoError(t, err) + image, err := resources.ParseImageLock(content, gatewayImageName) + require.NoError(t, err) + require.Equal(t, "ghcr.io/codefly-dev/service-warehouse@"+valid, image.FullName()) + for _, digest := range []string{"", "sha256:" + strings.Repeat("A", 64), "sha256:abc"} { + _, err = run(lock, "IMAGE=ghcr.io/codefly-dev/service-warehouse", "DIGEST="+digest) + require.Error(t, err, "digest %q must be rejected", digest) + } +} diff --git a/sbom_test.go b/sbom_test.go index d6b4e83..0334a94 100644 --- a/sbom_test.go +++ b/sbom_test.go @@ -244,26 +244,30 @@ func expectedFromReleasePipeline(t *testing.T, reference string) []*builderv0.Im return expected } -// publishedPlatforms reads the explicit inputs supplied to core's publisher. +// publishedPlatforms reads the actual build step in this repository's publisher. func publishedPlatforms(t *testing.T) []string { t.Helper() var workflow struct { Jobs map[string]struct { - Uses string `yaml:"uses"` - With struct { - Platforms string `yaml:"platforms"` - } `yaml:"with"` + Steps []struct { + Uses string `yaml:"uses"` + With struct { + Platforms string `yaml:"platforms"` + } `yaml:"with"` + } `yaml:"steps"` } `yaml:"jobs"` } require.NoError(t, yaml.Unmarshal([]byte(workflowBody(t, "publish-gateway-image.yml")), &workflow)) var published []string for _, job := range workflow.Jobs { - if !strings.HasPrefix(job.Uses, "codefly-dev/core/.github/workflows/publish-service-image.yml@") { - continue - } - for _, platform := range strings.Split(job.With.Platforms, ",") { - if platform = strings.TrimSpace(platform); platform != "" { - published = append(published, platform) + for _, step := range job.Steps { + if !strings.HasPrefix(step.Uses, "docker/build-push-action@") { + continue + } + for _, platform := range strings.Split(step.With.Platforms, ",") { + if platform = strings.TrimSpace(platform); platform != "" { + published = append(published, platform) + } } } } From 784ec02cdd54d7ef8c70d6cc3e99994ed978b466 Mon Sep 17 00:00:00 2001 From: Antoine Toussaint <312536060+antoinetoussaint-byte@users.noreply.github.com> Date: Sun, 11 Oct 2026 13:43:37 -0400 Subject: [PATCH 10/10] fixup: address r26 verification and preserve issue #12 tracking Refs #12. Return explicit readiness PASSED/FAILED results for CLI 0.1.185, reject the reserved anonymous-mode key across deployment configuration sources, and use anonymous Git reads in the public repository's publisher guard. The coordinator must supply an explicit squash body containing the reference above. The historical closing directive in commit 632feea must be omitted from that squash message so the issue stays open until the release is verified. --- .github/workflows/agent-ci.yml | 2 +- .github/workflows/publish-gateway-image.yml | 15 ++--- AGENTS.md | 18 ++++-- README.md | 31 ++++++++--- builder.go | 47 +++++++++++++++- composition_test.go | 61 +++++++++++++++++++++ publisher_test.go | 15 +++-- readiness_test.go | 41 ++++++++++++++ runtime.go | 20 ++++++- runtime_e2e_test.go | 5 ++ 10 files changed, 219 insertions(+), 36 deletions(-) diff --git a/.github/workflows/agent-ci.yml b/.github/workflows/agent-ci.yml index 5513fe2..8e670ec 100644 --- a/.github/workflows/agent-ci.yml +++ b/.github/workflows/agent-ci.yml @@ -19,4 +19,4 @@ jobs: with: # The CLI release the gate runs with. agent.codefly.yaml selects the # source-packager version (0.0.69); go.mod selects Go 1.27.2. - codefly-cli-version: "0.1.181" + codefly-cli-version: "0.1.185" diff --git a/.github/workflows/publish-gateway-image.yml b/.github/workflows/publish-gateway-image.yml index bd454d5..0d0fcbe 100644 --- a/.github/workflows/publish-gateway-image.yml +++ b/.github/workflows/publish-gateway-image.yml @@ -32,24 +32,17 @@ jobs: - name: Require the commit to be on the default branch, or refuse id: default-branch - env: - GH_TOKEN: ${{ github.token }} run: | set -euo pipefail - # Authentication exists for these remote reads only; no persisted - # credential-bearing git config can enter the Docker build context. - git_remote() { - local authorization - authorization=$(printf 'x-access-token:%s' "$GH_TOKEN" | base64 | tr -d '\n') - git -c "http.https://github.com/.extraheader=AUTHORIZATION: basic ${authorization}" "$@" - } - authority=$(git_remote ls-remote --symref origin HEAD \ + # This repository is public. Remote reads need no token; registry + # login below is the only consumer of the job's GITHUB_TOKEN. + authority=$(git ls-remote --symref origin HEAD \ | awk '$1 == "ref:" && $3 == "HEAD" { sub("refs/heads/", "", $2); print $2 }') if [ -z "$authority" ]; then echo "::error::cannot read this repository's default branch from origin; refusing to publish." >&2 exit 1 fi - git_remote fetch --quiet --no-tags --force origin \ + git fetch --quiet --no-tags --force origin \ "+refs/heads/${authority}:refs/remotes/origin/${authority}" if ! git merge-base --is-ancestor "$GITHUB_SHA" "refs/remotes/origin/${authority}"; then echo "::error::${GITHUB_REF_NAME} points at ${GITHUB_SHA}, which is not reachable from refs/remotes/origin/${authority}; refusing to publish." >&2 diff --git a/AGENTS.md b/AGENTS.md index 956cd08..1dd1220 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -32,7 +32,9 @@ this file needs a fix in the same PR. manifest. Deployments require an environment-managed `SWH_AUTH_TOKEN` secret reference; -consumers bind the same key to `warehouse.token`. The templates render a +consumers bind the same key to `warehouse.token`. Reject caller-supplied +`SWH_ALLOW_ANONYMOUS` keys from every configuration source, even false/empty +values; only the template may supply its single literal false. The templates render a same-namespace ingress NetworkPolicy. Anonymous access is only an explicit local gateway opt-in. README owns the secret binding procedure and policy limits. @@ -121,14 +123,16 @@ output for `--- PASS`, never `ok` alone. `codefly agent ci` is the full agent conformance gate. Its workflow (`agent-ci.yml`) is `workflow_dispatch` only until a full run has been seen green -here. Run `GOTOOLCHAIN=go1.27.2 codefly agent ci` from the repo root. The manifest +here. Run `GOTOOLCHAIN=go1.27.2 codefly agent ci` from the repo root with CLI +0.1.185, also pinned in `agent-ci.yml`. The manifest pins source packager `codefly.dev/go:0.0.69`, whose image recipes use immutable digests. The gateway build pins Go 1.27.2 by tag and image-index digest and fixes `GOTOOLCHAIN`; keep `go.mod`, CI and the image toolchain in sync. A fresh service declares `backend: mem`; Runtime.Test probes health and authenticated Capabilities -even when Start was not called. Before the first published lock, full conformance -still fails closed on the absent image; a passing binary audit is not full -conformance. `codefly ci run` operates on a consuming workspace: this agent +even when Start was not called, and returns an explicit PASSED/FAILED run result. +Before the first published lock, set `SWH_GATEWAY_IMAGE` to a local build to +exercise the generated-service test phase; published-image audit still refuses +the absent digest. A passing binary audit is not full conformance. `codefly ci run` operates on a consuming workspace: this agent repository has no workspace manifest. Run the server locally against the zero-dependency backend — it needs no @@ -166,6 +170,10 @@ every consumer of it — the agent, `Deploy`, the release — refuses an empty o rather than run `:latest`. The tag must match `version:` in `agent.codefly.yaml` or the job fails by design. +The coordinator must use an explicit squash body with `Refs #12`; commit +`632feea` contains a closing directive that must not enter the squash message. +Keep the issue open until the release has been verified. + Core is pinned in two places that move together: the `codefly-dev/core` version in `go.mod` and the commit SHA of core's reusable workflows in `release.yml` and `agent-ci.yml` (never `@main`: `release.yml` passes `secrets: inherit`). The image diff --git a/README.md b/README.md index c25c385..d4daac7 100644 --- a/README.md +++ b/README.md @@ -144,7 +144,10 @@ choose the engine. Newly generated services declare `backend: mem` (the catalog-only backend, with no query execution). Removing that setting falls back to the gateway's unsupported `duckdb` default and startup fails. The agent's `Test` step performs the same health and authenticated `Capabilities` check as -`Start`, so conformance's Init → Test path rejects a dead gateway. The agent delivers `SWH_BACKEND`, `SWH_DATABASE`, +`Start`, so conformance's Init → Test path rejects a dead gateway. It returns an +explicit run-level `PASSED` or `FAILED` result and consistent counts; CLI 0.1.185 +requires that verdict rather than inferring it from legacy status fields. +The agent delivers `SWH_BACKEND`, `SWH_DATABASE`, `SWH_DATASET` and `SWH_LOCATION`, and refuses `SWH_CREDENTIALS_FILE`, which nothing could project into the container. `SWH_GATEWAY_IMAGE` makes the agent run a locally built image instead of the pinned one (`docker build -t @@ -181,8 +184,12 @@ is a namespace boundary, not a declared-consumer allowlist; same-namespace pods still need the token. Cross-namespace consumers require an explicit policy overlay. No service mesh or mTLS is installed by this agent. -`SWH_ALLOW_ANONYMOUS=true` is refused in deployment configuration. A configured -`SWH_CREDENTIALS_FILE` is also refused because nothing mounts that file: grant +The key `SWH_ALLOW_ANONYMOUS` is reserved in deployments: any caller-supplied +occurrence is refused, including secret references, templated/secret/plain +configuration, raw ConfigMap data and environment overrides, even with an empty +or `false` value. The template owns the single literal `false` entry. Core +v0.18.1 exposes no ConfigMap-reference input; it cannot render a ConfigMap-key +reference supplied through that API. A configured `SWH_CREDENTIALS_FILE` is also refused because nothing mounts that file: grant the dedicated ServiceAccount a workload identity, or supply a mount and variable through an external overlay. Other `SWH_*` values are bound onto the container by name from the environment's service configuration after the render. @@ -244,7 +251,8 @@ denied `installation not allowed to Write organization package` when publishing a service package. `publish-gateway-image.yml` therefore defines the job here, logs in with its own `GITHUB_TOKEN` and `packages: write`, refuses unmerged commits before login, checks out with `persist-credentials: false`, and pushes -by digest with no tags. Replace these local publishing steps with core 0.18.2's +by digest with no tags. The public Git remote reads are anonymous; the branch +check receives no token and puts no credentials in Git's command line. Replace these local publishing steps with core 0.18.2's composite action once core#770 is available and adopted; keep the job in this repository. The other core integrations remain pinned to v0.18.1. @@ -252,7 +260,11 @@ The coordinator performs the following first-release sequence. The manifest's seed is `0.0.1`; `codefly publish` increments it, so the **first release is `v0.0.2`** and the image publisher must receive **`0.0.2`**. -1. Merge the reviewed gateway PR after its CI is green. +1. Squash the reviewed gateway PR after its CI is green, using an explicit + squash body with `Refs #12`. Commit `632feea` has a closing directive, so the + default concatenated commit-message body is unsafe. The squash message must + omit the closing keyword `closes` before issue `#12`; keep that issue open + through release verification. 2. On clean, synced `main`, run `codefly publish --dry-run` and confirm `0.0.1` → `0.0.2`. 3. Dispatch `publish-gateway-image` on `main` with `version=0.0.2` (no `v`). @@ -310,9 +322,12 @@ The release-gate regression tests use `git` and `jq` (also present in CI). The source packager is pinned to `codefly.dev/go:0.0.69`, whose image recipes use immutable digests. `go.mod` and CI require Go 1.27.2; the gateway Dockerfile pins the Go 1.27.2 image index and sets `GOTOOLCHAIN=go1.27.2`. For local conformance, -run `GOTOOLCHAIN=go1.27.2 codefly agent ci` so host toolchain selection is explicit. -Before the first image lock, the agent-binary audit can pass but full generated -service conformance still fails closed on the unpublished image. +run `GOTOOLCHAIN=go1.27.2 codefly agent ci` with CLI **0.1.185**, as pinned in +`agent-ci.yml`. Before the first image lock, also set +`SWH_GATEWAY_IMAGE=service-warehouse:e2e` to a locally built image when verifying +the generated service's test phase. This override is only for runtime and image +SBOM testing: the audit of the service's published image still fails closed on +the absent lock. A passing binary audit alone is not full conformance. The agent's end-to-end tests need Docker and are behind the `e2e` tag; the SBOM one also needs `syft` on `PATH`: diff --git a/builder.go b/builder.go index 20582e1..d9b4550 100644 --- a/builder.go +++ b/builder.go @@ -5,6 +5,7 @@ import ( "embed" "fmt" "regexp" + "strings" "github.com/codefly-dev/core/agents/communicate" "github.com/codefly-dev/core/agents/services" @@ -141,6 +142,10 @@ func (s *Builder) Deploy(ctx context.Context, req *builderv0.DeploymentRequest) return s.Builder.DeployError(err) } + if err := s.rejectAnonymousDeploymentConfiguration(req); err != nil { + return s.Builder.DeployError(err) + } + parameters := &deploymentTemplateParameters{ Backend: s.conf.backend, Database: s.conf.database, @@ -195,9 +200,6 @@ func (s *Builder) Deploy(ctx context.Context, req *builderv0.DeploymentRequest) if ref.GetName() == "" || ref.GetKey() == "" { return s.Builder.DeployError(fmt.Errorf("deployment requires an SWH_AUTH_TOKEN Kubernetes secret reference; bind the same secret key to consumers' warehouse.token configuration")) } - if value, _ := resources.GetConfigurationValue(ctx, req.GetConfiguration(), configurationGroup, "SWH_ALLOW_ANONYMOUS"); value != "" && value != "false" { - return s.Builder.DeployError(fmt.Errorf("SWH_ALLOW_ANONYMOUS is only an explicit local gateway opt-in; deployments require authentication")) - } return s.Builder.DeployKustomize(ctx, req, services.KustomizeDeployment{ EnvironmentVariables: s.EnvironmentVariables, Templates: deploymentFS, @@ -206,6 +208,45 @@ func (s *Builder) Deploy(ctx context.Context, req *builderv0.DeploymentRequest) }) } +// rejectAnonymousDeploymentConfiguration reserves this key for the template's +// literal false. Reject its presence, regardless of value, group or delivery +// mechanism: a reference's value cannot be known at render time. Core v0.18.1 +// exposes secret references but no ConfigMap-reference field; raw configuration +// and environment-manager values cover its ConfigMap/Secret data paths. +func (s *Builder) rejectAnonymousDeploymentConfiguration(req *builderv0.DeploymentRequest) error { + reject := func(key string) error { + if key == "SWH_ALLOW_ANONYMOUS" || strings.HasSuffix(key, "__SWH_ALLOW_ANONYMOUS") { + return fmt.Errorf("SWH_ALLOW_ANONYMOUS is only an explicit local gateway opt-in; the key is reserved in deployments, including values and references") + } + return nil + } + for key := range req.GetDeployment().GetKubernetes().GetSecretReferences() { + if err := reject(key); err != nil { + return err + } + } + configurations := append([]*v0.Configuration{req.GetConfiguration()}, req.GetDependenciesConfigurations()...) + for _, configuration := range configurations { + for _, info := range configuration.GetInfos() { + for _, value := range info.GetConfigurationValues() { + if err := reject(value.GetKey()); err != nil { + return err + } + } + } + } + values, err := s.EnvironmentVariables.All() + if err != nil { + return err + } + for _, value := range values { + if err := reject(value.Key); err != nil { + return err + } + } + return nil +} + // servicePort is the port the rendered Service publishes for the gRPC endpoint: // the in-cluster port core allocated to it, which is what every consumer is // given to dial. With no container mapping for this endpoint in the request (a diff --git a/composition_test.go b/composition_test.go index dc6689d..641609f 100644 --- a/composition_test.go +++ b/composition_test.go @@ -598,3 +598,64 @@ func TestDeployRequiresAuthenticationAndRendersNetworkBoundary(t *testing.T) { }) } } + +func TestDeployRejectsAnonymousKeyFromEveryConfigurationSource(t *testing.T) { + const key = "SWH_ALLOW_ANONYMOUS" + cases := map[string]func(*Builder, *builderv0.DeploymentRequest){ + "secret reference": func(_ *Builder, req *builderv0.DeploymentRequest) { + req.GetDeployment().GetKubernetes().SecretReferences[key] = &builderv0.KubernetesSecretKeyReference{Name: "override", Key: "flag"} + }, + "nil secret reference": func(_ *Builder, req *builderv0.DeploymentRequest) { + req.GetDeployment().GetKubernetes().SecretReferences[key] = nil + }, + "secret configuration": func(_ *Builder, req *builderv0.DeploymentRequest) { + req.Configuration = configuration(map[string]string{key: "true"}) + req.Configuration.Infos[0].ConfigurationValues[0].Secret = true + }, + "templated configuration": func(_ *Builder, req *builderv0.DeploymentRequest) { + req.Configuration = configuration(map[string]string{key: ""}) + req.Configuration.Infos[0].ConfigurationValues[0].Template = &basev0.ConfigurationValueTemplate{} + }, + "another group": func(_ *Builder, req *builderv0.DeploymentRequest) { + req.Configuration = configuration(map[string]string{key: "true"}) + req.Configuration.Infos[0].Name = "another" + }, + "dependency configuration": func(_ *Builder, req *builderv0.DeploymentRequest) { + req.DependenciesConfigurations = []*basev0.Configuration{configuration(map[string]string{key: "true"})} + }, + "ConfigMap data via raw configuration": func(b *Builder, _ *builderv0.DeploymentRequest) { + require.NoError(t, b.EnvironmentVariables.AddRawConfigurations(context.Background(), configuration(map[string]string{key: "true"}))) + }, + "environment override": func(b *Builder, _ *builderv0.DeploymentRequest) { + b.EnvironmentVariables.SetOverrides(map[string]string{key: "true"}) + }, + "configuration carrier": func(b *Builder, _ *builderv0.DeploymentRequest) { + b.EnvironmentVariables.SetOverrides(map[string]string{"CODEFLY__SERVICE_CONFIGURATION__MODULE__WAREHOUSE__WAREHOUSE__" + key: "true"}) + }, + } + // Presence is prohibited: false or an unresolved/empty value must not create + // a second entry that can later be rebound from another delivery source. + for _, value := range []string{"true", "false", "", "1", "TRUE", "yes"} { + cases["literal "+value] = func(_ *Builder, req *builderv0.DeploymentRequest) { + req.Configuration = configuration(map[string]string{key: value}) + } + } + for _, profile := range []builderv0.KubernetesOutputProfile{ephemeral, restricted} { + for name, prepare := range cases { + t.Run(profile.String()+"/"+name, func(t *testing.T) { + builder := newDeployBuilder(t, context.Background()) + destination := t.TempDir() + req := deployRequest(destination, nil, profile) + prepare(builder, req) + response, err := builder.Deploy(context.Background(), req) + require.NoError(t, err) + require.Equal(t, builderv0.DeploymentStatus_ERROR, response.GetState().GetState()) + require.Contains(t, response.GetState().GetMessage(), key) + require.Contains(t, response.GetState().GetMessage(), "key is reserved") + files, err := os.ReadDir(destination) + require.NoError(t, err) + require.Empty(t, files, "a forbidden key must fail before rendering") + }) + } + } +} diff --git a/publisher_test.go b/publisher_test.go index 3081a86..4d3488d 100644 --- a/publisher_test.go +++ b/publisher_test.go @@ -13,10 +13,11 @@ import ( ) type publisherStep struct { - ID string `yaml:"id"` - Uses string `yaml:"uses"` - Run string `yaml:"run"` - With map[string]any `yaml:"with"` + ID string `yaml:"id"` + Uses string `yaml:"uses"` + Run string `yaml:"run"` + Env map[string]string `yaml:"env"` + With map[string]any `yaml:"with"` } type publisherJob struct { @@ -116,10 +117,14 @@ func TestPublisherRefusesCommitOutsideDefaultBranch(t *testing.T) { unmerged := git(dir, "rev-parse", "HEAD") guard := publisherStepByID(t, "default-branch") + require.Empty(t, guard.Env, "public git reads must receive no token") + require.NotContains(t, guard.Run, "$GH_TOKEN") + require.NotContains(t, guard.Run, "$GITHUB_TOKEN") + require.NotContains(t, guard.Run, "extraheader") run := func(sha string) (string, error) { command := exec.Command("bash", "-c", guard.Run) command.Dir = dir - command.Env = append(os.Environ(), "GH_TOKEN=fixture-only", "GITHUB_SHA="+sha, "GITHUB_REF_NAME=trunk") + command.Env = append(os.Environ(), "GITHUB_SHA="+sha, "GITHUB_REF_NAME=trunk") out, err := command.CombinedOutput() return string(out), err } diff --git a/readiness_test.go b/readiness_test.go index 2d6c557..bdb12a7 100644 --- a/readiness_test.go +++ b/readiness_test.go @@ -8,6 +8,7 @@ import ( "github.com/codefly-dev/core/callertoken" basev0 "github.com/codefly-dev/core/generated/go/codefly/base/v0" + runtimev0 "github.com/codefly-dev/core/generated/go/codefly/services/runtime/v0" "github.com/codefly-dev/core/resources" "github.com/stretchr/testify/require" "google.golang.org/grpc" @@ -127,3 +128,43 @@ func TestWaitForReadyReturnsOnCancellation(t *testing.T) { } func insecureCredentials() credentials.TransportCredentials { return insecure.NewCredentials() } + +// The CLI judges Result, not legacy Status. A successful readiness assertion +// must not decode as UNKNOWN, and a failed assertion must report FAILED. +func TestRuntimeTestReportsRunLevelOutcome(t *testing.T) { + address, health := startGateway(t, "server-token") + for _, tc := range []struct { + name string + serving healthv1.HealthCheckResponse_ServingStatus + want runtimev0.TestRunResult_State + passed, failed int32 + }{ + {"serving", healthv1.HealthCheckResponse_SERVING, runtimev0.TestRunResult_PASSED, 1, 0}, + {"not serving", healthv1.HealthCheckResponse_NOT_SERVING, runtimev0.TestRunResult_FAILED, 0, 1}, + } { + t.Run(tc.name, func(t *testing.T) { + health.SetServingStatus("", tc.serving) + rt := runtimeAt(t, address, "server-token") + budget := 5 * time.Second + if tc.failed > 0 { + budget = 200 * time.Millisecond + } + ctx, cancel := context.WithTimeout(context.Background(), budget) + defer cancel() + response, err := rt.Test(ctx, &runtimev0.TestRequest{}) + require.NoError(t, err) + require.Equal(t, tc.want, response.GetResult().GetState()) + require.EqualValues(t, 1, response.GetCounts().GetTotal()) + require.Equal(t, tc.passed, response.GetCounts().GetPassed()) + require.Equal(t, tc.failed, response.GetCounts().GetFailed()) + require.Equal(t, tc.passed, response.GetTestsPassed()) + require.Equal(t, tc.failed, response.GetTestsFailed()) + if tc.failed > 0 { + require.Equal(t, runtimev0.TestStatus_ERROR, response.GetStatus().GetState()) + require.Contains(t, response.GetResult().GetMessage(), "not ready") + } else { + require.Equal(t, runtimev0.TestStatus_SUCCESS, response.GetStatus().GetState()) + } + }) + } +} diff --git a/runtime.go b/runtime.go index 653c77d..2fa673a 100644 --- a/runtime.go +++ b/runtime.go @@ -340,8 +340,22 @@ func (s *Runtime) Test(ctx context.Context, req *runtimev0.TestRequest) (*runtim defer s.Wool.Catch() // Conformance calls Init then Test without Start. Creating a container is // not proof that it stayed alive or accepts the credential we handed out. - if err := s.WaitForReady(s.Wool.Inject(ctx)); err != nil { - return s.Runtime.TestError(err) + readinessErr := s.WaitForReady(s.Wool.Inject(ctx)) + passed, failed := int32(1), int32(0) + state := runtimev0.TestRunResult_PASSED + var failures []string + if readinessErr != nil { + passed, failed = 0, 1 + state = runtimev0.TestRunResult_FAILED + failures = []string{readinessErr.Error()} } - return s.Runtime.TestResponseWithResults(1, 1, 0, 0, 0, nil, nil) + response, err := s.Runtime.TestResponseWithResults(1, passed, failed, 0, 0, failures, readinessErr) + if err != nil { + return nil, err + } + // CLI 0.1.185 judges the run-level verdict; legacy status/counters alone + // decode as UNKNOWN. Keep both representations consistent for older hosts. + response.Result = &runtimev0.TestRunResult{State: state, Message: response.GetStatus().GetMessage()} + response.Counts = &runtimev0.TestCounts{Total: 1, Passed: passed, Failed: failed} + return response, nil } diff --git a/runtime_e2e_test.go b/runtime_e2e_test.go index 1fdd573..87093ed 100644 --- a/runtime_e2e_test.go +++ b/runtime_e2e_test.go @@ -78,6 +78,8 @@ func TestRuntimeEndToEnd(t *testing.T) { require.NoError(t, err) require.Equal(t, runtimev0.TestStatus_SUCCESS, tested.GetStatus().GetState(), tested.GetStatus().GetMessage()) require.EqualValues(t, 1, tested.GetTestsPassed()) + require.Equal(t, runtimev0.TestRunResult_PASSED, tested.GetResult().GetState()) + require.EqualValues(t, 1, tested.GetCounts().GetPassed()) start, err := rt.Start(ctx, &runtimev0.StartRequest{}) require.NoError(t, err) @@ -276,6 +278,9 @@ func TestRuntimeEndToEndRejectsDeadGateway(t *testing.T) { result, err := rt.Test(probe, &runtimev0.TestRequest{}) require.NoError(t, err) require.Equal(t, runtimev0.TestStatus_ERROR, result.GetStatus().GetState()) + require.Equal(t, runtimev0.TestRunResult_FAILED, result.GetResult().GetState()) + require.EqualValues(t, 1, result.GetCounts().GetFailed()) + require.Contains(t, result.GetResult().GetMessage(), "not ready") require.Contains(t, result.GetStatus().GetMessage(), "not ready") require.NotContains(t, result.GetStatus().GetMessage(), "no attempt completed") }