|
| 1 | +package main |
| 2 | + |
| 3 | +import ( |
| 4 | + "context" |
| 5 | + "fmt" |
| 6 | + "os/exec" |
| 7 | + "path/filepath" |
| 8 | + "strings" |
| 9 | + "testing" |
| 10 | + "time" |
| 11 | + |
| 12 | + "github.com/codefly-dev/core/agents/services/sbom" |
| 13 | + basev0 "github.com/codefly-dev/core/generated/go/codefly/base/v0" |
| 14 | + agentv0 "github.com/codefly-dev/core/generated/go/codefly/services/agent/v0" |
| 15 | + builderv0 "github.com/codefly-dev/core/generated/go/codefly/services/builder/v0" |
| 16 | + "github.com/stretchr/testify/require" |
| 17 | +) |
| 18 | + |
| 19 | +// imageSBOMRequest asks for image scope, optionally naming subjects. |
| 20 | +func imageSBOMRequest(subjects ...*builderv0.ImageSubject) *builderv0.SBOMRequest { |
| 21 | + return &builderv0.SBOMRequest{ |
| 22 | + Scope: builderv0.SBOMScope_SBOM_SCOPE_IMAGE, |
| 23 | + Subjects: subjects, |
| 24 | + } |
| 25 | +} |
| 26 | + |
| 27 | +// emitRecipePlan runs a recipe build and returns the emitted plan together with |
| 28 | +// the directory the Dockerfile was written to. |
| 29 | +func emitRecipePlan(t *testing.T, builder *Builder) (*builderv0.DockerBuildPlan, string) { |
| 30 | + t.Helper() |
| 31 | + outputDir := t.TempDir() |
| 32 | + ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second) |
| 33 | + defer cancel() |
| 34 | + resp, err := builder.Build(ctx, recipeBuildRequest(outputDir)) |
| 35 | + require.NoError(t, err) |
| 36 | + require.Equal(t, builderv0.BuildStatus_SUCCESS, resp.GetState().GetState(), resp.GetState().GetMessage()) |
| 37 | + plan := resp.GetResult().GetDockerBuildPlan() |
| 38 | + require.NotNil(t, plan) |
| 39 | + return plan, outputDir |
| 40 | +} |
| 41 | + |
| 42 | +// TestImageSBOMWithoutSubjectsIsPreconditionFailure pins the answer this agent |
| 43 | +// owes when it is asked to enumerate images it never builds. The CLI executes |
| 44 | +// the emitted recipe, so the digest exists only on its side. Reporting |
| 45 | +// UNSUPPORTED would deny an implementation that exists, and a no-image reason |
| 46 | +// would deny an image the service really ships — both are the false coverage |
| 47 | +// claim the contract exists to prevent. |
| 48 | +func TestImageSBOMWithoutSubjectsIsPreconditionFailure(t *testing.T) { |
| 49 | + builder, _ := createdBuilder(t) |
| 50 | + |
| 51 | + resp, err := builder.SBOM(t.Context(), imageSBOMRequest()) |
| 52 | + require.NoError(t, err) |
| 53 | + |
| 54 | + require.Equal(t, builderv0.SBOMStatus_ERROR, resp.GetState().GetState()) |
| 55 | + require.Equal(t, basev0.FailureCode_FAILURE_CODE_PRECONDITION_FAILED, resp.GetState().GetFailure().GetCode()) |
| 56 | + require.Equal(t, builderv0.SBOMScope_SBOM_SCOPE_IMAGE, resp.GetScope()) |
| 57 | + require.Equal(t, builderv0.NoImageReason_NO_IMAGE_REASON_UNSPECIFIED, resp.GetNoImageReason()) |
| 58 | + require.Empty(t, resp.GetImages()) |
| 59 | +} |
| 60 | + |
| 61 | +// TestSourceScopeIsNotRoutedToImageEvidence guards the scope condition itself. |
| 62 | +// Builder.SBOM now sits in front of the inherited source inventory, so an |
| 63 | +// inverted comparison would answer every source request with image evidence |
| 64 | +// and nothing else in this repository would notice. |
| 65 | +// |
| 66 | +// It asserts the response shape rather than its contents so it holds with or |
| 67 | +// without uv installed: a source answer never carries image scope, image |
| 68 | +// inventories, or a no-image reason, whether the inventory succeeded or failed. |
| 69 | +func TestSourceScopeIsNotRoutedToImageEvidence(t *testing.T) { |
| 70 | + builder, _ := createdBuilder(t) |
| 71 | + |
| 72 | + for _, scope := range []builderv0.SBOMScope{ |
| 73 | + builderv0.SBOMScope_SBOM_SCOPE_UNSPECIFIED, |
| 74 | + builderv0.SBOMScope_SBOM_SCOPE_SOURCE, |
| 75 | + } { |
| 76 | + t.Run(scope.String(), func(t *testing.T) { |
| 77 | + resp, err := builder.SBOM(t.Context(), &builderv0.SBOMRequest{Scope: scope}) |
| 78 | + require.NoError(t, err) |
| 79 | + require.NotEqual(t, builderv0.SBOMScope_SBOM_SCOPE_IMAGE, resp.GetScope()) |
| 80 | + require.Empty(t, resp.GetImages()) |
| 81 | + require.Equal(t, builderv0.NoImageReason_NO_IMAGE_REASON_UNSPECIFIED, resp.GetNoImageReason()) |
| 82 | + }) |
| 83 | + } |
| 84 | +} |
| 85 | + |
| 86 | +// TestImageSBOMExpectationsMatchTheEmittedRecipe verifies the specialization's |
| 87 | +// actual build-recipe path against the coverage helper the fleet grades every |
| 88 | +// agent with: one subject per shipped platform, carrying the recipe's role and |
| 89 | +// final image reference. |
| 90 | +func TestImageSBOMExpectationsMatchTheEmittedRecipe(t *testing.T) { |
| 91 | + builder, _ := createdBuilder(t) |
| 92 | + plan, _ := emitRecipePlan(t, builder) |
| 93 | + |
| 94 | + expected := sbom.ExpectedFromBuildPlan(builder.Identity.Name, plan) |
| 95 | + |
| 96 | + require.Len(t, expected, 2) |
| 97 | + var platforms []string |
| 98 | + for _, subject := range expected { |
| 99 | + platforms = append(platforms, subject.GetPlatform()) |
| 100 | + require.Equal(t, "app", subject.GetRole()) |
| 101 | + require.Equal(t, builder.Identity.Name, subject.GetService()) |
| 102 | + require.NotEmpty(t, subject.GetReference()) |
| 103 | + } |
| 104 | + require.ElementsMatch(t, []string{"linux/amd64", "linux/arm64"}, platforms) |
| 105 | +} |
| 106 | + |
| 107 | +// TestImageSBOMCoverageRejectsFalseClaims is the release gate: every shape that |
| 108 | +// could pass off "no evidence" as complete image coverage has to fail |
| 109 | +// ValidateCoverage against the service's own declared build. |
| 110 | +func TestImageSBOMCoverageRejectsFalseClaims(t *testing.T) { |
| 111 | + builder, _ := createdBuilder(t) |
| 112 | + plan, _ := emitRecipePlan(t, builder) |
| 113 | + expected := sbom.ExpectedFromBuildPlan(builder.Identity.Name, plan) |
| 114 | + |
| 115 | + t.Run("no-image reason contradicting the declared build", func(t *testing.T) { |
| 116 | + resp, err := builder.Base.Builder.SBOMNoImage(builderv0.NoImageReason_NO_IMAGE_REASON_EXTERNALLY_MANAGED, "vendor image") |
| 117 | + require.NoError(t, err) |
| 118 | + require.Error(t, sbom.ValidateCoverage(expected, resp)) |
| 119 | + }) |
| 120 | + |
| 121 | + t.Run("source inventory is not image coverage", func(t *testing.T) { |
| 122 | + resp := &builderv0.SBOMResponse{ |
| 123 | + State: &builderv0.SBOMStatus{State: builderv0.SBOMStatus_COMPLETE}, |
| 124 | + Scope: builderv0.SBOMScope_SBOM_SCOPE_SOURCE, |
| 125 | + } |
| 126 | + require.Error(t, sbom.ValidateCoverage(expected, resp)) |
| 127 | + }) |
| 128 | + |
| 129 | + t.Run("precondition failure never reads as coverage", func(t *testing.T) { |
| 130 | + resp, err := builder.SBOM(t.Context(), imageSBOMRequest()) |
| 131 | + require.NoError(t, err) |
| 132 | + require.Error(t, sbom.ValidateCoverage(expected, resp)) |
| 133 | + }) |
| 134 | + |
| 135 | + t.Run("one platform does not cover a multi-architecture image", func(t *testing.T) { |
| 136 | + covered := expected[0] |
| 137 | + resp, err := builder.Base.Builder.SBOMImageResponse([]*builderv0.ImageSBOM{{ |
| 138 | + Digest: "sha256:" + strings.Repeat("a", 64), |
| 139 | + Platform: covered.GetPlatform(), |
| 140 | + Subjects: []*builderv0.ImageSubject{covered}, |
| 141 | + Bom: &agentv0.Bom{Components: []*agentv0.Component{{BomRef: "pkg:apk/alpine/musl@1.2.5"}}}, |
| 142 | + Sha256: strings.Repeat("b", 64), |
| 143 | + }}) |
| 144 | + require.NoError(t, err) |
| 145 | + require.Error(t, sbom.ValidateCoverage(expected, resp)) |
| 146 | + }) |
| 147 | +} |
| 148 | + |
| 149 | +// TestImageSBOMPropagatesScanFailures proves an unreachable image is reported as |
| 150 | +// an image-scope error rather than silently omitted from an otherwise complete |
| 151 | +// response. |
| 152 | +func TestImageSBOMPropagatesScanFailures(t *testing.T) { |
| 153 | + builder, _ := createdBuilder(t) |
| 154 | + |
| 155 | + subject := &builderv0.ImageSubject{ |
| 156 | + // A digest-pinned reference on a closed port fails resolution fast, and |
| 157 | + // on a host without docker fails the scan instead — an error either way. |
| 158 | + Reference: "127.0.0.1:1/missing@sha256:" + strings.Repeat("0", 64), |
| 159 | + Service: builder.Identity.Name, |
| 160 | + Role: "app", |
| 161 | + } |
| 162 | + |
| 163 | + ctx, cancel := context.WithTimeout(t.Context(), 2*time.Minute) |
| 164 | + defer cancel() |
| 165 | + resp, err := builder.SBOM(ctx, imageSBOMRequest(subject)) |
| 166 | + require.NoError(t, err) |
| 167 | + |
| 168 | + require.Equal(t, builderv0.SBOMStatus_ERROR, resp.GetState().GetState()) |
| 169 | + require.Equal(t, builderv0.SBOMScope_SBOM_SCOPE_IMAGE, resp.GetScope()) |
| 170 | + require.Empty(t, resp.GetImages()) |
| 171 | + require.Error(t, sbom.ValidateCoverage([]*builderv0.ImageSubject{subject}, resp)) |
| 172 | +} |
| 173 | + |
| 174 | +// TestImageSBOMInventoriesTheBuiltImage is the end-to-end check: it builds the |
| 175 | +// specialization's real runtime image from the recipe this agent emits, scans |
| 176 | +// it, and requires the inventory to carry both halves a source SBOM can never |
| 177 | +// prove — the base image's OS packages and the application's installed |
| 178 | +// dependencies — bound to the digest that was actually scanned. |
| 179 | +func TestImageSBOMInventoriesTheBuiltImage(t *testing.T) { |
| 180 | + for _, tool := range []string{"docker", "uv", "syft"} { |
| 181 | + if _, err := exec.LookPath(tool); err != nil { |
| 182 | + t.Skipf("%s is not installed: the real runtime image cannot be built and scanned", tool) |
| 183 | + } |
| 184 | + } |
| 185 | + |
| 186 | + builder, _ := createdBuilder(t) |
| 187 | + plan, outputDir := emitRecipePlan(t, builder) |
| 188 | + |
| 189 | + ctx, cancel := context.WithTimeout(t.Context(), 20*time.Minute) |
| 190 | + defer cancel() |
| 191 | + |
| 192 | + // The Dockerfile installs from a frozen lockfile, which the scaffold does |
| 193 | + // not ship: it is resolved on first run. |
| 194 | + lock := exec.CommandContext(ctx, "uv", "lock") |
| 195 | + lock.Dir = filepath.Join(builder.Location, "code") |
| 196 | + if out, err := lock.CombinedOutput(); err != nil { |
| 197 | + // The tooling guard above already passed, so uv is installed: a failure |
| 198 | + // here is a broken scaffold or lockfile, not an unequipped host. |
| 199 | + t.Fatalf("uv lock failed, the image cannot be built: %v\n%s", err, out) |
| 200 | + } |
| 201 | + |
| 202 | + tag := fmt.Sprintf("codefly-fastapi-sbom-test:%d", time.Now().UnixMilli()) |
| 203 | + // The recipe's context "." is the service directory; the Dockerfile lives |
| 204 | + // in the emitted output directory. |
| 205 | + build := exec.CommandContext(ctx, "docker", "build", |
| 206 | + "-f", filepath.Join(outputDir, plan.GetRecipes()[0].GetDockerfile()), |
| 207 | + "-t", tag, builder.Location) |
| 208 | + if out, err := build.CombinedOutput(); err != nil { |
| 209 | + // Likewise docker is installed, so this is a broken Dockerfile template |
| 210 | + // or recipe. Skipping would turn the regression this test exists to |
| 211 | + // catch into a green run. |
| 212 | + t.Fatalf("docker build failed, the image cannot be scanned: %v\n%s", err, out) |
| 213 | + } |
| 214 | + t.Cleanup(func() { _ = exec.Command("docker", "rmi", "-f", tag).Run() }) |
| 215 | + |
| 216 | + // The image was loaded into the local daemon and never pushed, so its |
| 217 | + // immutable identity is the local image ID. The platform is left unstated: |
| 218 | + // the daemon holds exactly one, and it is read back rather than asserted. |
| 219 | + // |
| 220 | + // This drives Builder.SBOM, the method the CLI actually calls, rather than |
| 221 | + // core's helper. Scope routing and scanner-source resolution are what this |
| 222 | + // repository owns, and a test calling the helper directly passed with |
| 223 | + // either of them broken. |
| 224 | + subject := &builderv0.ImageSubject{ |
| 225 | + Reference: tag, |
| 226 | + Role: plan.GetRecipes()[0].GetName(), |
| 227 | + Service: builder.Identity.Name, |
| 228 | + } |
| 229 | + resp, err := builder.SBOM(ctx, imageSBOMRequest(subject)) |
| 230 | + require.NoError(t, err) |
| 231 | + require.Equal(t, builderv0.SBOMStatus_COMPLETE, resp.GetState().GetState(), resp.GetState().GetMessage()) |
| 232 | + require.Equal(t, builderv0.SBOMScope_SBOM_SCOPE_IMAGE, resp.GetScope()) |
| 233 | + require.Len(t, resp.GetImages(), 1) |
| 234 | + |
| 235 | + evidence := resp.GetImages()[0] |
| 236 | + require.True(t, strings.HasPrefix(evidence.GetDigest(), "sha256:"), evidence.GetDigest()) |
| 237 | + require.NotEmpty(t, evidence.GetPlatform()) |
| 238 | + require.NotEmpty(t, evidence.GetTool()) |
| 239 | + require.NotEmpty(t, evidence.GetSha256()) |
| 240 | + require.Equal(t, []*builderv0.ImageSubject{subject}, evidence.GetSubjects()) |
| 241 | + |
| 242 | + names := map[string]bool{} |
| 243 | + for _, component := range evidence.GetBom().GetComponents() { |
| 244 | + names[strings.ToLower(component.GetName())] = true |
| 245 | + } |
| 246 | + |
| 247 | + // The CI log is where this evidence is retrieved from, so name the digest |
| 248 | + // and platform the inventory is bound to rather than only asserting them. |
| 249 | + t.Logf("image evidence: digest=%s platform=%s tool=%s components=%d document-sha256=%s", |
| 250 | + evidence.GetDigest(), evidence.GetPlatform(), evidence.GetTool(), |
| 251 | + len(evidence.GetBom().GetComponents()), evidence.GetSha256()) |
| 252 | + |
| 253 | + // The alpine base layer: packages no lockfile inventory would ever list. |
| 254 | + var osPackages []string |
| 255 | + for _, name := range []string{"musl", "busybox", "alpine-baselayout", "ca-certificates-bundle"} { |
| 256 | + if names[name] { |
| 257 | + osPackages = append(osPackages, name) |
| 258 | + } |
| 259 | + } |
| 260 | + require.NotEmpty(t, osPackages, "image inventory carries no base OS packages") |
| 261 | + |
| 262 | + // The application dependencies as they are installed in the image. |
| 263 | + for _, name := range []string{"fastapi", "uvicorn", "pydantic"} { |
| 264 | + require.True(t, names[name], "image inventory is missing application dependency %s", name) |
| 265 | + } |
| 266 | + |
| 267 | + require.NoError(t, sbom.ValidateCoverage([]*builderv0.ImageSubject{subject}, resp)) |
| 268 | +} |
0 commit comments