diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..9050b4d --- /dev/null +++ b/.dockerignore @@ -0,0 +1,5 @@ +.git +*.md +templates +dist +*.duckdb diff --git a/.github/scripts/allow-unpublished-image.sh b/.github/scripts/allow-unpublished-image.sh new file mode 100644 index 0000000..c161f82 --- /dev/null +++ b/.github/scripts/allow-unpublished-image.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +# CI's sole empty-lock exception: a full pre-release history with no prior lock. +set -euo pipefail +if [ "$(git rev-parse --is-shallow-repository)" != false ]; then + echo "cannot allow an unpublished image with incomplete git history" >&2 + exit 1 +fi +if [ -n "$(git tag --list 'v*')" ]; then + echo "gateway-image.json cannot be empty after a release tag exists" >&2 + exit 1 +fi +# Follow every fetched branch and both parents of merges. Deleting a digest +# before the first tag must not reset the pre-release exception. +for commit in $(git rev-list --all -- gateway-image.json); do + if git cat-file -e "$commit:gateway-image.json" 2>/dev/null; then + digest=$(git show "$commit:gateway-image.json" | jq -er '.digest | strings') + if [ -n "$digest" ]; then + echo "gateway-image.json cannot be empty: a lock already exists in history at $commit" >&2 + exit 1 + fi + fi +done diff --git a/.github/workflows/agent-ci.yml b/.github/workflows/agent-ci.yml new file mode 100644 index 0000000..8e670ec --- /dev/null +++ b/.github/workflows/agent-ci.yml @@ -0,0 +1,22 @@ +# Full-lifecycle agent validation: `codefly agent ci` runs the source, release, +# and generated-service conformance gates against this agent, through the shared +# reusable workflow every codefly service agent uses. +# +# Manual until a full run has been seen green on this repository. Packaging +# needs Docker and source-packager/toolchain downloads; source validation alone +# does not prove the release and generated-service stages. Switch to +# pull_request once the first full run passes. +name: agent-ci + +on: + workflow_dispatch: + +jobs: + agent-ci: + # Pinned to an immutable core commit SHA, not @main (see release.yml): core's + # v0.18.1, the release go.mod requires. Bump the two together. + uses: codefly-dev/core/.github/workflows/agent-ci.yml@ea49e7a968ef5f2fd325b1c41f823a47eb20141c + with: + # The CLI release the gate runs with. agent.codefly.yaml selects the + # source-packager version (0.0.69); go.mod selects Go 1.27.2. + codefly-cli-version: "0.1.185" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 57a46e7..adcd336 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,15 +5,96 @@ on: branches: [main] pull_request: +env: + GOTOOLCHAIN: go1.27.2 + jobs: build-test: runs-on: ubuntu-latest steps: - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + fetch-depth: 0 - uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5 with: - go-version: "1.27.0" + go-version: "1.27.2" cache: true - run: go build ./... - run: go vet ./... - run: go test -race ./... + + # Runs the agent's Runtime against the gateway image this repository builds: + # the container starts, enforces the token the agent minted, and answers a + # consumer who presents it. The unit tests cannot say whether the image the + # Dockerfile produces serves what the agent expects of it. + agent-runtime-e2e: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + fetch-depth: 0 + - uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5 + with: + go-version: "1.27.2" + cache: true + - name: build gateway image + run: docker build -t service-warehouse:e2e . + - name: end-to-end run through the agent runtime + env: + SWH_GATEWAY_IMAGE: service-warehouse:e2e + run: go test -v -tags e2e -count=1 -timeout 300s -run TestRuntimeEndToEnd . + + # Inventories a real gateway image through the agent's own image-SBOM path and + # checks it against the shared coverage contract, so a change that would ship + # an uninventoried image fails here rather than at release. + image-sbom: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + fetch-depth: 0 + - uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5 + with: + go-version: "1.27.2" + cache: true + - name: build gateway image + run: docker build -t service-warehouse:e2e . + # The daemon holds this image and no registry serves it, so the scan needs + # syft on PATH: the managed containerized scanner has no Docker access. + - uses: anchore/sbom-action/download-syft@3ad7283483fc7af8ff2b4ea19663c2d5ca935e26 # v0.24.2 + - name: assert the image contract against a real image + env: + SWH_GATEWAY_IMAGE: service-warehouse:e2e + run: go test -v -tags e2e -count=1 -timeout 300s -run TestImageSBOM . + - name: write evidence through the release command + run: go run ./cmd/image-sbom -image service-warehouse:e2e -local -out sbom + # The local image is single-platform and reached through the daemon. The + # image a release publishes is neither: it is multi-architecture and + # resolved through the registry. Without this step the two-platform path + # the release depends on would never execute in CI. It inventories the + # digest gateway-image.json records — the image the agent actually names — + # so a lock pointing at something that does not resolve, or that ships + # fewer platforms than evidence has to cover, fails here rather than at + # release time. The empty-lock exception ends at the first nonempty lock + # in fetched history or any v* tag. Full history is required above, so + # deleting an established digest cannot quietly skip the published scan. + - name: inventory the published multi-architecture image + run: | + set -euo pipefail + digest=$(jq -er '.digest' gateway-image.json) + if [ -z "$digest" ]; then + bash .github/scripts/allow-unpublished-image.sh + echo "::notice::pre-release tree with no prior image lock: no published image to inventory" + exit 0 + fi + go run ./cmd/image-sbom -image "$(jq -er '.name + "@" + .digest' gateway-image.json)" -out sbom-published + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: gateway-image-sbom + path: | + sbom/ + sbom-published/ + # A skipped scan writes nothing; failing here keeps a green job from + # standing in for evidence that was never produced. sbom/ is always + # written, so this fails only when the local scan did not run. + if-no-files-found: error diff --git a/.github/workflows/publish-gateway-image.yml b/.github/workflows/publish-gateway-image.yml new file mode 100644 index 0000000..0d0fcbe --- /dev/null +++ b/.github/workflows/publish-gateway-image.yml @@ -0,0 +1,114 @@ +name: publish-gateway-image + +# STOPGAP pending codefly-dev/core#770 (the composite publisher in core 0.18.2). +# GHCR judges the workflow's defining repository: core's reusable workflow was +# denied "installation not allowed to Write organization package" when pushing +# a service package. Keep this job and its GITHUB_TOKEN in the service repo until +# the shared composite action can run inside it. The first dispatch is 0.0.2. +on: + workflow_dispatch: + inputs: + version: + description: Upcoming stable agent version (without v; first release 0.0.2) + required: true + type: string + +permissions: + contents: read + +jobs: + publish: + if: github.event_name == 'workflow_dispatch' + runs-on: ubuntu-latest + timeout-minutes: 30 + permissions: + contents: read + packages: write + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + fetch-depth: 0 + persist-credentials: false + + - name: Require the commit to be on the default branch, or refuse + id: default-branch + run: | + set -euo pipefail + # This repository is public. Remote reads need no token; registry + # login below is the only consumer of the job's GITHUB_TOKEN. + authority=$(git ls-remote --symref origin HEAD \ + | awk '$1 == "ref:" && $3 == "HEAD" { sub("refs/heads/", "", $2); print $2 }') + if [ -z "$authority" ]; then + echo "::error::cannot read this repository's default branch from origin; refusing to publish." >&2 + exit 1 + fi + git fetch --quiet --no-tags --force origin \ + "+refs/heads/${authority}:refs/remotes/origin/${authority}" + if ! git merge-base --is-ancestor "$GITHUB_SHA" "refs/remotes/origin/${authority}"; then + echo "::error::${GITHUB_REF_NAME} points at ${GITHUB_SHA}, which is not reachable from refs/remotes/origin/${authority}; refusing to publish." >&2 + exit 1 + fi + + - name: Validate the version and derive the image name + id: inputs + env: + REPOSITORY: ${{ github.repository }} + VERSION: ${{ inputs.version }} + run: | + set -euo pipefail + if [[ ! "$VERSION" =~ ^(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)$ ]]; then + echo "version must be a stable semantic version without a v prefix" >&2 + exit 1 + fi + image=$(printf 'ghcr.io/%s' "$REPOSITORY" | tr '[:upper:]' '[:lower:]') + { + echo "image=$image" + echo "version=$VERSION" + } >> "$GITHUB_OUTPUT" + + # The gateway uses cgo; cross-platform compilation needs an emulator for + # the non-native build stage rather than just a GOARCH override. + - uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 # v3 + - uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1 + - uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0 + id: build + with: + context: . + platforms: linux/amd64,linux/arm64 + labels: | + org.opencontainers.image.version=${{ steps.inputs.outputs.version }} + org.opencontainers.image.source=https://github.com/${{ github.repository }} + org.opencontainers.image.revision=${{ github.sha }} + outputs: type=image,name=${{ steps.inputs.outputs.image }},push-by-digest=true,name-canonical=true,push=true + + - name: Record the digest lock + id: lock + env: + IMAGE: ${{ steps.inputs.outputs.image }} + DIGEST: ${{ steps.build.outputs.digest }} + run: | + set -euo pipefail + if [[ ! "$DIGEST" =~ ^sha256:[0-9a-f]{64}$ ]]; then + echo "build returned no valid lowercase sha256 image-index digest" >&2 + exit 1 + fi + mkdir -p release-evidence + jq -n --arg name "$IMAGE" --arg digest "$DIGEST" '{name: $name, digest: $digest}' > release-evidence/gateway-image.json + { + echo 'Commit the artifact as gateway-image.json in a reviewed PR before publishing the agent:' + echo + echo '```json' + cat release-evidence/gateway-image.json + echo '```' + } >> "$GITHUB_STEP_SUMMARY" + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: gateway-image-lock-${{ steps.inputs.outputs.version }} + path: release-evidence/gateway-image.json + if-no-files-found: error diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..f921fe5 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,181 @@ +name: release + +# Everything a `v*` tag publishes, in dependency order: the tags that resolve +# the gateway container the agent runs, then the GitHub release carrying the +# agent binary. The agent pins the gateway digest gateway-image.json records, so +# a release published without those tags leaves consumers unable to resolve the +# image by the version they asked for. One workflow, so a failure anywhere stops +# what follows. +# +# The image itself is built and pushed by digest before the tag, by +# publish-gateway-image.yml: a digest cannot be recorded in a commit the tag has +# already been cut from. +on: + push: + tags: + - 'v*' + +permissions: + contents: read + +env: + # The repository this runs in, never a typed name (see + # publish-gateway-image.yml). + IMAGE: ghcr.io/${{ github.repository }} + +jobs: + # A tag whose tests fail publishes nothing. go-service-release.yml gates the + # binary on its own test job, but a caller cannot depend on a job inside the + # workflow it calls, so the image tags need this gate of their own — + # otherwise a failing test still leaves an untested image tagged :latest. + # Its own job rather than a step in gateway-image: that job holds packages: + # write and an authenticated ghcr session, and the whole test tree has no + # business running under a registry-write token. + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + - uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5 + with: + go-version-file: go.mod + cache: true + - run: go test ./... + + gateway-image: + needs: test + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3 + - uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Derive version from tag + id: version + run: echo "version=${GITHUB_REF_NAME#v}" >> "$GITHUB_OUTPUT" + # The image tag comes from the git tag, while the agent reports the + # version of the embedded agent.codefly.yaml and resolves its gateway by + # the digest recorded beside it. A tag ahead of the file publishes one + # image and ships an agent that calls itself the previous release, which + # exists, so nothing fails at run time and a consumer pinned to one + # version runs another. + - name: Tag matches the agent version + env: + VERSION: ${{ steps.version.outputs.version }} + run: | + set -euo pipefail + embedded=$(awk '$1 == "version:" {print $2}' agent.codefly.yaml) + if [ "$embedded" != "$VERSION" ]; then + echo "release tag $VERSION does not match agent.codefly.yaml $embedded" >&2 + exit 1 + fi + # The recorded digest is the one the agent runs and reports evidence for, + # so it has to be an image published for this version and not the one left + # over from the previous release — both are digests in the same repository + # and neither fails to resolve. publish-gateway-image.yml stamps the + # version it built for onto every platform of the image, which is what + # tells them apart. A lock that records no digest at all is the tree + # before its first image was published: there is nothing to tag, and + # `jq -e` alone would let an empty string through. + - name: Locked gateway image was published for this version + id: locked + env: + VERSION: ${{ steps.version.outputs.version }} + run: | + set -euo pipefail + name=$(jq -er '.name' gateway-image.json) + digest=$(jq -er '.digest' gateway-image.json) + if [ "$name" != "$IMAGE" ]; then + echo "gateway-image.json names $name, but this workflow publishes $IMAGE" >&2 + exit 1 + fi + if [[ ! "$digest" =~ ^sha256:[0-9a-f]{64}$ ]]; then + echo "gateway-image.json records no sha256 digest (got \"$digest\"): no gateway image has been published;" >&2 + echo "run publish-gateway-image.yml and commit the digest it prints before tagging" >&2 + exit 1 + fi + labelled=$(docker buildx imagetools inspect "$name@$digest" --format '{{json .Image}}' | + jq -ec '[.. | objects | .Labels? // empty | ."org.opencontainers.image.version"? // empty] | unique') + if [ "$labelled" != "$(jq -nc --arg v "$VERSION" '[$v]')" ]; then + echo "gateway-image.json records $digest, built for version(s) $labelled, not $VERSION;" >&2 + echo "run publish-gateway-image.yml and commit the digest it prints before tagging" >&2 + exit 1 + fi + echo "digest=$digest" >> "$GITHUB_OUTPUT" + - uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5 + with: + go-version-file: go.mod + cache: true + - uses: anchore/sbom-action/download-syft@3ad7283483fc7af8ff2b4ea19663c2d5ca935e26 # v0.24.2 + # Evidence for every platform the image ships, generated by the same code + # path the agent's Builder.SBOM uses, so each document is bound to the child + # manifest digest inside the document itself. It scans the recorded digest + # rather than a tag, so the evidence describes exactly the bytes that are + # about to be tagged. + - name: image SBOM evidence + env: + DIGEST: ${{ steps.locked.outputs.digest }} + run: go run ./cmd/image-sbom -image "$IMAGE@$DIGEST" -out sbom + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: gateway-image-sbom-${{ steps.version.outputs.version }} + path: sbom/ + if-no-files-found: error + # Only now does the image become resolvable by the tags consumers use. A + # platform that resolved to no manifest, or a scan that failed, has already + # failed the job above, leaving the digest untagged and unreferenced. + - name: Publish the release tags + env: + VERSION: ${{ steps.version.outputs.version }} + DIGEST: ${{ steps.locked.outputs.digest }} + run: | + set -euo pipefail + docker buildx imagetools create \ + --tag "$IMAGE:$VERSION" \ + --tag "$IMAGE:latest" \ + "$IMAGE@$DIGEST" + for published in "$IMAGE:$VERSION" "$IMAGE:latest"; do + resolved=$(docker buildx imagetools inspect "$published" --format '{{json .Manifest}}' | jq -er '.digest') + if [ "$resolved" != "$DIGEST" ]; then + echo "$published resolves to $resolved, want $DIGEST" >&2 + exit 1 + fi + done + + # The binary ships last: only once the image it points at is resolvable and + # inventoried is the GitHub release that carries the agent worth creating. + release: + needs: gateway-image + # Pinned to an immutable core commit SHA, not @main: this call carries + # secrets: inherit, so a mutable ref would hand an upstream change this + # repo's GH_PAT with no commit here. It is core's v0.18.1, the release go.mod + # requires; bump the two together. This version of the workflow keeps the + # release PAT out of jobs that run unreviewed code and refuses a tag not + # reachable from the default branch. + uses: codefly-dev/core/.github/workflows/go-service-release.yml@ea49e7a968ef5f2fd325b1c41f823a47eb20141c + secrets: inherit + + # Workflow artifacts expire. Copy the digest-bound documents and index onto + # the release after GoReleaser has created it; retries replace the same assets. + attach-image-evidence: + needs: [gateway-image, release] + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + pattern: gateway-image-sbom-* + merge-multiple: true + path: sbom + - name: Attach image SBOMs and index to the GitHub release + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + set -euo pipefail + gh release upload "$GITHUB_REF_NAME" sbom/* --repo "$GITHUB_REPOSITORY" --clobber diff --git a/.gitignore b/.gitignore index 4a6f7b5..5b363d0 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ /service-warehouse /dist/ *.duckdb +/.codefly/ diff --git a/.goreleaser.yaml b/.goreleaser.yaml new file mode 100644 index 0000000..23f41ab --- /dev/null +++ b/.goreleaser.yaml @@ -0,0 +1,41 @@ +version: 2 + +sboms: + - id: archives + artifacts: archive + documents: + - "${artifact}.sbom.json" + disable: false + +project_name: service-warehouse + +# The release asset is the codefly agent binary: the main package at the repo +# root, which is what makes codefly.dev/warehouse resolvable. It is built without +# cgo and must stay so: the gateway (cmd/service-warehouse) is the binary that +# carries the warehouse drivers, and ships as the container image the release +# workflow publishes before this one runs. TestAgentLinksNoBackendDriver holds +# the drivers out of this package. +builds: + - id: darwin + main: . + env: + - CGO_ENABLED=0 + goos: + - darwin + goarch: + - amd64 + - arm64 + - id: linux + main: . + env: + - CGO_ENABLED=0 + goos: + - linux + goarch: + - amd64 + - arm64 + +archives: + - id: default + formats: [tar.gz] + name_template: "{{ .ProjectName }}_{{ .Version }}_{{ .Os }}_{{ .Arch }}" diff --git a/AGENTS.md b/AGENTS.md index 39a1449..1dd1220 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -19,9 +19,24 @@ this file needs a fix in the same PR. tenant, a permission, a product noun, a business rule, or which application is asking. The server is bound to one database and authorizes nothing; the caller's authority is settled before the request arrives. -- **Depends on:** the warehouse drivers and the gRPC contract, nothing else. - The SBOM scanner and the image-SBOM contract live in `codefly-dev/core` and - are not reimplemented here. +- **Depends on:** the warehouse drivers, the gRPC contract, and core's lightweight + `callertoken` package. Caller authentication, image lock parsing, the SBOM + scanner and evidence writing live in `codefly-dev/core` and are not copied here. +- **The agent is a second program in this module.** The repo-root `main` + package (`main.go`, `runtime.go`, `builder.go`) is the codefly agent that + lets a workspace compose the gateway; it links core's agent runtime, which + `cmd/service-warehouse` must not (it uses only core's `callertoken`). The agent + links no backend, so no cgo driver reaches its `CGO_ENABLED=0` release build + (`TestAgentLinksNoBackendDriver`). Like the gateway, it authorizes nothing: it + starts the gateway, hands out the one token it enforces, and renders the + manifest. + +Deployments require an environment-managed `SWH_AUTH_TOKEN` secret reference; +consumers bind the same key to `warehouse.token`. Reject caller-supplied +`SWH_ALLOW_ANONYMOUS` keys from every configuration source, even false/empty +values; only the template may supply its single literal false. The templates render a +same-namespace ingress NetworkPolicy. Anonymous access is only an explicit local +gateway opt-in. README owns the secret binding procedure and policy limits. Three rules follow from that: @@ -58,8 +73,10 @@ preferences; each is a rule an agent broke at real cost. become a fix by working, by being small, by being local, or by the real fix belonging to someone else. 4. **Never hardcode what the system resolves.** `internal/config/config.go` is - the entire configuration surface and every knob is an `SWH_*` variable — a - listen address, a DSN, a credentials path, a query cap. If you are typing a + the entire gateway configuration surface and every knob is an `SWH_*` + variable — a listen address, a DSN, a credentials path, a query cap. (The + agent has one of its own, `SWH_GATEWAY_IMAGE`, a locally built image to run + instead of the pinned one; it never reaches the gateway.) If you are typing a connection string, a port, or a credential into Go source, a test, or a manifest, you are encoding something true only on your machine for the next ten minutes. A test that needs a backend uses `mem`, which needs nothing. @@ -78,10 +95,10 @@ preferences; each is a rule an agent broke at real cost. ## Build and test -Derived from [`.github/workflows/ci.yml`](.github/workflows/ci.yml): those -three commands are the whole gate, so a green local run of all three is the -definition of done. CI pins the Go version as a literal in that workflow -(`go-version: "1.27.0"`) rather than reading `go.mod`, so bumping `go.mod` +Derived from [`.github/workflows/ci.yml`](.github/workflows/ci.yml): the +`build-test` job is those three commands, so a green local run of all three is +the definition of done. CI pins the Go version as a literal in that workflow +(`go-version: "1.27.2"`) rather than reading `go.mod`, so bumping `go.mod` alone moves your toolchain and not CI's — change both. ```sh @@ -90,11 +107,41 @@ go vet ./... go test -race ./... ``` +Two more CI jobs need Docker and are not part of that gate. They build the image +(`docker build -t service-warehouse:e2e .`) and run the agent against it, behind +the `e2e` build tag: + +```sh +SWH_GATEWAY_IMAGE=service-warehouse:e2e go test -v -tags e2e -count=1 -run TestRuntimeEndToEnd . +SWH_GATEWAY_IMAGE=service-warehouse:e2e go test -v -tags e2e -count=1 -run TestImageSBOM . # needs syft on PATH +``` + +**A green e2e run is not evidence that anything ran.** With +`SWH_GATEWAY_IMAGE` unset both tests call `t.Skip`, the package still prints +`ok`, and `go test` exits 0. The SBOM one also skips without `syft`. Read `-v` +output for `--- PASS`, never `ok` alone. + +`codefly agent ci` is the full agent conformance gate. Its workflow +(`agent-ci.yml`) is `workflow_dispatch` only until a full run has been seen green +here. Run `GOTOOLCHAIN=go1.27.2 codefly agent ci` from the repo root with CLI +0.1.185, also pinned in `agent-ci.yml`. The manifest +pins source packager `codefly.dev/go:0.0.69`, whose image recipes use immutable +digests. The gateway build pins Go 1.27.2 by tag and image-index digest and fixes +`GOTOOLCHAIN`; keep `go.mod`, CI and the image toolchain in sync. A fresh service +declares `backend: mem`; Runtime.Test probes health and authenticated Capabilities +even when Start was not called, and returns an explicit PASSED/FAILED run result. +Before the first published lock, set `SWH_GATEWAY_IMAGE` to a local build to +exercise the generated-service test phase; published-image audit still refuses +the absent digest. A passing binary audit is not full conformance. `codefly ci run` operates on a consuming workspace: this agent +repository has no workspace manifest. + Run the server locally against the zero-dependency backend — it needs no -database, no credential, and no network: +database, no warehouse credential, and no network. It still refuses an +unauthenticated listener, so a local run states that choice (or sets +`SWH_AUTH_TOKEN`): ```sh -SWH_BACKEND=mem go run ./cmd/service-warehouse +SWH_BACKEND=mem SWH_ALLOW_ANONYMOUS=true go run ./cmd/service-warehouse ``` The default `SWH_BACKEND=duckdb` refuses to start, by design: the DuckDB @@ -111,6 +158,30 @@ version is pinned anywhere in the repo (codefly-dev/service-warehouse#7). If `buf generate` leaves you with a diff that is only those banner lines, revert it — you changed your toolchain, not the contract. +## Releases + +A `v*` tag publishes the gateway image's tags first and the agent binary second, +from one workflow, in that order. The image itself is published *before* the +tag, because the agent pins its digest and a digest cannot be recorded in a +commit the tag was already cut from; `README.md` ("Releasing") has the nine +first-release steps. The manifest seed is 0.0.1; publish bumps the first release +to v0.0.2, so dispatch the image publisher with 0.0.2. `gateway-image.json` records no digest until the first image exists, and +every consumer of it — the agent, `Deploy`, the release — refuses an empty one +rather than run `:latest`. The tag must match `version:` in `agent.codefly.yaml` +or the job fails by design. + +The coordinator must use an explicit squash body with `Refs #12`; commit +`632feea` contains a closing directive that must not enter the squash message. +Keep the issue open until the release has been verified. + +Core is pinned in two places that move together: the `codefly-dev/core` version +in `go.mod` and the commit SHA of core's reusable workflows in `release.yml` and +`agent-ci.yml` (never `@main`: `release.yml` passes `secrets: inherit`). The image +publisher is an explicitly labelled stopgap pending +[core#770](https://github.com/codefly-dev/core/issues/770), the core 0.18.2 composite +action: its job must stay in this repository so GHCR accepts its own job token. +Do not restore the reusable publisher call; README owns the transition procedure. + ## Where the depth is Keep this file short; depth belongs in the file that owns the subject. @@ -121,9 +192,10 @@ Keep this file short; depth belongs in the file that owns the subject. - `internal/backend/backend.go` — the provider-agnostic interface every backend implements, and the portable type model. - [`cmd/service-warehouse/distribution_test.go`](cmd/service-warehouse/distribution_test.go) - — gates the claim that this repo ships no container image. It fails the day - the repo gains a way to build or deploy one; that is the day image SBOM - coverage is owed, and the gate is scoped to this repo only. + — gates the claim that this repo ships one image, the gateway, and that its + SBOM evidence exists. It lists every path that builds or deploys an image and + fails on one that is not accounted for, so a second image is the day more + image SBOM coverage is owed. The gate is scoped to this repo only. - A repeated multi-step procedure belongs in a `.claude/skills//SKILL.md` of its own, or in a nested `AGENTS.md` next to the code it describes — not in another paragraph here. There are none yet: this repo has three diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..4819382 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,25 @@ +# The warehouse gateway image. A client speaks only the codefly/warehouse/v0 +# gRPC contract; every backend is compiled in and selected at runtime by +# SWH_BACKEND. The image ships no warehouse credentials: those arrive as +# environment at deploy time. +# +# cgo is on, and the runtime image carries glibc and libstdc++ (distroless/cc). +# The embedded DuckDB engine is a cgo driver (internal/backend/duckdb), so a +# CGO_ENABLED=0 build onto distroless/static would keep building until the +# driver lands and then stop. Both stages name Debian 13 explicitly: the binary +# links against the build stage's glibc and needs one at least as new in the +# image it runs in, which "golang:1.27" moving to the next Debian would break. +FROM golang:1.27.2-trixie@sha256:e58d6f83b3416618d8bcac2b3dde1b7f7e3c4a77d25e88637f8bbae81536c48d AS build +ENV GOTOOLCHAIN=go1.27.2 +WORKDIR /src +COPY go.mod go.sum ./ +RUN go mod download +COPY . . +RUN CGO_ENABLED=1 GOOS=linux go build -trimpath -ldflags="-s -w" \ + -o /out/service-warehouse ./cmd/service-warehouse + +FROM gcr.io/distroless/cc-debian13:nonroot +COPY --from=build /out/service-warehouse /service-warehouse +# The gRPC listen port; SWH_LISTEN can override it. +EXPOSE 9465 +ENTRYPOINT ["/service-warehouse"] diff --git a/README.md b/README.md index 211d342..d4daac7 100644 --- a/README.md +++ b/README.md @@ -93,53 +93,218 @@ next milestone is `duckdb` (SQL execution + Arrow encoding), which makes the | `SWH_CREDENTIALS_FILE` | — | BigQuery service-account JSON (else ADC) | | `SWH_MAX_QUERY_BYTES` | `0` | per-query scan cap (0 = backend default) | | `SWH_QUERY_TIMEOUT` | `0` | per-query timeout (Go duration; 0 = backend default) | +| `SWH_AUTH_TOKEN` | — | shared secret every caller must present; required unless `SWH_ALLOW_ANONYMOUS=true` | +| `SWH_ALLOW_ANONYMOUS` | `false` | `true` accepts callers with no token; refused together with `SWH_AUTH_TOKEN` | + +## Authentication + +The server holds the warehouse credentials and runs SQL against the database it +is bound to, and the listen address says nothing about who can reach it. So it +**refuses to start on an unauthenticated listener** unless the operator says +otherwise: `SWH_AUTH_TOKEN` must be set, or `SWH_ALLOW_ANONYMOUS` must be `true` +as an explicit local-development opt-in. The generated deployment requires +authentication and never enables anonymous access. + +With `SWH_AUTH_TOKEN` set, every RPC — unary and streaming, `Capabilities` +included — must carry the secret as the `x-codefly-token` gRPC metadata header, +the key the Codefly host already uses to authenticate agent plugins. Anything +else is rejected with `UNAUTHENTICATED` before it reaches a backend. The token +is compared in constant time and never logged by core's `callertoken` package. +Only `grpc.health.v1.Health/Check` and `/Watch` are exempt; `/List` still needs +the token. These probes report that the process is serving, not that a warehouse +is reachable. An interceptor configured with an empty server token fails closed; +the explicit anonymous mode installs no token interceptor. + +This is authentication only — whether the caller holds the deployment's token. +The server still authorizes nothing: which caller may run which statement is +settled before the request arrives. + +## Running as a codefly service + +A workspace composes this gateway as `codefly.dev/warehouse`, the same way it +composes [`service-object-storage`](https://github.com/codefly-dev/service-object-storage) +as `codefly.dev/object-storage`. The agent is the repo-root `main` package +([`main.go`](main.go), [`runtime.go`](runtime.go), [`builder.go`](builder.go)), +declared by [`agent.codefly.yaml`](agent.codefly.yaml); it is released as a GitHub +release asset a consumer pins by version. The gateway it runs is a container +image, pinned by digest in [`gateway-image.json`](gateway-image.json). + +**What a consumer receives.** The configuration group `warehouse`, with +`connection` (`grpc://host:port`), `endpoint` (`host:port`) and `token`, a +secret value to send as the `x-codefly-token` metadata header. The agent mints a +fresh 64-character lowercase hex token (256 random bits) for every local run +unless an operator pins one, so a run revokes the previous run's. + +**Locally** (`codefly run`) the agent starts the gateway container, publishes its +port, waits until the gateway reports itself serving *and* accepts the agent's +token, and hands the endpoint and token to consumers. That wait attests the +process and the credential, not the warehouse: the gateway does not probe its +backend and no RPC reports that. Set `backend:` in `service.codefly.yaml` to +choose the engine. Newly generated services declare `backend: mem` (the +catalog-only backend, with no query execution). Removing that setting falls back +to the gateway's unsupported `duckdb` default and startup fails. The agent's +`Test` step performs the same health and authenticated `Capabilities` check as +`Start`, so conformance's Init → Test path rejects a dead gateway. It returns an +explicit run-level `PASSED` or `FAILED` result and consistent counts; CLI 0.1.185 +requires that verdict rather than inferring it from legacy status fields. +The agent delivers `SWH_BACKEND`, `SWH_DATABASE`, +`SWH_DATASET` and `SWH_LOCATION`, and refuses `SWH_CREDENTIALS_FILE`, which +nothing could project into the container. `SWH_GATEWAY_IMAGE` makes the agent +run a locally built image instead of the pinned one (`docker build -t +service-warehouse:local .`); it configures the agent, not the gateway. + +**Deployed** the agent renders a Deployment, Service, dedicated ServiceAccount +and NetworkPolicy, with gRPC health probes on the gateway's overall service. +The listener requires a token. Supply `SWH_AUTH_TOKEN` through the deployment's +Kubernetes `secret_references`, referencing a secret provisioned by the +environment's secret manager; plaintext tokens in deployment requests are +refused. Both ephemeral and restricted output profiles use `secretKeyRef`: + +```yaml +secret_references: + SWH_AUTH_TOKEN: + name: warehouse-auth + key: token +``` -## Distribution and SBOM evidence - -This repository **publishes no container image**. It builds to the -`service-warehouse` Go binary; nothing here builds, pushes, or deploys an image, -and the tree carries no Codefly agent manifest and no `Builder` implementation — -so no `Builder.SBOM` RPC is served and none is claimed. +Provision a random 64-character hex token once per environment and bind **the +same secret key** to each consumer's `warehouse.token` secret configuration. +For a consumer that uses Codefly's configuration carriers, its secret-reference +map names `CODEFLY__SERVICE_SECRET_CONFIGURATION______WAREHOUSE__TOKEN` +(the warehouse producer's module and service, uppercased) and points to that +same `warehouse-auth` / `token` key. No token is embedded in an endpoint or +manifest. The agent does not create or rotate deployed secrets, or automatically +copy secret references between services: the environment owns both bindings. +An absent secret prevents pod startup; an absent or wrong consumer token is +rejected before any SQL reaches the backend. + +The NetworkPolicy allows TCP ingress to the gateway port from pods in its own +namespace only. Enforcement requires a CNI that implements NetworkPolicy. This +is a namespace boundary, not a declared-consumer allowlist; same-namespace pods +still need the token. Cross-namespace consumers require an explicit policy +overlay. No service mesh or mTLS is installed by this agent. + +The key `SWH_ALLOW_ANONYMOUS` is reserved in deployments: any caller-supplied +occurrence is refused, including secret references, templated/secret/plain +configuration, raw ConfigMap data and environment overrides, even with an empty +or `false` value. The template owns the single literal `false` entry. Core +v0.18.1 exposes no ConfigMap-reference input; it cannot render a ConfigMap-key +reference supplied through that API. A configured `SWH_CREDENTIALS_FILE` is also refused because nothing mounts that file: grant +the dedicated ServiceAccount a workload identity, or supply a mount and variable +through an external overlay. Other `SWH_*` values are bound onto the container +by name from the environment's service configuration after the render. -Under the fleet image-SBOM contract ([`codefly-dev/core` `docs/sbom.md`][sbom], -released in `v0.3.29`) that is the `NO_IMAGE_REASON_NO_IMAGE` case: a service -that legitimately ships no image, which is a different thing from an agent whose -implementation is missing (`UNSUPPORTED`). A source or lockfile inventory never -counts as image coverage, so the absence of an image is recorded here as a -status — the Go module's dependency list does not satisfy it. +## Distribution and SBOM evidence -`TestRepositoryShipsNoImage` in +This repository publishes **one container image**, the gateway +(`ghcr.io/codefly-dev/service-warehouse`, for `linux/amd64` and `linux/arm64`), +and the agent binary that runs it. The service ships no other image: it has no +init, migration or sidecar image, and the warehouse behind it is not run by this +service. + +Under the fleet image-SBOM contract ([`codefly-dev/core` `docs/sbom.md`][sbom]) +that image owes evidence, and the agent serves it. `Builder.SBOM` at image scope +returns one CycloneDX inventory per shipped platform, covering OS packages and +installed application dependencies, each bound to the image digest and platform +that was scanned and attributed to the service. The scanner is core's +(`sbom.Image`, through `BuilderWrapper.SBOMImages`), and the response is checked +against core's `sbom.ValidateCoverage`. + +- [`internal/gatewayimage`](internal/gatewayimage/image.go) declares the shipped + platforms and role. Core's `sbom.ImageSubjects` derives one subject per declared + platform; local images get their digest and platform from the Docker daemon. +- [`cmd/image-sbom`](cmd/image-sbom/main.go) is a thin entry point to core's + `CollectImageEvidence` and `WriteImageEvidenceIndex`. A release runs it + against the digest [`gateway-image.json`](gateway-image.json) records and + saves them as the `gateway-image-sbom-` workflow artifact before + the image tags are created, then attaches every CycloneDX document and the + index to the GitHub release. Release assets retain the evidence after workflow + artifacts expire; `attach-image-evidence` must finish green before release + verification is complete. +- CI builds the image and checks the same contract against it (`image-sbom`), and + runs the agent against it (`agent-runtime-e2e`). +- `ValidationCapabilities.image_sbom` is **served but not advertised**: core + holds the advertisement back until every consumer that evaluates it runs a core + that can represent the phase, and a present `Validation` is authoritative for + every operation it omits. `TestImageSBOMIsServedButNotAdvertised` marks it. + +`TestEveryImageThisRepositoryShipsIsCovered` in [`cmd/service-warehouse/distribution_test.go`](cmd/service-warehouse/distribution_test.go) -gates that status. It fails when this repository gains a way to **build** an -image (a Dockerfile, an `agent.codefly.yaml`, a compose file, or a workflow -running `docker build`/`docker buildx`/`docker/build-push-action`) *or* a way to -**deploy** one (a Kubernetes manifest, Helm values, or a Kustomize overlay -naming an image). Both matter: evidence binds to the digest actually built or -selected for deployment, so an image built elsewhere and deployed from here owes -coverage just the same. +gates this. It lists every path that builds an image (a Dockerfile, an +`agent.codefly.yaml`, a compose file, a workflow running +`docker build`/`docker buildx`/`docker/build-push-action`) or deploys one (a +Kubernetes manifest, Helm values, a Kustomize overlay naming an image), and fails +on any that is not accounted for, or on one that stopped producing its signal, +or when the tests holding the evidence are gone. A second image therefore has to +arrive with its evidence and with an edit to that list. The gate is scoped to this repository, which is the limit of what it can prove. If another repository ever packages this binary into an image, the service ships -an image that nothing here can see — that case has to be recorded where that +an image that nothing here can see, and that case has to be recorded where that image is built. -Introducing image distribution means this lands **with** the image rather than -after it: - -- a valid CycloneDX SBOM for each final runtime image — OS packages and - installed application dependencies — covering every shipped platform and every - service-owned runtime, init, migration, and sidecar image; -- evidence bound to the immutable digest and platform actually built or - deployed, served at image scope through the shared contract - (`BuilderWrapper.SBOMImages`) and checked by `sbom.ValidateCoverage`; -- the SBOM artifact, checksum, image digest, platform, and service identity - carried into the build/release report and retrievable with the image; -- failures reported as failures — a failed scan, a missing image, a stale digest - or an omitted platform must never read as complete coverage. - -The scanner itself stays in `core`; this repository does not reimplement it. - -[sbom]: https://github.com/codefly-dev/core/blob/v0.3.29/docs/sbom.md +### Releasing + +**Publisher stopgap pending [core#770](https://github.com/codefly-dev/core/issues/770):** +GHCR checks the workflow's defining repository. Core's reusable publisher was +denied `installation not allowed to Write organization package` when publishing +a service package. `publish-gateway-image.yml` therefore defines the job here, +logs in with its own `GITHUB_TOKEN` and `packages: write`, refuses unmerged +commits before login, checks out with `persist-credentials: false`, and pushes +by digest with no tags. The public Git remote reads are anonymous; the branch +check receives no token and puts no credentials in Git's command line. Replace these local publishing steps with core 0.18.2's +composite action once core#770 is available and adopted; keep the job in this +repository. The other core integrations remain pinned to v0.18.1. + +The coordinator performs the following first-release sequence. The manifest's +seed is `0.0.1`; `codefly publish` increments it, so the **first release is +`v0.0.2`** and the image publisher must receive **`0.0.2`**. + +1. Squash the reviewed gateway PR after its CI is green, using an explicit + squash body with `Refs #12`. Commit `632feea` has a closing directive, so the + default concatenated commit-message body is unsafe. The squash message must + omit the closing keyword `closes` before issue `#12`; keep that issue open + through release verification. +2. On clean, synced `main`, run `codefly publish --dry-run` and confirm + `0.0.1` → `0.0.2`. +3. Dispatch `publish-gateway-image` on `main` with `version=0.0.2` (no `v`). + The in-repository publisher pushes by digest and records `gateway-image.json` + in the `gateway-image-lock-0.0.2` artifact and run summary. It uses + `persist-credentials: false` and refuses a commit outside the default branch's + history. +4. Make the new `ghcr.io/codefly-dev/service-warehouse` package **public** in + GitHub package settings. Verify an anonymous pull by the returned digest + using a fresh Docker configuration with no registry login. +5. Commit the returned lock in a reviewed PR. Let CI inventory both declared + platforms (`linux/amd64`, `linux/arm64`), then merge that PR. +6. Run the full `codefly agent ci` on synced `main`, including its audit and + generated-service `codefly ci run`. After it has passed, enable the PR trigger + in `agent-ci.yml`. +7. Confirm the repository's `GH_PAT` release secret is configured, and retain + source packager `codefly.dev/go:0.0.69` and the patched Go 1.27.2 toolchain. +8. From clean, synced `main` with Docker available, run `codefly publish`. + Verify that the tag matches `agent.codefly.yaml`, the locked image's version + labels match `0.0.2`, image tags resolve to the locked digest, and the workflow + attaches the image SBOM documents and index alongside the agent archives. +9. Verify the release and the production loader's archive resolution. Close + [#12](https://github.com/codefly-dev/service-warehouse/issues/12) only then, + with the release URL and the per-row answer below. + +**Per-row error answer at this revision:** `RowError` contains `row_index` and +free-form `string error`; it has **no permanent/retryable code**. Consumers cannot +branch on retryability without parsing text. The separate API work in +[#14](https://github.com/codefly-dev/service-warehouse/pull/14) must land before +claiming otherwise. This agent PR references #12; it does not close it. + +The lock is parsed by core's `resources.ParseImageLock`: it must name +`ghcr.io/codefly-dev/service-warehouse`, contain only `name` and `digest`, and +use `sha256:` followed by 64 lowercase hex characters. An empty digest records +an unpublished image and is refused by the agent and release. CI permits the +initial empty lock only with complete history, no `v*` tag, and no prior nonempty +lock in fetched history. Clearing an established lock fails CI even before the +first tag. + +[sbom]: https://github.com/codefly-dev/core/blob/v0.18.1/docs/sbom.md ## Develop @@ -149,9 +314,32 @@ codefly generate proto --proto ./proto --output ./gen go build ./... go vet ./... -go test ./... # unit tests (mem backend + bufconn server) +go test -race ./... # unit tests (mem backend + bufconn server) and the agent's ``` +The release-gate regression tests use `git` and `jq` (also present in CI). + +The source packager is pinned to `codefly.dev/go:0.0.69`, whose image recipes use +immutable digests. `go.mod` and CI require Go 1.27.2; the gateway Dockerfile pins +the Go 1.27.2 image index and sets `GOTOOLCHAIN=go1.27.2`. For local conformance, +run `GOTOOLCHAIN=go1.27.2 codefly agent ci` with CLI **0.1.185**, as pinned in +`agent-ci.yml`. Before the first image lock, also set +`SWH_GATEWAY_IMAGE=service-warehouse:e2e` to a locally built image when verifying +the generated service's test phase. This override is only for runtime and image +SBOM testing: the audit of the service's published image still fails closed on +the absent lock. A passing binary audit alone is not full conformance. + +The agent's end-to-end tests need Docker and are behind the `e2e` tag; the SBOM +one also needs `syft` on `PATH`: + +```bash +docker build -t service-warehouse:e2e . +SWH_GATEWAY_IMAGE=service-warehouse:e2e go test -v -tags e2e -count=1 . +``` + +With `SWH_GATEWAY_IMAGE` unset they skip, and a skipped run still prints `ok`: +read `-v` for `--- PASS`. + `buf` is never run on the host. `codefly generate proto` runs it inside the versioned proto companion image (`ghcr.io/codefly-dev/proto`), so the plugin versions and the `goimports` pass are the image's, fixed by its tag — two @@ -178,4 +366,11 @@ internal/server/ gRPC Warehouse implementation + proto↔backend ma internal/config/ env configuration internal/serr/ normalized error model cmd/service-warehouse/ the server binary +cmd/image-sbom/ writes the image's SBOM evidence (run by the release) +internal/gatewayimage/ the shipped platforms and runtime role; core derives evidence +main.go runtime.go builder.go the codefly agent (codefly.dev/warehouse) that runs the gateway +agent.codefly.yaml the agent's manifest and version +gateway-image.json the gateway image the agent pins, by digest +Dockerfile the gateway image +templates/ the agent's README, factory files and Kubernetes manifests ``` diff --git a/agent.codefly.yaml b/agent.codefly.yaml new file mode 100644 index 0000000..5545fe6 --- /dev/null +++ b/agent.codefly.yaml @@ -0,0 +1,10 @@ +publisher: codefly.dev +kind: codefly:service +name: warehouse +version: 0.0.1 +source: + directory: . + agent: codefly.dev/go:0.0.69 +release: + owner: workflow + workflow: release.yml diff --git a/builder.go b/builder.go new file mode 100644 index 0000000..d9b4550 --- /dev/null +++ b/builder.go @@ -0,0 +1,340 @@ +package main + +import ( + "context" + "embed" + "fmt" + "regexp" + "strings" + + "github.com/codefly-dev/core/agents/communicate" + "github.com/codefly-dev/core/agents/services" + "github.com/codefly-dev/core/agents/services/sbom" + v0 "github.com/codefly-dev/core/generated/go/codefly/base/v0" + builderv0 "github.com/codefly-dev/core/generated/go/codefly/services/builder/v0" + "github.com/codefly-dev/core/resources" + "github.com/codefly-dev/core/standards" + "github.com/codefly-dev/core/wool" + + "github.com/codefly-dev/service-warehouse/internal/gatewayimage" +) + +type Builder struct { + *services.DefaultBuilder + *Service +} + +// deploymentTemplateParameters carries the values the Kubernetes templates need +// beyond the resolved configuration. An empty value renders no variable at all, +// leaving the choice to the gateway. +type deploymentTemplateParameters struct { + Backend string + Database string + Dataset string + Location string + // ServicePort is the port the Kubernetes Service publishes: the in-cluster + // port core allocated to the gRPC endpoint, the one every consumer is told + // to dial. It forwards to ContainerPort. + ServicePort uint32 + // ContainerPort is the port the gateway listens on inside its container, + // which the template also hands the gateway as SWH_LISTEN. + ContainerPort uint32 +} + +// renderedValue is what a configuration value may look like to be written into a +// manifest. The templates place these inside double-quoted YAML scalars, so a +// quote, a newline or a template action in one would not be a value but a way to +// write the rest of the manifest. Every real value (a backend kind, a BigQuery +// project, a dataset, a region) fits; one that does not is bound onto the +// container by name by the CLI instead of rendered here. +var renderedValue = regexp.MustCompile(`^[A-Za-z0-9._:/@-]*$`) + +func NewBuilder() *Builder { + service := NewService() + return &Builder{ + DefaultBuilder: services.NewDefaultBuilder(service.Builder), + Service: service, + } +} + +func (s *Builder) Load(ctx context.Context, req *builderv0.LoadRequest) (*builderv0.LoadResponse, error) { + defer s.Wool.Catch() + return s.Builder.LoadService(ctx, req, services.BuilderLoad{ + Settings: s.Settings, + Requirements: requirements, + FactoryTemplates: factoryFS, + ResolveEndpoints: func(ctx context.Context, endpoints []*v0.Endpoint) error { + endpoint, err := resolveServingGRPCEndpoint(ctx, endpoints) + if err != nil { + return err + } + s.GrpcEndpoint = endpoint + return nil + }, + }) +} + +func (s *Builder) Audit(ctx context.Context, req *builderv0.AuditRequest) (*builderv0.AuditResponse, error) { + defer s.Wool.Catch() + ctx = s.Wool.Inject(ctx) + image, err := publishedGatewayImage() + if err != nil { + return s.Builder.AuditError(err) + } + return s.Builder.AuditContainer(ctx, req, image.FullName()) +} + +// SBOM answers a source-scope request with the package inventory this agent has +// always produced, and an image-scope request with evidence bound to the digest +// and platform of every image the service ships. A caller that resolved the +// images itself passes them as subjects; an empty subject list asks the agent to +// enumerate its own. +func (s *Builder) SBOM(ctx context.Context, req *builderv0.SBOMRequest) (*builderv0.SBOMResponse, error) { + defer s.Wool.Catch() + ctx = s.Wool.Inject(ctx) + if req.GetScope() != builderv0.SBOMScope_SBOM_SCOPE_IMAGE { + image, err := publishedGatewayImage() + if err != nil { + return s.Builder.SBOMError(err) + } + return s.Builder.SBOMContainer(ctx, image.FullName()) + } + if subjects := req.GetSubjects(); len(subjects) > 0 { + return s.Builder.SBOMImages(ctx, subjects) + } + subjects, err := s.imageSubjects(ctx) + if err != nil { + return s.Builder.SBOMImageError(err) + } + return s.Builder.SBOMImages(ctx, subjects) +} + +// imageSubjects enumerates the images this service ships, and the way to reach +// them. The gateway is the only one: the service has no init, migration or +// sidecar image, and the warehouse behind it is not run by this service. +// +// Core derives the subjects from the same gateway metadata the release command +// uses, so release evidence describes the images the agent reports. +func (s *Builder) imageSubjects(ctx context.Context) ([]*builderv0.ImageSubject, error) { + image, overridden, err := effectiveGatewayImage() + if err != nil { + return nil, err + } + published := sbom.PublishedImage{Service: s.Base.Unique(), Role: gatewayimage.Role, Reference: image.FullName()} + if !overridden { + published.Platforms = gatewayimage.Platforms + } + return sbom.ImageSubjects(ctx, published, overridden) +} + +func (s *Builder) Deploy(ctx context.Context, req *builderv0.DeploymentRequest) (*builderv0.DeploymentResponse, error) { + defer s.Wool.Catch() + ctx = s.Wool.Inject(ctx) + image, err := publishedGatewayImage() + if err != nil { + return s.Builder.DeployError(err) + } + s.Base.SetDockerImage(image) + + // Resolve the effective configuration from the deployment request; without + // this the template only ever sees the zero-value config. + if err := s.LoadConfiguration(ctx, req.GetConfiguration()); err != nil { + return s.Builder.DeployError(err) + } + + if err := s.rejectAnonymousDeploymentConfiguration(req); err != nil { + return s.Builder.DeployError(err) + } + + parameters := &deploymentTemplateParameters{ + Backend: s.conf.backend, + Database: s.conf.database, + Dataset: s.conf.dataset, + Location: s.conf.location, + ContainerPort: gatewayContainerPort, + } + servicePort, err := s.servicePort(ctx, req.GetNetworkMappings()) + if err != nil { + return s.Builder.DeployError(err) + } + parameters.ServicePort = servicePort + + // A backend kind is not validated against a list here: the gateway owns the + // set of backends it compiles in and refuses an unknown one at startup, + // naming the registered kinds, and a second list in the agent would drift + // from it. What this layer owns is the manifest, so it checks that every + // value is one the manifest can carry. + for _, value := range []struct{ name, value string }{ + {"SWH_BACKEND", parameters.Backend}, + {"SWH_DATABASE", parameters.Database}, + {"SWH_DATASET", parameters.Dataset}, + {"SWH_LOCATION", parameters.Location}, + } { + if !renderedValue.MatchString(value.value) { + return s.Builder.DeployError(fmt.Errorf("%s %q cannot be rendered into the manifest: "+ + "it may hold only letters, digits and . _ : / @ -; bind a value outside that set onto the container "+ + "through the environment's service configuration instead", value.name, value.value)) + } + } + // A credentials-file path cannot be honored by this deployment: nothing + // projects the file into the pod, and the configuration channel carries the + // path, never its contents. Emitting the variable anyway names a path that + // does not exist, and the gateway then fails on its first query after + // passing every probe. Keyless auth is the remedy where the platform has it + // (a workload identity bound to the pod's ServiceAccount); elsewhere the + // file has to be mounted by an overlay this agent does not own. Either way, + // reject the path rather than render a manifest that lies about it. + if s.conf.credentialsFile != "" { + return s.Builder.DeployError(fmt.Errorf("SWH_CREDENTIALS_FILE cannot be delivered by this deployment: "+ + "nothing here mounts %s into the pod, so the gateway would name a path that does not exist; "+ + "unset it and bind the workload's Kubernetes ServiceAccount to a cloud identity (workload identity), "+ + "or mount the file from an overlay of your own and patch SWH_CREDENTIALS_FILE onto the container there", + s.conf.credentialsFile)) + } + // Deployment credentials are provided by the environment's secret store, + // never embedded in manifests. Consumers bind the same key to warehouse.token. + if s.conf.authToken != "" { + return s.Builder.DeployError(fmt.Errorf("SWH_AUTH_TOKEN must be delivered as a Kubernetes secret reference, not a literal configuration value")) + } + ref := req.GetDeployment().GetKubernetes().GetSecretReferences()["SWH_AUTH_TOKEN"] + if ref.GetName() == "" || ref.GetKey() == "" { + return s.Builder.DeployError(fmt.Errorf("deployment requires an SWH_AUTH_TOKEN Kubernetes secret reference; bind the same secret key to consumers' warehouse.token configuration")) + } + return s.Builder.DeployKustomize(ctx, req, services.KustomizeDeployment{ + EnvironmentVariables: s.EnvironmentVariables, + Templates: deploymentFS, + Parameters: parameters, + PodOverlay: workloadPodOverlay(), + }) +} + +// rejectAnonymousDeploymentConfiguration reserves this key for the template's +// literal false. Reject its presence, regardless of value, group or delivery +// mechanism: a reference's value cannot be known at render time. Core v0.18.1 +// exposes secret references but no ConfigMap-reference field; raw configuration +// and environment-manager values cover its ConfigMap/Secret data paths. +func (s *Builder) rejectAnonymousDeploymentConfiguration(req *builderv0.DeploymentRequest) error { + reject := func(key string) error { + if key == "SWH_ALLOW_ANONYMOUS" || strings.HasSuffix(key, "__SWH_ALLOW_ANONYMOUS") { + return fmt.Errorf("SWH_ALLOW_ANONYMOUS is only an explicit local gateway opt-in; the key is reserved in deployments, including values and references") + } + return nil + } + for key := range req.GetDeployment().GetKubernetes().GetSecretReferences() { + if err := reject(key); err != nil { + return err + } + } + configurations := append([]*v0.Configuration{req.GetConfiguration()}, req.GetDependenciesConfigurations()...) + for _, configuration := range configurations { + for _, info := range configuration.GetInfos() { + for _, value := range info.GetConfigurationValues() { + if err := reject(value.GetKey()); err != nil { + return err + } + } + } + } + values, err := s.EnvironmentVariables.All() + if err != nil { + return err + } + for _, value := range values { + if err := reject(value.Key); err != nil { + return err + } + } + return nil +} + +// servicePort is the port the rendered Service publishes for the gRPC endpoint: +// the in-cluster port core allocated to it, which is what every consumer is +// given to dial. With no container mapping for this endpoint in the request (a +// render with no network input) it keeps the container port, so that render is +// unchanged. +func (s *Builder) servicePort(ctx context.Context, mappings []*v0.NetworkMapping) (uint32, error) { + if s.GrpcEndpoint != nil { + for _, mapping := range mappings { + endpoint := mapping.GetEndpoint() + if endpoint.GetApi() != standards.GRPC || + endpoint.GetModule() != s.GrpcEndpoint.GetModule() || + endpoint.GetService() != s.GrpcEndpoint.GetService() || + endpoint.GetName() != s.GrpcEndpoint.GetName() { + continue + } + // An external endpoint is reached through its DNS entry, never this + // Service, and core gives it no container view. + if resources.IsExternalEndpoint(endpoint) { + continue + } + instance, err := resources.FindNetworkInstanceInNetworkMappings(ctx, mappings, endpoint, resources.NewContainerNetworkAccess()) + if err != nil { + return 0, err + } + return instance.GetPort(), nil + } + } + return gatewayContainerPort, nil +} + +// workloadPodOverlay gives the gateway its own Kubernetes ServiceAccount, named +// after the service (core defaults an unnamed account to the service's DNS +// name) and rendered in the namespace the deployment targets. Running under the +// namespace default would make the gateway's cloud identity the one every other +// workload in that namespace shares: keyless warehouses (workload identity on +// GKE, and the AKS/EKS equivalents) grant on the Kubernetes principal, so a +// grant to `default` is a grant to all of them. With a dedicated account the +// warehouse's IAM can name exactly this workload. +// +// The account carries no annotations: a direct workload-identity grant needs +// none. An environment that declares a runtime identity for this service has +// the CLI project it onto this same account (same name), merging annotations. +func workloadPodOverlay() *services.PodTemplateOverlay { + return &services.PodTemplateOverlay{ServiceAccount: &services.WorkloadServiceAccount{}} +} + +func (s *Builder) Create(ctx context.Context, req *builderv0.CreateRequest) (*builderv0.CreateResponse, error) { + defer s.Wool.Catch() + if s.Settings.Backend == "" { + s.Settings.Backend = "mem" + } + if err := s.Templates(ctx, s.Information, services.WithFactory(factoryFS)); err != nil { + return s.Builder.CreateError(err) + } + if err := s.CreateEndpoints(ctx); err != nil { + return s.Builder.CreateErrorf(err, "cannot create endpoints") + } + return s.Builder.CreateResponse(ctx, s.Settings) +} + +func (s *Builder) CreateEndpoints(ctx context.Context) error { + grpc, err := resources.LoadGrpcAPI(ctx, nil) + if err != nil { + return s.Wool.Wrapf(err, "cannot load grpc api") + } + endpoint := s.Base.BaseEndpoint(standards.GRPC) + // Reachable by the other modules of the workspace that depend on it. Core's + // reach vocabulary is private, internal and public: the cross-module reach + // service-object-storage declares as "external" is "internal" here, and + // "external" is now a location, which this service's endpoint is not. + endpoint.Visibility = resources.VisibilityInternal + s.GrpcEndpoint, err = resources.NewAPI(ctx, endpoint, resources.ToGrpcAPI(grpc)) + if err != nil { + return s.Wool.Wrapf(err, "cannot create grpc endpoint") + } + s.Endpoints = []*v0.Endpoint{s.GrpcEndpoint} + s.Wool.Debug("created endpoints", wool.Field("endpoints", resources.MakeManyEndpointSummary(s.Endpoints))) + return nil +} + +func (s *Builder) Communicate(stream builderv0.Builder_CommunicateServer) error { + asker := communicate.NewQuestionAsker(stream) + _, err := asker.RunSequence(nil) + return err +} + +//go:embed templates/factory +var factoryFS embed.FS + +//go:embed templates/deployment +var deploymentFS embed.FS diff --git a/cmd/image-sbom/main.go b/cmd/image-sbom/main.go new file mode 100644 index 0000000..fccf60c --- /dev/null +++ b/cmd/image-sbom/main.go @@ -0,0 +1,77 @@ +// Command image-sbom writes CycloneDX evidence for the images this service +// ships, through the same shared scanner Builder.SBOM uses. +// +// The release pipeline runs it so published evidence comes from the agent's own +// code path. Generating it with a separate scanner invocation produced +// documents that were not bound to the image digest and that drifted from the +// contract the agent implements. Run it from the repository root: the service +// identity it records is read from agent.codefly.yaml there. +package main + +import ( + "context" + "flag" + "fmt" + "os" + + "github.com/codefly-dev/core/agents/services/sbom" + "github.com/codefly-dev/core/resources" + "github.com/codefly-dev/core/shared" + + "github.com/codefly-dev/service-warehouse/internal/gatewayimage" +) + +func main() { + image := flag.String("image", "", "image reference to inventory") + out := flag.String("out", "sbom", "directory to write CycloneDX documents into") + local := flag.Bool("local", false, "inventory an image held by the local Docker daemon rather than a registry image") + service := flag.String("service", "", "service identity recorded on each subject (default: the agent's publisher/name from agent.codefly.yaml)") + flag.Parse() + + if *image == "" { + fmt.Fprintln(os.Stderr, "image-sbom: -image is required") + os.Exit(2) + } + if *service == "" { + identity, err := agentIdentity() + if err != nil { + fmt.Fprintf(os.Stderr, "image-sbom: %v\n", err) + os.Exit(1) + } + *service = identity + } + + ctx := context.Background() + published := sbom.PublishedImage{Service: *service, Role: gatewayimage.Role, Reference: *image} + if !*local { + published.Platforms = gatewayimage.Platforms + } + subjects, err := sbom.ImageSubjects(ctx, published, *local) + if err != nil { + fmt.Fprintf(os.Stderr, "image-sbom: %v\n", err) + os.Exit(1) + } + documents, err := sbom.CollectImageEvidence(ctx, *out, subjects) + if err != nil { + fmt.Fprintf(os.Stderr, "image-sbom: %v\n", err) + os.Exit(1) + } + if err := sbom.WriteImageEvidenceIndex(*out, documents); err != nil { + fmt.Fprintf(os.Stderr, "image-sbom: %v\n", err) + os.Exit(1) + } + for _, document := range documents { + fmt.Printf("%s %s %s\n", document.Platform, document.Digest, document.Path) + } +} + +// agentIdentity reads the identity a release has no workspace to take from: the +// publisher and name the manifest declares, which is also what a workspace +// resolves the agent by. +func agentIdentity() (string, error) { + agent, err := resources.LoadFromFs[resources.Agent](shared.Embed(os.DirFS("."))) + if err != nil { + return "", fmt.Errorf("read agent.codefly.yaml (run from the repository root, or pass -service): %w", err) + } + return agent.Publisher + "/" + agent.Name, nil +} diff --git a/cmd/service-warehouse/auth_test.go b/cmd/service-warehouse/auth_test.go new file mode 100644 index 0000000..937d4a4 --- /dev/null +++ b/cmd/service-warehouse/auth_test.go @@ -0,0 +1,247 @@ +package main + +import ( + "context" + "errors" + "io" + "net" + "strings" + "testing" + "time" + + "github.com/codefly-dev/core/callertoken" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials/insecure" + healthv1 "google.golang.org/grpc/health/grpc_health_v1" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" + "google.golang.org/grpc/test/bufconn" + "google.golang.org/protobuf/types/known/emptypb" + + whv0 "github.com/codefly-dev/service-warehouse/gen/codefly/warehouse/v0" + "github.com/codefly-dev/service-warehouse/internal/backend" + "github.com/codefly-dev/service-warehouse/internal/backend/mem" +) + +// serverToken is the secret the test gateway enforces. +const serverToken = "89d0d1f6d0e2f5a4c3b2a1908f7e6d5c" + +// startGateway serves the server run() serves — built by the same newGRPCServer +// — over an in-memory listener and returns it with a way to dial it. +func startGateway(t *testing.T, token string) (*grpc.Server, func(...grpc.DialOption) *grpc.ClientConn) { + t.Helper() + lis := bufconn.Listen(1 << 20) + srv := newGRPCServer(mem.New(backend.Config{}), token) + go func() { _ = srv.Serve(lis) }() + t.Cleanup(srv.Stop) + + return srv, func(opts ...grpc.DialOption) *grpc.ClientConn { + t.Helper() + conn, err := grpc.NewClient("passthrough:///bufnet", append(opts, + grpc.WithContextDialer(func(context.Context, string) (net.Conn, error) { return lis.Dial() }), + grpc.WithTransportCredentials(insecure.NewCredentials()), + )...) + require.NoError(t, err) + t.Cleanup(func() { _ = conn.Close() }) + return conn + } +} + +// rpc is one method a peer could call, as the server itself registered it. +type rpc struct { + name string + call func(context.Context, *grpc.ClientConn) error +} + +// registeredRPCs lists every method of every service the server registers +// except health Check/Watch, read from the server rather than from a list kept +// beside it: an RPC added to the proto, or a service added to the server, is +// covered by the refusal tests the moment it exists, without anyone remembering +// to extend them. +// +// Each call sends an empty message, which every request type decodes, so the +// outcome is decided by the interceptor alone. A call that gets past it ends in +// whatever status its handler gives an empty request; that is not asserted here. +func registeredRPCs(t *testing.T, srv *grpc.Server) []rpc { + t.Helper() + var rpcs []rpc + for service, info := range srv.GetServiceInfo() { + for _, m := range info.Methods { + full := "/" + service + "/" + m.Name + if callertoken.IsHealthMethod(full) { + continue + } + rpcs = append(rpcs, rpc{name: full, call: caller(full, m)}) + } + } + require.NotEmpty(t, rpcs) + return rpcs +} + +func caller(full string, m grpc.MethodInfo) func(context.Context, *grpc.ClientConn) error { + if !m.IsClientStream && !m.IsServerStream { + return func(ctx context.Context, conn *grpc.ClientConn) error { + return conn.Invoke(ctx, full, &emptypb.Empty{}, &emptypb.Empty{}) + } + } + return func(ctx context.Context, conn *grpc.ClientConn) error { + st, err := conn.NewStream(ctx, &grpc.StreamDesc{ + ServerStreams: m.IsServerStream, + ClientStreams: m.IsClientStream, + }, full) + if err != nil { + return err + } + // A refused stream makes SendMsg fail with io.EOF; RecvMsg then carries + // the real status. + if err := st.SendMsg(&emptypb.Empty{}); err != nil && !errors.Is(err, io.EOF) { + return err + } + if err := st.CloseSend(); err != nil { + return err + } + for { + if err := st.RecvMsg(&emptypb.Empty{}); err != nil { + if errors.Is(err, io.EOF) { + return nil + } + return err + } + } + } +} + +// TestGatewayCoversTheWholeContract pins what the two tests below enumerate: the +// Warehouse service as the proto declares it, the streaming methods included. +// Without it, those tests would still pass if the enumeration quietly saw less +// than the server serves. +func TestGatewayCoversTheWholeContract(t *testing.T) { + srv, _ := startGateway(t, serverToken) + + prefix := "/" + whv0.Warehouse_ServiceDesc.ServiceName + "/" + declared := len(whv0.Warehouse_ServiceDesc.Methods) + len(whv0.Warehouse_ServiceDesc.Streams) + var covered int + for _, r := range registeredRPCs(t, srv) { + if strings.HasPrefix(r.name, prefix) { + covered++ + } + } + require.Equal(t, declared, covered) + require.NotEmpty(t, whv0.Warehouse_ServiceDesc.Streams, "the contract has streaming methods, which take the stream interceptor") +} + +// TestUnauthenticatedPeerIsRefused is the core guarantee: a peer that reaches +// the port without the deployment's token can run nothing — over unary and +// streaming RPCs alike. +func TestUnauthenticatedPeerIsRefused(t *testing.T) { + srv, dial := startGateway(t, serverToken) + + peers := map[string]*grpc.ClientConn{ + "no credentials": dial(), + "wrong token": dial(callertoken.DialOption("00000000000000000000000000000000")), + "prefix of token": dial(callertoken.DialOption(serverToken[:len(serverToken)-1])), + "token plus extra": dial(callertoken.DialOption(serverToken + "x")), + "empty token": dial(callertoken.DialOption("")), + } + + for peerName, conn := range peers { + for _, r := range registeredRPCs(t, srv) { + t.Run(peerName+"/"+r.name, func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + err := r.call(ctx, conn) + require.Error(t, err) + require.Equal(t, codes.Unauthenticated, status.Code(err), "got %v", err) + }) + } + } +} + +// TestAuthorizedPeerReachesTheHandler pairs with the refusal test: the same +// calls must get through once the token is presented, or the gateway would be +// locked shut rather than protected. A handler-level status (InvalidArgument, +// Unimplemented) still proves the interceptor let the call through. +func TestAuthorizedPeerReachesTheHandler(t *testing.T) { + srv, dial := startGateway(t, serverToken) + conn := dial(callertoken.DialOption(serverToken)) + + for _, r := range registeredRPCs(t, srv) { + t.Run(r.name, func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + err := r.call(ctx, conn) + require.NotEqual(t, codes.Unauthenticated, status.Code(err), "got %v", err) + }) + } +} + +// TestTokenTravelsUnderTheEcosystemKey pins the wire contract: the gateway reads +// the same metadata key the codefly host uses for agent plugins, so a caller +// built against that convention interoperates. +func TestTokenTravelsUnderTheEcosystemKey(t *testing.T) { + require.Equal(t, "x-codefly-token", callertoken.MetadataKey) + + _, dial := startGateway(t, serverToken) + conn := dial() + ctx := metadata.AppendToOutgoingContext(context.Background(), "x-codefly-token", serverToken) + ctx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + _, err := whv0.NewWarehouseClient(conn).Capabilities(ctx, &whv0.CapabilitiesRequest{}) + require.NoError(t, err) +} + +// TestTokenUnderAnotherHeaderIsRefused guards against a caller that presents the +// right secret the wrong way: an HTTP-style Authorization bearer is not the +// gateway's credential and must not be accepted. +func TestTokenUnderAnotherHeaderIsRefused(t *testing.T) { + _, dial := startGateway(t, serverToken) + ctx := metadata.AppendToOutgoingContext(context.Background(), "authorization", "Bearer "+serverToken) + ctx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + _, err := whv0.NewWarehouseClient(dial()).Capabilities(ctx, &whv0.CapabilitiesRequest{}) + require.Equal(t, codes.Unauthenticated, status.Code(err), "got %v", err) +} + +// TestHealthNeedsNoToken pins the one exemption: the standard gRPC health +// Check and Watch answer a peer that presents no token. A readiness probe — the Codefly +// CLI's, a load balancer's, a mesh's — carries none, and a guarded health service +// makes the gateway look permanently unready to all of them. Every data RPC stays +// refused; TestUnauthenticatedPeerIsRefused holds that side. +func TestHealthNeedsNoToken(t *testing.T) { + _, dial := startGateway(t, serverToken) + health := healthv1.NewHealthClient(dial()) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + resp, err := health.Check(ctx, &healthv1.HealthCheckRequest{}) + require.NoError(t, err) + require.Equal(t, healthv1.HealthCheckResponse_SERVING, resp.GetStatus()) + + stream, err := health.Watch(ctx, &healthv1.HealthCheckRequest{}) + require.NoError(t, err) + first, err := stream.Recv() + require.NoError(t, err) + require.Equal(t, healthv1.HealthCheckResponse_SERVING, first.GetStatus()) + + _, err = health.List(ctx, &healthv1.HealthListRequest{}) + require.Equal(t, codes.Unauthenticated, status.Code(err), "List enumerates services and is not exempt") + _, err = healthv1.NewHealthClient(dial(callertoken.DialOption(serverToken))).List(ctx, &healthv1.HealthListRequest{}) + require.NoError(t, err) +} + +// TestWithoutATokenTheListenerIsAnonymous is the other side of the config guard: +// config.FromEnv resolves an empty token only when the operator chose +// SWH_ALLOW_ANONYMOUS, and then the server must take callers with no credential +// rather than refuse everyone. +func TestWithoutATokenTheListenerIsAnonymous(t *testing.T) { + _, dial := startGateway(t, "") + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + _, err := whv0.NewWarehouseClient(dial()).Capabilities(ctx, &whv0.CapabilitiesRequest{}) + require.NoError(t, err) +} diff --git a/cmd/service-warehouse/distribution_test.go b/cmd/service-warehouse/distribution_test.go index 1876dc2..a6b8320 100644 --- a/cmd/service-warehouse/distribution_test.go +++ b/cmd/service-warehouse/distribution_test.go @@ -24,6 +24,7 @@ var imageBuildCommands = []string{ "docker build", "docker buildx", "docker/build-push-action", + "codefly-dev/core/.github/workflows/publish-service-image.yml@", } // manifestExtensions can name an image this service deploys. Evidence binds to @@ -121,23 +122,93 @@ func contentSignal(path, rel string) (string, error) { return "", nil } -// TestRepositoryShipsNoImage is the gate behind the documented no-image status. -// The moment this repository builds, publishes, or deploys an image, the status -// is false and image-scope SBOM evidence has to land with the image. -func TestRepositoryShipsNoImage(t *testing.T) { +// shippedImage is every path in this repository that builds, publishes, or +// deploys an image, with what imageSignals says about it. The service ships one +// image, the gateway, so these are the pieces of that one image: the recipe that +// builds it, the workflows that build, tag and test it, the manifest of the +// agent that runs it, and the template that deploys it. +// +// A path not listed is an image this repository ships without evidence for it, +// and a second Dockerfile, a compose file or another workflow building an image +// lands here first. The list is exact in both directions: a listed path that no +// longer produces its signal means the image changed shape, and the evidence has +// to be re-checked against it. +var shippedImage = []string{ + ".github/workflows/ci.yml: workflow runs docker build", + ".github/workflows/publish-gateway-image.yml: workflow runs docker/build-push-action", + ".github/workflows/release.yml: workflow runs docker build", + "Dockerfile: container image build definition", + "agent.codefly.yaml: Codefly agent manifest, whose Builder ships image build recipes", + "templates/deployment/kustomize/base/deployment.yaml.tmpl: manifest names a container image to deploy", +} + +// imageCoverage are the tests, in the agent's package at the repository root, +// that hold the gateway image's SBOM evidence: one subject per shipped platform, +// the evidence validated against the shared contract, and the platforms the +// pipeline builds held to the platforms the evidence covers. They are looked up +// by name so that deleting one fails here, where the image is accounted for. +var imageCoverage = []string{ + "TestImageSubjectsCoverEveryPublishedPlatform", + "TestImageEvidenceSatisfiesTheCoverageContract", + "TestShippedPlatformsMatchTheReleasePipeline", +} + +// TestEveryImageThisRepositoryShipsIsCovered is the gate behind the documented +// distribution status. This repository builds, publishes and deploys one +// container image, the gateway, so image-scope SBOM evidence is owed and is +// served by the agent's Builder.SBOM (codefly-dev/core docs/sbom.md: +// BuilderWrapper.SBOMImages, checked by sbom.ValidateCoverage). What this holds +// is that nothing else joins it without that evidence: any image-producing path +// outside shippedImage fails, and the tests that hold the evidence have to exist. +func TestEveryImageThisRepositoryShipsIsCovered(t *testing.T) { found, err := imageSignals(repoRoot) if err != nil { t.Fatal(err) } + listed := map[string]bool{} + for _, signal := range shippedImage { + listed[signal] = true + } + seen := map[string]bool{} for _, signal := range found { - t.Errorf("image-producing path found: %s", signal) + seen[signal] = true + if !listed[signal] { + t.Errorf("image-producing path not accounted for: %s", signal) + } } - if len(found) > 0 { - t.Log("service-warehouse documents no image, so nothing here publishes image SBOM evidence. " + - "Shipping or deploying an image requires a CycloneDX SBOM per final image — OS packages " + - "and application dependencies — bound to the digest and platform actually shipped, served " + - "through the fleet contract (codefly-dev/core docs/sbom.md: BuilderWrapper.SBOMImages, " + - "checked by sbom.ValidateCoverage), then README.md and this gate updated to match.") + for _, signal := range shippedImage { + if !seen[signal] { + t.Errorf("accounted-for image path no longer produces its signal: %s", signal) + } + } + if t.Failed() { + t.Log("service-warehouse ships one image, the gateway. A new way to build or deploy an image " + + "requires a CycloneDX SBOM per final image — OS packages and application dependencies — bound " + + "to the digest and platform actually shipped, served through the fleet contract (codefly-dev/core " + + "docs/sbom.md: BuilderWrapper.SBOMImages, checked by sbom.ValidateCoverage), then README.md and " + + "this list updated to match.") + } + + covered := map[string]bool{} + tests, err := filepath.Glob(filepath.Join(repoRoot, "*_test.go")) + if err != nil { + t.Fatal(err) + } + for _, name := range tests { + content, err := os.ReadFile(name) + if err != nil { + t.Fatal(err) + } + for _, test := range imageCoverage { + if bytes.Contains(content, []byte("func "+test+"(")) { + covered[test] = true + } + } + } + for _, test := range imageCoverage { + if !covered[test] { + t.Errorf("the image's SBOM coverage test %s is gone from the agent package", test) + } } } @@ -167,7 +238,7 @@ func TestImageSignalsFindsAWorkflowImageBuild(t *testing.T) { // the production list cannot fail when a command is dropped from it, which // would silently narrow the gate. commands := []string{"docker build", "docker buildx", "docker/build-push-action"} - if len(commands) != len(imageBuildCommands) { + if len(commands)+1 != len(imageBuildCommands) { t.Fatalf("imageBuildCommands is %v; update this test to match", imageBuildCommands) } for _, command := range commands { @@ -181,6 +252,14 @@ func TestImageSignalsFindsAWorkflowImageBuild(t *testing.T) { } } +// Shared image publishers are image-producing paths even without local steps. +func TestImageSignalsFindsTheCoreImagePublisher(t *testing.T) { + root := t.TempDir() + path := filepath.Join(".github", "workflows", "publish.yml") + write(t, root, path, "jobs:\n publish:\n uses: codefly-dev/core/.github/workflows/publish-service-image.yml@revision\n") + requireSignal(t, root, path) +} + // TestImageSignalsFindsADeployedImage covers an image this repository deploys // without building: the evidence contract binds to the digest built *or // selected for deployment*, so a manifest naming an image is image coverage diff --git a/cmd/service-warehouse/main.go b/cmd/service-warehouse/main.go index 4743980..76abd0f 100644 --- a/cmd/service-warehouse/main.go +++ b/cmd/service-warehouse/main.go @@ -14,7 +14,10 @@ import ( "syscall" "time" + "github.com/codefly-dev/core/callertoken" "google.golang.org/grpc" + grpchealth "google.golang.org/grpc/health" + healthv1 "google.golang.org/grpc/health/grpc_health_v1" whv0 "github.com/codefly-dev/service-warehouse/gen/codefly/warehouse/v0" "github.com/codefly-dev/service-warehouse/internal/backend" @@ -55,8 +58,7 @@ func run() error { return err } - grpcServer := grpc.NewServer() - whv0.RegisterWarehouseServer(grpcServer, server.New(be)) + grpcServer := newGRPCServer(be, cfg.AuthToken) sig := make(chan os.Signal, 1) signal.Notify(sig, os.Interrupt, syscall.SIGTERM) @@ -66,11 +68,41 @@ func run() error { gracefulStop(grpcServer, shutdownGrace) }() - log.Printf("warehouse gateway listening on %s (backend=%s database=%s)", - cfg.ListenAddr, cfg.Backend.Kind, cfg.Backend.Database) + // Config resolution refuses an empty token unless the operator accepted an + // unauthenticated listener, so the empty case here is that choice. The token + // itself is never logged. + authMode := "none" + if cfg.AuthToken != "" { + authMode = "token" + } + log.Printf("warehouse gateway listening on %s (backend=%s database=%s auth=%s)", + cfg.ListenAddr, cfg.Backend.Kind, cfg.Backend.Database, authMode) return grpcServer.Serve(lis) } +// newGRPCServer assembles the gateway's gRPC server: the Warehouse service over +// be, the standard health service, and — when token is set — the interceptors +// that refuse any call that does not present it. The health service is exempt +// from them (only Check and Watch) so a readiness probe, which carries +// no token, can still reach it. +// +// The health service reports the process as serving and nothing more. It is +// liveness, not a verdict on the backend: this server does not probe its +// warehouse, so it claims nothing about whether one is reachable. +func newGRPCServer(be backend.Backend, token string) *grpc.Server { + var options []grpc.ServerOption + if token != "" { + options = append(options, + grpc.ChainUnaryInterceptor(callertoken.UnaryServerInterceptor(token)), + grpc.ChainStreamInterceptor(callertoken.StreamServerInterceptor(token)), + ) + } + srv := grpc.NewServer(options...) + healthv1.RegisterHealthServer(srv, grpchealth.NewServer()) + whv0.RegisterWarehouseServer(srv, server.New(be)) + return srv +} + // gracefulStop drains in-flight RPCs, but escalates to a hard Stop if they do // not finish within grace, so a stuck stream can never make shutdown hang. func gracefulStop(srv *grpc.Server, grace time.Duration) { diff --git a/composition_test.go b/composition_test.go new file mode 100644 index 0000000..641609f --- /dev/null +++ b/composition_test.go @@ -0,0 +1,661 @@ +package main + +import ( + "context" + "os" + "path/filepath" + "slices" + "strings" + "testing" + + "github.com/codefly-dev/core/agents/services" + agenttesting "github.com/codefly-dev/core/agents/testing" + basev0 "github.com/codefly-dev/core/generated/go/codefly/base/v0" + agentv0 "github.com/codefly-dev/core/generated/go/codefly/services/agent/v0" + builderv0 "github.com/codefly-dev/core/generated/go/codefly/services/builder/v0" + "github.com/codefly-dev/core/resources" + "github.com/codefly-dev/core/standards" + "github.com/stretchr/testify/require" + "gopkg.in/yaml.v3" +) + +// withPublishedLock makes the agent see a gateway image that was published, for +// the tests of what it does with one. They have to mean the same thing before +// the first image exists and after, and the file on disk decides neither. +func withPublishedLock(t *testing.T) { + t.Helper() + previous := gatewayImageLockJSON + gatewayImageLockJSON = []byte(`{"name": "` + registry + `/service-warehouse", "digest": "` + digest + `"}`) + t.Cleanup(func() { gatewayImageLockJSON = previous }) +} + +func TestNewService_EmbedsBase(t *testing.T) { + svc := NewService() + require.NotNil(t, svc.Base, "services.Base embedding broken") + require.NotNil(t, svc.Settings) +} + +func TestLoadConfigurationDefaults(t *testing.T) { + svc := NewService() + require.NoError(t, svc.LoadConfiguration(context.Background(), nil)) + // Nothing is defaulted here: an unset value is the gateway's to decide, and + // a default typed in the agent would be a second opinion about it. + require.Equal(t, resolved{}, svc.conf) +} + +func configuration(values map[string]string) *basev0.Configuration { + var configured []*basev0.ConfigurationValue + for key, value := range values { + configured = append(configured, &basev0.ConfigurationValue{Key: key, Value: value}) + } + return &basev0.Configuration{Infos: []*basev0.ConfigurationInformation{{ + Name: configurationGroup, + ConfigurationValues: configured, + }}} +} + +func TestLoadConfigurationRuntimeOverridesSettings(t *testing.T) { + svc := NewService() + svc.Backend, svc.Database, svc.Dataset, svc.WarehouseLocation = "duckdb", "local.duckdb", "scratch", "eu" + + require.NoError(t, svc.LoadConfiguration(context.Background(), configuration(map[string]string{ + "SWH_BACKEND": "bigquery", + "SWH_DATABASE": "example-project", + "SWH_DATASET": "analytics", + "SWH_LOCATION": "us-central1", + }))) + require.Equal(t, resolved{backend: "bigquery", database: "example-project", dataset: "analytics", location: "us-central1"}, svc.conf) + + // A value the environment does not supply keeps the service's own setting. + require.NoError(t, svc.LoadConfiguration(context.Background(), configuration(map[string]string{"SWH_DATASET": "other"}))) + require.Equal(t, resolved{backend: "duckdb", database: "local.duckdb", dataset: "other", location: "eu"}, svc.conf) +} + +// A secret or a path delivered through a YAML block scalar or an env file +// arrives with a trailing newline. Untrimmed, the token is handed to consumers +// in one form and enforced in another, and a path names nothing. +func TestLoadConfigurationNormalizesWhitespace(t *testing.T) { + svc := NewService() + require.NoError(t, svc.LoadConfiguration(context.Background(), configuration(map[string]string{ + "SWH_AUTH_TOKEN": "pinned-token\n", + "SWH_CREDENTIALS_FILE": " /secrets/key.json\n", + "SWH_BACKEND": "mem\n", + }))) + require.Equal(t, "pinned-token", svc.conf.authToken) + require.Equal(t, "/secrets/key.json", svc.conf.credentialsFile) + require.Equal(t, "mem", svc.conf.backend) +} + +func TestResolveGatewayToken(t *testing.T) { + rt := NewRuntime() + + pinned := "operator-pinned" + rt.conf.authToken = pinned + got, err := rt.resolveGatewayToken() + require.NoError(t, err) + require.Equal(t, pinned, got, "an operator who pins a token is not silently overridden") + + rt.conf.authToken = "" + first, err := rt.resolveGatewayToken() + require.NoError(t, err) + second, err := rt.resolveGatewayToken() + require.NoError(t, err) + require.Regexp(t, `^[0-9a-f]{64}$`, first, "32 random bytes, hex encoded") + require.NotEqual(t, first, second, "each run mints a fresh token, which is what revokes the last run's") +} + +func TestResolveServingGRPCEndpoint(t *testing.T) { + endpoint, err := resolveServingGRPCEndpoint(context.Background(), []*basev0.Endpoint{{Name: "grpc", Api: "grpc"}}) + require.NoError(t, err) + require.Equal(t, "grpc", endpoint.Name) + + _, err = resolveServingGRPCEndpoint(context.Background(), nil) + require.Error(t, err, "expected error when no grpc endpoint is present") +} + +func TestConnectionString(t *testing.T) { + require.Equal(t, "grpc://localhost:9465", NewService().createConnectionString("localhost:9465")) +} + +func headlessService(t *testing.T) *Service { + t.Helper() + svc := NewService() + identity := &basev0.ServiceIdentity{Workspace: "workspace", Module: "module", Name: "warehouse", Version: "1.2.3", WorkspacePath: t.TempDir(), RelativeToWorkspace: "."} + require.NoError(t, svc.Base.HeadlessLoad(context.Background(), identity)) + return svc +} + +func connectionValues(t *testing.T, svc *Service) map[string]*basev0.ConfigurationValue { + t.Helper() + conf, err := svc.CreateConnectionConfiguration(context.Background(), nil, + &basev0.NetworkInstance{Address: "localhost:9465", Access: resources.NewNativeNetworkAccess()}) + require.NoError(t, err) + require.Len(t, conf.Infos, 1) + require.Equal(t, configurationGroup, conf.Infos[0].Name) + values := map[string]*basev0.ConfigurationValue{} + for _, value := range conf.Infos[0].ConfigurationValues { + values[value.Key] = value + } + return values +} + +// TestConnectionConfigurationCarriesTokenAsSecret pins how a consumer receives +// the gateway credential: as its own secret-marked value, never folded into the +// connection string or endpoint that get logged and templated everywhere. +func TestConnectionConfigurationCarriesTokenAsSecret(t *testing.T) { + svc := headlessService(t) + const token = "b6f1a0c9d8e7f6a5b4c3d2e1f0a9b8c7" + svc.gatewayToken = token + + values := connectionValues(t, svc) + + require.Contains(t, values, "token") + require.Equal(t, token, values["token"].Value) + require.True(t, values["token"].Secret, "token must be marked secret so it is never logged or displayed") + for _, key := range []string{"connection", "endpoint"} { + require.NotContains(t, values[key].GetValue(), token, "%s leaks the token", key) + } + require.Equal(t, "grpc://localhost:9465", values["connection"].Value) + require.Equal(t, "localhost:9465", values["endpoint"].Value) +} + +// The Builder shares the Service but never generates a token: it must not +// advertise an empty credential a consumer would then try to present. +func TestConnectionConfigurationWithoutTokenOmitsIt(t *testing.T) { + require.NotContains(t, connectionValues(t, headlessService(t)), "token") +} + +func TestAgentInformationDocumentsTheConfigurationGroup(t *testing.T) { + info, err := NewService().GetAgentInformation(context.Background(), &agentv0.AgentInformationRequest{}) + require.NoError(t, err) + require.Len(t, info.GetConfigurationDetails(), 1) + group := info.GetConfigurationDetails()[0] + require.Equal(t, configurationGroup, group.GetName()) + var fields []string + for _, field := range group.GetFields() { + fields = append(fields, field.GetName()) + } + require.Equal(t, []string{"connection", "endpoint", "token"}, fields) + require.Contains(t, info.GetReadMe(), "warehouse") +} + +func TestSettings_YAMLRoundTrip(t *testing.T) { + var s Settings + require.NoError(t, yaml.Unmarshal([]byte("backend: bigquery\ndatabase: example-project\ndataset: analytics\nlocation: eu\n"), &s)) + require.Equal(t, Settings{Backend: "bigquery", Database: "example-project", Dataset: "analytics", WarehouseLocation: "eu"}, s) +} + +func TestDeploymentTemplates(t *testing.T) { + cases := map[string]deploymentTemplateParameters{ + "nothing chosen": {ServicePort: gatewayContainerPort, ContainerPort: gatewayContainerPort}, + "bigquery": { + Backend: "bigquery", Database: "example-project", Dataset: "analytics", Location: "us-central1", + ServicePort: gatewayContainerPort, ContainerPort: gatewayContainerPort, + }, + "another service port": {Backend: "mem", ServicePort: 9090, ContainerPort: gatewayContainerPort}, + } + for name, params := range cases { + t.Run(name, func(t *testing.T) { + dir := agenttesting.AssertKustomizeTemplatesWithOverlay(t, deploymentFS, ¶ms, workloadPodOverlay()) + manifest, err := os.ReadFile(filepath.Join(dir, "base", "deployment.yaml")) + require.NoError(t, err) + body := string(manifest) + + env := deploymentEnv(t, body) + for key, want := range map[string]string{ + "SWH_BACKEND": params.Backend, "SWH_DATABASE": params.Database, + "SWH_DATASET": params.Dataset, "SWH_LOCATION": params.Location, + } { + if want == "" { + require.NotContains(t, env, key, "an unset value must render no variable, leaving the choice to the gateway") + continue + } + require.Equal(t, want, env[key]) + } + require.Equal(t, ":9465", env["SWH_LISTEN"]) + require.Equal(t, "false", env["SWH_ALLOW_ANONYMOUS"]) + // No rendering names a credentials path: nothing in this manifest + // mounts one, so an emitted path would point at a file the pod does + // not have. Deploy rejects a configured path instead. Matched on the + // bare name so that explaining the absence in rendered YAML fails too. + require.NotContains(t, body, "SWH_CREDENTIALS_FILE") + requireProbeSemantics(t, body) + }) + } +} + +// deploymentEnv reads the gateway container's literal environment from a +// rendered Deployment, refusing a key rendered twice: the CLI's binding of the +// environment's configuration replaces literals by name. +func deploymentEnv(t *testing.T, body string) map[string]string { + t.Helper() + var deployment struct { + Spec struct { + Template struct { + Spec struct { + Containers []struct { + Env []struct { + Name string `yaml:"name"` + Value string `yaml:"value"` + } `yaml:"env"` + } `yaml:"containers"` + } `yaml:"spec"` + } `yaml:"template"` + } `yaml:"spec"` + } + require.NoError(t, yaml.Unmarshal([]byte(body), &deployment)) + require.Len(t, deployment.Spec.Template.Spec.Containers, 1) + env := map[string]string{} + for _, e := range deployment.Spec.Template.Spec.Containers[0].Env { + require.NotContains(t, env, e.Name, "env %s rendered twice; the CLI binding refuses a repeated key", e.Name) + env[e.Name] = e.Value + } + return env +} + +// requireProbeSemantics pins what the rendered probes attest. A TCP probe +// passes as soon as the gRPC port is bound, and the gateway's health service +// reports the process and not the warehouse, so all three probes check the +// overall service: naming one would attest something the gateway does not +// report, and a probe that followed a warehouse outage would drain every replica +// at once, since they share one warehouse and one credential set. +func requireProbeSemantics(t *testing.T, body string) { + t.Helper() + require.NotContains(t, body, "tcpSocket", "probes must not settle for a bound socket") + + var deployment struct { + Spec struct { + Template struct { + Spec struct { + Containers []map[string]any `yaml:"containers"` + } `yaml:"spec"` + } `yaml:"template"` + } `yaml:"spec"` + } + require.NoError(t, yaml.Unmarshal([]byte(body), &deployment)) + require.Len(t, deployment.Spec.Template.Spec.Containers, 1) + container := deployment.Spec.Template.Spec.Containers[0] + for _, name := range []string{"startupProbe", "readinessProbe", "livenessProbe"} { + probe, ok := container[name].(map[string]any) + require.True(t, ok, "%s is missing", name) + grpc, isGRPC := probe["grpc"].(map[string]any) + require.True(t, isGRPC, "%s is not a grpc probe: %v", name, probe) + require.Equal(t, gatewayContainerPort, grpc["port"], name) + require.NotContains(t, grpc, "service", "%s must check the overall service", name) + } +} + +// newDeployBuilder returns a Builder wired for a headless Deploy call, against a +// gateway image that was published. +func newDeployBuilder(t *testing.T, ctx context.Context) *Builder { + t.Helper() + withPublishedLock(t) + builder := NewBuilder() + identity := &basev0.ServiceIdentity{Workspace: "workspace", Module: "module", Name: "warehouse", Version: "1.2.3", WorkspacePath: t.TempDir(), RelativeToWorkspace: "."} + require.NoError(t, builder.Base.HeadlessLoad(ctx, identity)) + builder.Base.Information = &services.Information{Service: resources.ToServiceWithCase(resources.ServiceIdentityFromProto(identity))} + builder.Base.EnvironmentVariables.SetIdentity(identity) + return builder +} + +// deployRequest builds a Kubernetes DeploymentRequest carrying the given +// warehouse configuration values, rendering into destination. +func deployRequest(destination string, values map[string]string, profile builderv0.KubernetesOutputProfile) *builderv0.DeploymentRequest { + return &builderv0.DeploymentRequest{ + Environment: &basev0.Environment{Name: "test", Fixture: "dev-admin"}, + Deployment: &builderv0.Deployment{Kind: &builderv0.Deployment_Kubernetes{ + Kubernetes: &builderv0.KubernetesDeployment{ + Namespace: "codefly", + Destination: destination, + Profile: profile, + SecretReferences: map[string]*builderv0.KubernetesSecretKeyReference{ + "SWH_AUTH_TOKEN": {Name: "warehouse-auth", Key: "token"}, + }, + }, + }}, + Configuration: configuration(values), + } +} + +const ( + ephemeral = builderv0.KubernetesOutputProfile_KUBERNETES_OUTPUT_PROFILE_EPHEMERAL_LOCAL_APPLY_V1 + restricted = builderv0.KubernetesOutputProfile_KUBERNETES_OUTPUT_PROFILE_RESTRICTED_PORTABLE_V1 +) + +func renderedDeployment(t *testing.T, destination string) string { + t.Helper() + body, err := os.ReadFile(filepath.Join(destination, "base", "deployment.yaml")) + require.NoError(t, err) + return string(body) +} + +// TestDeployResolvesConfigurationIntoTheManifest drives the real Builder.Deploy +// path and asserts the rendered manifest names the configured values: the +// regression guard for a Deploy that never loaded the configuration, leaving the +// manifest on its zero value whatever the environment said. Every key goes +// through the full config-key -> resolved -> template wiring, so a typo'd map key +// fails here rather than shipping green. +func TestDeployResolvesConfigurationIntoTheManifest(t *testing.T) { + ctx := context.Background() + builder := newDeployBuilder(t, ctx) + destination := t.TempDir() + + resp, err := builder.Deploy(ctx, deployRequest(destination, map[string]string{ + "SWH_BACKEND": "bigquery", + "SWH_DATABASE": "example-project", + "SWH_DATASET": "analytics", + "SWH_LOCATION": "us-central1", + }, ephemeral)) + require.NoError(t, err) + require.Equal(t, builderv0.DeploymentStatus_SUCCESS, resp.GetState().GetState(), resp.GetState().GetMessage()) + + env := deploymentEnv(t, renderedDeployment(t, destination)) + require.Equal(t, "bigquery", env["SWH_BACKEND"]) + require.Equal(t, "example-project", env["SWH_DATABASE"]) + require.Equal(t, "analytics", env["SWH_DATASET"]) + require.Equal(t, "us-central1", env["SWH_LOCATION"]) + require.Equal(t, "warehouse", env["CODEFLY__SERVICE"], "the CLI binds environment configuration onto the container declaring this") +} + +// The composed service's own settings are what a render with no environment +// values carries: the environment's configuration is bound onto the container +// after the render, replacing the SWH_* literals by name. +func TestDeployRendersTheServiceSettingsWhenTheEnvironmentSuppliesNone(t *testing.T) { + ctx := context.Background() + builder := newDeployBuilder(t, ctx) + builder.Settings.Backend = "bigquery" + destination := t.TempDir() + + resp, err := builder.Deploy(ctx, deployRequest(destination, nil, restricted)) + require.NoError(t, err) + require.Equal(t, builderv0.DeploymentStatus_SUCCESS, resp.GetState().GetState(), resp.GetState().GetMessage()) + require.Equal(t, "bigquery", deploymentEnv(t, renderedDeployment(t, destination))["SWH_BACKEND"]) +} + +func TestDeployRefusesWhatItHasNoWayToDeliver(t *testing.T) { + ctx := context.Background() + for name, tc := range map[string]struct { + values map[string]string + want string + }{ + "a credentials file nothing mounts": {map[string]string{"SWH_CREDENTIALS_FILE": "/keys/sa.json"}, "SWH_CREDENTIALS_FILE cannot be delivered"}, + "a literal token": {map[string]string{"SWH_AUTH_TOKEN": "pinned"}, "SWH_AUTH_TOKEN must be delivered as a Kubernetes secret reference"}, + "a value that would write the manifest": { + map[string]string{"SWH_DATASET": "x\"\n - name: SWH_AUTH_TOKEN"}, "SWH_DATASET", + }, + "anonymous deployment": {map[string]string{"SWH_ALLOW_ANONYMOUS": "true"}, "only an explicit local"}, + "a templating action": {map[string]string{"SWH_DATABASE": "{{ .Image }}"}, "SWH_DATABASE"}, + } { + t.Run(name, func(t *testing.T) { + builder := newDeployBuilder(t, ctx) + destination := t.TempDir() + + resp, err := builder.Deploy(ctx, deployRequest(destination, tc.values, ephemeral)) + require.NoError(t, err) + require.Equal(t, builderv0.DeploymentStatus_ERROR, resp.GetState().GetState()) + require.Contains(t, resp.GetState().GetMessage(), tc.want) + entries, err := os.ReadDir(destination) + require.NoError(t, err) + require.Empty(t, entries, "a refused deploy must write nothing") + }) + } +} + +// With no digest recorded there is no image to deploy, and rendering one under +// a floating name is exactly what pinning exists to prevent. +func TestDeployRefusesWhileNoGatewayImageIsPublished(t *testing.T) { + ctx := context.Background() + builder := newDeployBuilder(t, ctx) + gatewayImageLockJSON = []byte(`{"name": "` + registry + `/service-warehouse", "digest": ""}`) + destination := t.TempDir() + + resp, err := builder.Deploy(ctx, deployRequest(destination, nil, restricted)) + require.NoError(t, err) + require.Equal(t, builderv0.DeploymentStatus_ERROR, resp.GetState().GetState()) + require.Contains(t, resp.GetState().GetMessage(), resources.ErrImageUnpublished.Error()) +} + +// TestDeployRestrictedBindsTheWorkloadIdentity pins what infra has to grant +// against: in the restricted profile a hosted cell renders, the pod runs under +// its own ServiceAccount, named after the service, in the target namespace, so a +// warehouse IAM member for that Kubernetes principal authorizes exactly this +// workload and nothing else; and the image is the digest the lock records. +func TestDeployRestrictedBindsTheWorkloadIdentity(t *testing.T) { + ctx := context.Background() + builder := newDeployBuilder(t, ctx) + destination := t.TempDir() + + resp, err := builder.Deploy(ctx, deployRequest(destination, map[string]string{ + "SWH_BACKEND": "bigquery", "SWH_DATABASE": "example-cell-platform", + }, restricted)) + require.NoError(t, err) + require.Equal(t, builderv0.DeploymentStatus_SUCCESS, resp.GetState().GetState(), resp.GetState().GetMessage()) + + files := map[string]string{} + require.NoError(t, filepath.WalkDir(destination, func(path string, d os.DirEntry, err error) error { + if err != nil || d.IsDir() { + return err + } + data, err := os.ReadFile(path) + rel, _ := filepath.Rel(destination, path) + files[rel] = string(data) + return err + })) + // No Namespace and no Secret in a restricted tree; nothing secret-shaped. + for rel, body := range files { + require.NotContains(t, body, "kind: Namespace", rel) + require.NotContains(t, body, "kind: Secret", rel) + require.NotContains(t, body, "SWH_CREDENTIALS_FILE", rel) + require.NotContains(t, body, "SWH_PASSWORD", rel) + require.NotContains(t, body, "envFrom", rel) + } + + account, ok := files[filepath.Join("base", "serviceaccount.yaml")] + require.True(t, ok, "restricted render must carry the workload's ServiceAccount; files: %v", slices.Sorted(func(yield func(string) bool) { + for name := range files { + if !yield(name) { + return + } + } + })) + var sa struct { + Kind string `yaml:"kind"` + Metadata struct { + Name string `yaml:"name"` + Namespace string `yaml:"namespace"` + Annotations map[string]string `yaml:"annotations"` + } `yaml:"metadata"` + } + require.NoError(t, yaml.Unmarshal([]byte(account), &sa)) + require.Equal(t, "ServiceAccount", sa.Kind) + require.Equal(t, "warehouse", sa.Metadata.Name) + require.Equal(t, "codefly", sa.Metadata.Namespace) + require.Empty(t, sa.Metadata.Annotations, "a direct workload-identity grant needs no cloud-account annotation") + + body := files[filepath.Join("base", "deployment.yaml")] + var deployment struct { + Spec struct { + Template struct { + Spec struct { + ServiceAccountName string `yaml:"serviceAccountName"` + Containers []struct { + Image string `yaml:"image"` + } `yaml:"containers"` + } `yaml:"spec"` + } `yaml:"template"` + } `yaml:"spec"` + } + require.NoError(t, yaml.Unmarshal([]byte(body), &deployment)) + require.Equal(t, "warehouse", deployment.Spec.Template.Spec.ServiceAccountName) + require.Equal(t, registry+"/service-warehouse@"+digest, deployment.Spec.Template.Spec.Containers[0].Image) + requireProbeSemantics(t, body) +} + +// TestDeployServicePublishesAllocatedPort: core allocates the gRPC endpoint an +// in-cluster port and hands consumers that port, so the Service must publish it +// and forward it to the container's listener. +func TestDeployServicePublishesAllocatedPort(t *testing.T) { + ctx := context.Background() + builder := newDeployBuilder(t, ctx) + builder.GrpcEndpoint = &basev0.Endpoint{Name: "grpc", Module: "module", Service: "warehouse", Api: standards.GRPC} + destination := t.TempDir() + req := deployRequest(destination, nil, restricted) + instance := resources.NewNetworkInstance("warehouse.codefly.svc.cluster.local", 9090) + instance.Access = resources.NewContainerNetworkAccess() + req.NetworkMappings = []*basev0.NetworkMapping{ + {Endpoint: builder.GrpcEndpoint, Instances: []*basev0.NetworkInstance{instance}}, + // Another service's endpoint is never published by this Service. + { + Endpoint: &basev0.Endpoint{Name: "grpc", Module: "module", Service: "other", Api: standards.GRPC}, + Instances: []*basev0.NetworkInstance{instance}, + }, + } + resp, err := builder.Deploy(ctx, req) + require.NoError(t, err) + require.Equal(t, builderv0.DeploymentStatus_SUCCESS, resp.GetState().GetState(), resp.GetState().GetMessage()) + service, err := os.ReadFile(filepath.Join(destination, "base", "service.yaml")) + require.NoError(t, err) + require.Contains(t, string(service), " - name: grpc\n port: 9090\n targetPort: 9465\n") +} + +// Without a mapping for the endpoint the Service keeps publishing the container +// port. +func TestDeployServiceFallsBackToContainerPort(t *testing.T) { + ctx := context.Background() + builder := newDeployBuilder(t, ctx) + destination := t.TempDir() + + resp, err := builder.Deploy(ctx, deployRequest(destination, nil, ephemeral)) + require.NoError(t, err) + require.Equal(t, builderv0.DeploymentStatus_SUCCESS, resp.GetState().GetState(), resp.GetState().GetMessage()) + service, err := os.ReadFile(filepath.Join(destination, "base", "service.yaml")) + require.NoError(t, err) + require.Contains(t, string(service), " - name: grpc\n port: 9465\n targetPort: 9465\n") +} + +// The endpoint a Create scaffolds must be one core's endpoint model accepts, and +// reachable from the modules that depend on this service. +func TestCreateEndpointsDeclaresAnInternalGRPCEndpoint(t *testing.T) { + builder := newDeployBuilder(t, context.Background()) + require.NoError(t, builder.CreateEndpoints(context.Background())) + require.Len(t, builder.Endpoints, 1) + endpoint := builder.Endpoints[0] + require.Equal(t, standards.GRPC, endpoint.GetApi()) + require.Equal(t, resources.VisibilityInternal, endpoint.GetVisibility()) + require.True(t, strings.HasPrefix(endpoint.GetService(), "warehouse"), endpoint.GetService()) +} + +// Both output profiles use the same external secret, so a portable rendering +// never needs a plaintext credential. Consumers reference that key as well. +func TestDeployRequiresAuthenticationAndRendersNetworkBoundary(t *testing.T) { + for _, profile := range []builderv0.KubernetesOutputProfile{ephemeral, restricted} { + t.Run(profile.String(), func(t *testing.T) { + ctx := context.Background() + builder := newDeployBuilder(t, ctx) + destination := t.TempDir() + req := deployRequest(destination, nil, profile) + req.GetDeployment().GetKubernetes().SecretReferences = nil + resp, err := builder.Deploy(ctx, req) + require.NoError(t, err) + require.Equal(t, builderv0.DeploymentStatus_ERROR, resp.GetState().GetState()) + require.Contains(t, resp.GetState().GetMessage(), "requires an SWH_AUTH_TOKEN Kubernetes secret reference") + entries, err := os.ReadDir(destination) + require.NoError(t, err) + require.Empty(t, entries) + + req = deployRequest(destination, nil, profile) + resp, err = builder.Deploy(ctx, req) + require.NoError(t, err) + require.Equal(t, builderv0.DeploymentStatus_SUCCESS, resp.GetState().GetState(), resp.GetState().GetMessage()) + var deployment map[string]any + require.NoError(t, yaml.Unmarshal([]byte(renderedDeployment(t, destination)), &deployment)) + container := deployment["spec"].(map[string]any)["template"].(map[string]any)["spec"].(map[string]any)["containers"].([]any)[0].(map[string]any) + found := false + for _, raw := range container["env"].([]any) { + env := raw.(map[string]any) + if env["name"] == "SWH_AUTH_TOKEN" { + found = true + require.NotContains(t, env, "value") + require.Equal(t, map[string]any{"secretKeyRef": map[string]any{"name": "warehouse-auth", "key": "token"}}, env["valueFrom"]) + } + } + require.True(t, found) + body, err := os.ReadFile(filepath.Join(destination, "base", "networkpolicy.yaml")) + require.NoError(t, err) + var policy map[string]any + require.NoError(t, yaml.Unmarshal(body, &policy)) + require.Equal(t, "NetworkPolicy", policy["kind"]) + spec := policy["spec"].(map[string]any) + require.Equal(t, map[string]any{"matchLabels": map[string]any{"app": "warehouse"}}, spec["podSelector"]) + require.Equal(t, []any{"Ingress"}, spec["policyTypes"]) + ingress := spec["ingress"].([]any) + require.Len(t, ingress, 1) + require.Equal(t, []any{map[string]any{"podSelector": map[string]any{}}}, ingress[0].(map[string]any)["from"]) + require.Equal(t, []any{map[string]any{"protocol": "TCP", "port": gatewayContainerPort}}, ingress[0].(map[string]any)["ports"]) + kustomization, err := os.ReadFile(filepath.Join(destination, "base", "kustomization.yaml")) + require.NoError(t, err) + require.Contains(t, string(kustomization), "networkpolicy.yaml") + }) + } +} + +func TestDeployRejectsAnonymousKeyFromEveryConfigurationSource(t *testing.T) { + const key = "SWH_ALLOW_ANONYMOUS" + cases := map[string]func(*Builder, *builderv0.DeploymentRequest){ + "secret reference": func(_ *Builder, req *builderv0.DeploymentRequest) { + req.GetDeployment().GetKubernetes().SecretReferences[key] = &builderv0.KubernetesSecretKeyReference{Name: "override", Key: "flag"} + }, + "nil secret reference": func(_ *Builder, req *builderv0.DeploymentRequest) { + req.GetDeployment().GetKubernetes().SecretReferences[key] = nil + }, + "secret configuration": func(_ *Builder, req *builderv0.DeploymentRequest) { + req.Configuration = configuration(map[string]string{key: "true"}) + req.Configuration.Infos[0].ConfigurationValues[0].Secret = true + }, + "templated configuration": func(_ *Builder, req *builderv0.DeploymentRequest) { + req.Configuration = configuration(map[string]string{key: ""}) + req.Configuration.Infos[0].ConfigurationValues[0].Template = &basev0.ConfigurationValueTemplate{} + }, + "another group": func(_ *Builder, req *builderv0.DeploymentRequest) { + req.Configuration = configuration(map[string]string{key: "true"}) + req.Configuration.Infos[0].Name = "another" + }, + "dependency configuration": func(_ *Builder, req *builderv0.DeploymentRequest) { + req.DependenciesConfigurations = []*basev0.Configuration{configuration(map[string]string{key: "true"})} + }, + "ConfigMap data via raw configuration": func(b *Builder, _ *builderv0.DeploymentRequest) { + require.NoError(t, b.EnvironmentVariables.AddRawConfigurations(context.Background(), configuration(map[string]string{key: "true"}))) + }, + "environment override": func(b *Builder, _ *builderv0.DeploymentRequest) { + b.EnvironmentVariables.SetOverrides(map[string]string{key: "true"}) + }, + "configuration carrier": func(b *Builder, _ *builderv0.DeploymentRequest) { + b.EnvironmentVariables.SetOverrides(map[string]string{"CODEFLY__SERVICE_CONFIGURATION__MODULE__WAREHOUSE__WAREHOUSE__" + key: "true"}) + }, + } + // Presence is prohibited: false or an unresolved/empty value must not create + // a second entry that can later be rebound from another delivery source. + for _, value := range []string{"true", "false", "", "1", "TRUE", "yes"} { + cases["literal "+value] = func(_ *Builder, req *builderv0.DeploymentRequest) { + req.Configuration = configuration(map[string]string{key: value}) + } + } + for _, profile := range []builderv0.KubernetesOutputProfile{ephemeral, restricted} { + for name, prepare := range cases { + t.Run(profile.String()+"/"+name, func(t *testing.T) { + builder := newDeployBuilder(t, context.Background()) + destination := t.TempDir() + req := deployRequest(destination, nil, profile) + prepare(builder, req) + response, err := builder.Deploy(context.Background(), req) + require.NoError(t, err) + require.Equal(t, builderv0.DeploymentStatus_ERROR, response.GetState().GetState()) + require.Contains(t, response.GetState().GetMessage(), key) + require.Contains(t, response.GetState().GetMessage(), "key is reserved") + files, err := os.ReadDir(destination) + require.NoError(t, err) + require.Empty(t, files, "a forbidden key must fail before rendering") + }) + } + } +} diff --git a/gateway-image.json b/gateway-image.json new file mode 100644 index 0000000..24d8284 --- /dev/null +++ b/gateway-image.json @@ -0,0 +1,4 @@ +{ + "name": "ghcr.io/codefly-dev/service-warehouse", + "digest": "" +} diff --git a/gateway_image.go b/gateway_image.go new file mode 100644 index 0000000..01d9274 --- /dev/null +++ b/gateway_image.go @@ -0,0 +1,54 @@ +package main + +import ( + "errors" + "fmt" + "os" + + "github.com/codefly-dev/core/resources" +) + +// gatewayImageName is the repository, inside core's image registry, this +// service's gateway image is published to. The release workflows derive the same +// name from the repository they run in (ghcr.io//), and +// TestGatewayImageNameIsThisRepository holds the two together. +const gatewayImageName = "service-warehouse" + +// parseGatewayImageLock delegates lock validation to core, adding the service's +// recovery instruction when no image has been published yet. +func parseGatewayImageLock(content []byte) (*resources.DockerImage, error) { + image, err := resources.ParseImageLock(content, gatewayImageName) + if errors.Is(err, resources.ErrImageUnpublished) { + return nil, fmt.Errorf("%w: run publish-gateway-image.yml for the upcoming version and commit the digest it prints", err) + } + return image, err +} + +// gatewayImageOverrideEnv names a locally built gateway image to run instead of +// the published one. It configures the agent, not the gateway: the gateway's own +// surface is the SWH_* variables internal/config reads. +const gatewayImageOverrideEnv = "SWH_GATEWAY_IMAGE" + +// publishedGatewayImage is the image gateway-image.json pins, whatever the +// override says. A deployment, an audit and a source-scope inventory describe +// the image a release ships, which a locally built stand-in is not. +func publishedGatewayImage() (*resources.DockerImage, error) { + return parseGatewayImageLock(gatewayImageLockJSON) +} + +// effectiveGatewayImage resolves the gateway image the agent actually runs, and +// reports whether it came from the override. An override is built locally and +// never pushed, so it is reachable through the Docker daemon rather than a +// registry and carries the single platform the daemon holds. +// +// The published image is pinned to the digest gateway-image.json records. A tag +// serves whatever was pushed to it last, so an agent naming one would run, and +// report evidence for, an image nobody checked was the one built for this +// version. +func effectiveGatewayImage() (*resources.DockerImage, bool, error) { + if override := os.Getenv(gatewayImageOverrideEnv); override != "" { + return &resources.DockerImage{Name: override}, true, nil + } + image, err := publishedGatewayImage() + return image, false, err +} diff --git a/gateway_image_test.go b/gateway_image_test.go new file mode 100644 index 0000000..33908f4 --- /dev/null +++ b/gateway_image_test.go @@ -0,0 +1,182 @@ +package main + +import ( + "fmt" + "os" + "os/exec" + "regexp" + "strings" + "testing" + + "github.com/codefly-dev/core/resources" + "github.com/stretchr/testify/require" + + "github.com/codefly-dev/service-warehouse/internal/config" + + // The mem backend is registered here, in a test, so config.FromEnv can + // resolve a configuration without a warehouse. The agent itself imports no + // backend: see TestAgentLinksNoBackendDriver. + _ "github.com/codefly-dev/service-warehouse/internal/backend/mem" +) + +const ( + registry = "ghcr.io/codefly-dev" + digest = "sha256:1205da0743e56b68c12c754438476e8fdd0f05cdb08ce1c9687ddb8d8f0e1669" +) + +func TestParseGatewayImageLock(t *testing.T) { + image, err := parseGatewayImageLock([]byte(`{ + "name": "` + registry + `/service-warehouse", + "digest": "` + digest + `" + }`)) + require.NoError(t, err) + require.Equal(t, registry+"/service-warehouse@"+digest, image.FullName()) +} + +// A lock the agent accepts but that names no immutable image is the failure +// pinning exists to prevent: it produces subjects whose evidence describes +// whatever the registry served, so every malformed shape is refused at load +// rather than carried into a scan. +func TestParseGatewayImageLockRejectsAnythingButASHA256Digest(t *testing.T) { + name := registry + "/service-warehouse" + for label, lock := range map[string]string{ + "no name": `{"digest": "` + digest + `"}`, + "another image": `{"name": "` + registry + `/service-object-storage", "digest": "` + digest + `"}`, + "another registry": `{"name": "registry.example.com/codefly-dev/service-warehouse", "digest": "` + digest + `"}`, + "tag as digest": `{"name": "` + name + `", "digest": "0.0.3"}`, + "other algorithm": `{"name": "` + name + `", "digest": "sha512:1205da07"}`, + "truncated": `{"name": "` + name + `", "digest": "sha256:1205da07"}`, + "not hex": `{"name": "` + name + `", "digest": "sha256:zzzzda0743e56b68c12c754438476e8fdd0f05cdb08ce1c9687ddb8d8f0e1669"}`, + "not json": `sha256:1205da07`, + "uppercase digest": `{"name": "` + name + `", "digest": "` + strings.ToUpper(digest) + `"}`, + "uppercase hex": `{"name": "` + name + `", "digest": "sha256:` + strings.ToUpper(strings.TrimPrefix(digest, "sha256:")) + `"}`, + "unknown key": `{"name": "` + name + `", "digest": "` + digest + `", "tag": "latest"}`, + "trailing json": `{"name": "` + name + `", "digest": "` + digest + `"} {}`, + } { + t.Run(label, func(t *testing.T) { + _, err := parseGatewayImageLock([]byte(lock)) + require.Error(t, err, "accepted a lock that pins no image") + require.NotErrorIs(t, err, resources.ErrImageUnpublished) + }) + } +} + +// A lock with no digest is the tree before its first image was published. A +// reference with no digest and no tag is the registry's :latest, so the only +// safe reading is a refusal that says what to do. +func TestAnUnpublishedLockIsRefusedNeverRunAsLatest(t *testing.T) { + image, err := parseGatewayImageLock([]byte(`{"name": "` + registry + `/service-warehouse", "digest": ""}`)) + require.ErrorIs(t, err, resources.ErrImageUnpublished) + require.Nil(t, image) + require.Contains(t, err.Error(), "publish-gateway-image.yml") +} + +// The embedded copy is what the agent runs; the file on disk is what the +// release reads to tag and to inventory. They are the same bytes, and a test +// reading the file keeps the embed from silently going stale. +func TestGatewayImageMatchesTheLockOnDisk(t *testing.T) { + lock, err := os.ReadFile("gateway-image.json") + require.NoError(t, err) + require.Equal(t, string(lock), string(gatewayImageLockJSON)) + + want, wantErr := parseGatewayImageLock(lock) + got, gotErr := publishedGatewayImage() + require.Equal(t, wantErr, gotErr) + if wantErr == nil { + require.Equal(t, want.FullName(), got.FullName()) + } +} + +// The override names a locally built image for a run. It must never decide what +// a deployment or an audit describes, which is the image a release ships. +func TestTheOverrideDecidesTheRunButNotThePublishedImage(t *testing.T) { + t.Setenv(gatewayImageOverrideEnv, "service-warehouse:local") + + image, overridden, err := effectiveGatewayImage() + require.NoError(t, err) + require.True(t, overridden) + require.Equal(t, "service-warehouse:local", image.FullName()) + + published, err := publishedGatewayImage() + if err == nil { + require.NotEqual(t, "service-warehouse:local", published.FullName()) + } else { + require.ErrorIs(t, err, resources.ErrImageUnpublished) + } + + t.Setenv(gatewayImageOverrideEnv, "") + _, overridden, _ = effectiveGatewayImage() + require.False(t, overridden) +} + +// The image's name is derived, not typed: the agent takes the registry from core +// and the repository name from gatewayImageName, while the workflows publish to +// the repository they run in. This is where those meet, so that renaming the +// repository, or the constant, fails here instead of publishing one image and +// running another. +func TestGatewayImageNameIsThisRepository(t *testing.T) { + gomod, err := os.ReadFile("go.mod") + require.NoError(t, err) + module := regexp.MustCompile(`(?m)^module github\.com/(\S+)$`).FindSubmatch(gomod) + require.NotNil(t, module, "go.mod names no github.com module") + repository := string(module[1]) + + image := resources.PublishedImage(gatewayImageName, "") + require.Equal(t, "ghcr.io/"+repository, image.Repository+"/"+image.Name, + "the agent runs an image other than the one this repository publishes") + + lock, err := os.ReadFile("gateway-image.json") + require.NoError(t, err) + require.Contains(t, string(lock), `"name": "ghcr.io/`+repository+`"`) + + require.Contains(t, workflowBody(t, "release.yml"), "IMAGE: ghcr.io/${{ github.repository }}") + // The in-repository publisher derives the image name from this repository. + require.Contains(t, workflowBody(t, "publish-gateway-image.yml"), "REPOSITORY: ${{ github.repository }}") +} + +// The container port is stated once in the agent and handed to the gateway as +// SWH_LISTEN and to the templates as the container port. The gateway's own +// default and the image's EXPOSE are the other two statements of it, and none of +// the three is derived from another. +func TestGatewayContainerPortIsTheGatewaysDefault(t *testing.T) { + t.Setenv("SWH_BACKEND", "mem") + t.Setenv("SWH_ALLOW_ANONYMOUS", "true") + t.Setenv("SWH_AUTH_TOKEN", "") + t.Setenv("SWH_LISTEN", "") + + cfg, err := config.FromEnv() + require.NoError(t, err) + require.Equal(t, fmt.Sprintf(":%d", gatewayContainerPort), cfg.ListenAddr) + + dockerfile, err := os.ReadFile("Dockerfile") + require.NoError(t, err) + require.Regexp(t, regexp.MustCompile(fmt.Sprintf(`(?m)^EXPOSE %d$`, gatewayContainerPort)), string(dockerfile)) +} + +// The agent is released without cgo (.goreleaser.yaml) and must stay free of the +// gateway's backends: the embedded DuckDB engine is a cgo driver, and linking it +// here would turn the release's CGO_ENABLED=0 build into one that fails the day +// the driver lands. +func TestAgentLinksNoBackendDriver(t *testing.T) { + out, err := exec.Command("go", "list", "-deps", "-f", "{{.ImportPath}}", ".").Output() + require.NoError(t, err) + for _, dependency := range strings.Fields(string(out)) { + require.False(t, strings.Contains(dependency, "service-warehouse/internal/backend") || + strings.Contains(dependency, "duckdb"), + "the agent links %s, a backend the gateway carries", dependency) + } +} + +// The gateway may use core's standalone caller token, but must not acquire the +// agent runtime or any of its resource orchestration dependencies. +func TestGatewayLinksOnlyCoreCallerToken(t *testing.T) { + out, err := exec.Command("go", "list", "-deps", "-f", "{{.ImportPath}}", "./cmd/service-warehouse").Output() + require.NoError(t, err) + var linked []string + for _, dependency := range strings.Fields(string(out)) { + if strings.HasPrefix(dependency, "github.com/codefly-dev/core/") { + linked = append(linked, dependency) + } + } + require.Equal(t, []string{"github.com/codefly-dev/core/callertoken"}, linked) +} diff --git a/go.mod b/go.mod index f277afc..4faa5c2 100644 --- a/go.mod +++ b/go.mod @@ -1,17 +1,113 @@ module github.com/codefly-dev/service-warehouse -go 1.27.0 +go 1.27.2 require ( + github.com/codefly-dev/core v0.18.1 github.com/stretchr/testify v1.12.1 - google.golang.org/grpc v1.83.1 + google.golang.org/grpc v1.83.2 google.golang.org/protobuf v1.36.12 + gopkg.in/yaml.v3 v3.0.1 ) require ( + buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.36.12-20260825204119-511051f7f437.2 // indirect + buf.build/go/protovalidate v1.4.0 // indirect + cel.dev/cel-go v0.32.0 // indirect + cel.dev/expr v0.25.3 // indirect + github.com/Azure/go-ansiterm v0.0.0-20261006220739-8c912ac31dd4 // indirect + github.com/Masterminds/semver v1.5.0 // indirect + github.com/Masterminds/semver/v3 v3.5.0 // indirect + github.com/Microsoft/go-winio v0.6.2 // indirect + github.com/VividCortex/ewma v1.2.0 // indirect + github.com/antlr4-go/antlr/v4 v4.13.1 // indirect + github.com/blang/semver v3.5.1+incompatible // indirect + github.com/blang/semver/v4 v4.0.0 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/cheggaaa/pb/v3 v3.2.1 // indirect + github.com/clipperhouse/uax29/v2 v2.7.0 // indirect + github.com/codefly-dev/gortk v0.2.0 // indirect + github.com/containerd/errdefs v1.0.0 // indirect + github.com/containerd/errdefs/pkg v0.3.0 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/distribution/reference v0.6.0 // indirect + github.com/docker/docker v28.5.2+incompatible // indirect + github.com/docker/go-connections v0.8.1 // indirect + github.com/docker/go-units v0.5.0 // indirect + github.com/fatih/color v1.19.0 // indirect + github.com/felixge/httpsnoop v1.1.0 // indirect + github.com/fsnotify/fsnotify v1.10.1 // indirect + github.com/go-errors/errors v1.4.2 // indirect + github.com/go-logr/logr v1.4.4 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/go-ole/go-ole v1.3.0 // indirect + github.com/go-openapi/analysis v1.0.0 // indirect + github.com/go-openapi/errors v0.22.8 // indirect + github.com/go-openapi/jsonpointer v1.0.1 // indirect + github.com/go-openapi/jsonreference v1.0.2 // indirect + github.com/go-openapi/loads v0.25.3 // indirect + github.com/go-openapi/spec v1.0.1 // indirect + github.com/go-openapi/strfmt v0.27.2 // indirect + github.com/go-openapi/swag v0.28.0 // indirect + github.com/go-openapi/swag/cmdutils v0.28.0 // indirect + github.com/go-openapi/swag/conv v0.29.1 // indirect + github.com/go-openapi/swag/fileutils v0.28.0 // indirect + github.com/go-openapi/swag/jsonutils v0.29.1 // indirect + github.com/go-openapi/swag/loading v0.29.1 // indirect + github.com/go-openapi/swag/mangling v0.29.1 // indirect + github.com/go-openapi/swag/netutils v0.28.0 // indirect + github.com/go-openapi/swag/pools v0.29.1 // indirect + github.com/go-openapi/swag/stringutils v0.29.1 // indirect + github.com/go-openapi/swag/typeutils v0.29.1 // indirect + github.com/go-openapi/swag/yamlutils v0.29.1 // indirect + github.com/go-viper/mapstructure/v2 v2.5.0 // indirect + github.com/gofrs/flock v0.13.1 // indirect + github.com/google/gnostic-models v0.6.9 // indirect + github.com/google/go-github/v89 v89.0.0 // indirect + github.com/google/go-querystring v1.2.0 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/hashicorp/errwrap v1.1.0 // indirect + github.com/hashicorp/go-multierror v1.1.1 // indirect + github.com/kr/text v0.2.0 // indirect + github.com/lufia/plan9stats v0.0.0-20260330125221-c963978e514e // indirect + github.com/mattn/go-colorable v0.1.15 // indirect + github.com/mattn/go-isatty v0.0.24 // indirect + github.com/mattn/go-runewidth v0.0.27 // indirect + github.com/mitchellh/mapstructure v1.5.0 // indirect + github.com/moby/docker-image-spec v1.3.1 // indirect + github.com/moby/patternmatcher v0.6.1 // indirect + github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 // indirect + github.com/oklog/ulid/v2 v2.1.2 // indirect + github.com/opencontainers/go-digest v1.0.0 // indirect + github.com/opencontainers/image-spec v1.1.1 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect + github.com/santhosh-tekuri/jsonschema/v6 v6.0.3 // indirect + github.com/shirou/gopsutil/v3 v3.24.5 // indirect + github.com/shoenig/go-m1cpu v0.2.1 // indirect + github.com/tklauser/go-sysconf v0.4.0 // indirect + github.com/tklauser/numcpus v0.12.0 // indirect + github.com/xlab/treeprint v1.2.0 // indirect + github.com/yoheimuta/go-protoparser/v4 v4.14.2 // indirect + github.com/yusufpapurcu/wmi v1.2.4 // indirect + go.opentelemetry.io/auto/sdk v1.2.1 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.71.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.70.0 // indirect + go.opentelemetry.io/otel v1.46.0 // indirect + go.opentelemetry.io/otel/metric v1.46.0 // indirect + go.opentelemetry.io/otel/trace v1.46.0 // indirect + go.yaml.in/yaml/v2 v2.4.2 // indirect go.yaml.in/yaml/v3 v3.0.5 // indirect - golang.org/x/net v0.55.0 // indirect - golang.org/x/sys v0.45.0 // indirect - golang.org/x/text v0.37.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa // indirect + golang.org/x/exp v0.0.0-20260820142414-ca536658362e // indirect + golang.org/x/net v0.60.0 // indirect + golang.org/x/sys v0.48.0 // indirect + golang.org/x/term v0.46.0 // indirect + golang.org/x/text v0.42.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20260819154853-08b0e4226688 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260825221802-da73d73af1c5 // indirect + gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect + k8s.io/kube-openapi v0.0.0-20241212222426-2c72e554b1e7 // indirect + sigs.k8s.io/kustomize/api v0.21.1 // indirect + sigs.k8s.io/kustomize/kyaml v0.21.1 // indirect + sigs.k8s.io/yaml v1.5.0 // indirect ) diff --git a/go.sum b/go.sum index d46fd74..a221a4e 100644 --- a/go.sum +++ b/go.sum @@ -1,42 +1,325 @@ +buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.36.12-20260825204119-511051f7f437.2 h1:NbnlmV26O7oZ1iM5tsCI+GEx+3ZSdrhvnKQ/eWSrLiY= +buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.36.12-20260825204119-511051f7f437.2/go.mod h1:TCt1lluMFnctISJXvkIQ4x3ABrPuUKCWKyjKdkJNBpw= +buf.build/go/protovalidate v1.4.0 h1:UjLrYbt5VX7+TMOs2+pG5FhZhIG1mSfK4EIopbb4LcM= +buf.build/go/protovalidate v1.4.0/go.mod h1:8vJfzNT6NIG2qm3uFsJDXMlRmG+bQJzbcIn1Aa0vPGs= +cel.dev/cel-go v0.32.0 h1:irvpFKr5EuGPyxeME03ERh0rii1TX+BDAnB9eL3IvNk= +cel.dev/cel-go v0.32.0/go.mod h1:DnVip7tpJSsgZymwfT+m1tnEVy3ivAjSMXPx12YrMkU= +cel.dev/expr v0.25.3 h1:A2jO8jwOugrrovveCWfj0KEZOfqiLgAcwjpHPhzIGw0= +cel.dev/expr v0.25.3/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4= +dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8= +dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA= +github.com/Azure/go-ansiterm v0.0.0-20261006220739-8c912ac31dd4 h1:m2xcjRABtG288vnlV3U/PKh/HnPURxZp50RKNUwBxM8= +github.com/Azure/go-ansiterm v0.0.0-20261006220739-8c912ac31dd4/go.mod h1:MTUE7ZuZO89FAtAwozGg6AWD83pcdefnEq5gF82MB8s= +github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3QEww= +github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= +github.com/Masterminds/semver/v3 v3.5.0 h1:kQceYJfbupGfZOKZQg0kou0DgAKhzDg2NZPAwZ/2OOE= +github.com/Masterminds/semver/v3 v3.5.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= +github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= +github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= +github.com/ProtonMail/go-crypto v1.4.1 h1:9RfcZHqEQUvP8RzecWEUafnZVtEvrBVL9BiF67IQOfM= +github.com/ProtonMail/go-crypto v1.4.1/go.mod h1:e1OaTyu5SYVrO9gKOEhTc+5UcXtTUa+P3uLudwcgPqo= +github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow= +github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4= +github.com/antlr4-go/antlr/v4 v4.13.1 h1:SqQKkuVZ+zWkMMNkjy5FZe5mr5WURWnlpmOuzYWrPrQ= +github.com/antlr4-go/antlr/v4 v4.13.1/go.mod h1:GKmUxMtwp6ZgGwZSva4eWPC5mS6vUAmOABFgjdkM7Nw= +github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ= +github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= +github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= +github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= +github.com/brianvoe/gofakeit/v6 v6.28.0 h1:Xib46XXuQfmlLS2EXRuJpqcw8St6qSZz75OUo0tgAW4= +github.com/brianvoe/gofakeit/v6 v6.28.0/go.mod h1:Xj58BMSnFqcn/fAQeSK+/PLtC5kSb7FJIq4JyGa8vEs= +github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM= +github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= -github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/cheggaaa/pb/v3 v3.2.1 h1:aprZbFRG+B7+ug76S8QZ6Y1PW168UHzOmbC3wa+aU6I= +github.com/cheggaaa/pb/v3 v3.2.1/go.mod h1:U9hSVxoKqJrZIE3PkFG1xXXNaK/Ilzg+EF/scpXzEdw= +github.com/clipperhouse/uax29/v2 v2.7.0 h1:+gs4oBZ2gPfVrKPthwbMzWZDaAFPGYK72F0NJv2v7Vk= +github.com/clipperhouse/uax29/v2 v2.7.0/go.mod h1:EFJ2TJMRUaplDxHKj1qAEhCtQPW2tJSwu5BF98AuoVM= +github.com/cloudflare/circl v1.6.3 h1:9GPOhQGF9MCYUeXyMYlqTR6a5gTrgR/fBLXvUgtVcg8= +github.com/cloudflare/circl v1.6.3/go.mod h1:2eXP6Qfat4O/Yhh8BznvKnJ+uzEoTQ6jVKJRn81BiS4= +github.com/codefly-dev/core v0.18.1 h1:W1RtH/Hjly+rh+BQlWB2HYYO3jdQmAe+bkhERro85bU= +github.com/codefly-dev/core v0.18.1/go.mod h1:FKBqn6GMGIsx9b3LNY6kKCRxCEjjeswvqt2CxSZWq20= +github.com/codefly-dev/gortk v0.2.0 h1:7bOlS5valYz2zil+fZctQNcPCYBcPj86abcw9N8h1hQ= +github.com/codefly-dev/gortk v0.2.0/go.mod h1:dDWUMFgAP063OCGhTEpV7FFUj9+zTcxxO/nV80VKEwg= +github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI= +github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M= +github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE= +github.com/containerd/errdefs/pkg v0.3.0/go.mod h1:NJw6s9HwNuRhnjJhM7pylWwMyAkmCQvQ4GpJHEqRLVk= +github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= +github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/cyphar/filepath-securejoin v0.7.0 h1:s0Y3ITPy6sQn5xt54DuYvTF8hu134ooYLUb58DX/HjE= +github.com/cyphar/filepath-securejoin v0.7.0/go.mod h1:ymLGms/u3BYaviIiuKFnUx8EkQEZeK6cInNoAPJA3o4= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= +github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= +github.com/dlclark/regexp2 v1.12.0 h1:0j4c5qQmnC6XOWNjP3PIXURXN2gWx76rd3KvgdPkCz8= +github.com/dlclark/regexp2 v1.12.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= +github.com/docker/docker v28.5.2+incompatible h1:DBX0Y0zAjZbSrm1uzOkdr1onVghKaftjlSWt4AFexzM= +github.com/docker/docker v28.5.2+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/go-connections v0.8.1 h1:JibmG5hULs5qXSr/cp/w3Pw5fZuStt4MOHMUExb29/M= +github.com/docker/go-connections v0.8.1/go.mod h1:no1qkHdjq7kLMGUXYAduOhYPSJxxvgWBh7ogVvptn3Q= +github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= +github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= +github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ= +github.com/fatih/color v1.19.0 h1:Zp3PiM21/9Ld6FzSKyL5c/BULoe/ONr9KlbYVOfG8+w= +github.com/fatih/color v1.19.0/go.mod h1:zNk67I0ZUT1bEGsSGyCZYZNrHuTkJJB+r6Q9VuMi0LE= +github.com/felixge/httpsnoop v1.1.0 h1:3YtUj32ZZkqZtt3sZZsClsymw/QDuVfpNhoA31zeORc= +github.com/felixge/httpsnoop v1.1.0/go.mod h1:Zqxgdd+1Rkcz8euOqdr7lqgCRJztwr5hp9vDSi5UZCE= +github.com/fsnotify/fsnotify v1.10.1 h1:b0/UzAf9yR5rhf3RPm9gf3ehBPpf0oZKIjtpKrx59Ho= +github.com/fsnotify/fsnotify v1.10.1/go.mod h1:TLheqan6HD6GBK6PrDWyDPBaEV8LspOxvPSjC+bVfgo= +github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= +github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= +github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 h1:+zs/tPmkDkHx3U66DAb0lQFJrpS6731Oaa12ikc+DiI= +github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376/go.mod h1:an3vInlBmSxCcxctByoQdvwPiA7DTK7jaaFDBTtu0ic= +github.com/go-git/go-billy/v5 v5.9.0 h1:jItGXszUDRtR/AlferWPTMN4j38BQ88XnXKbilmmBPA= +github.com/go-git/go-billy/v5 v5.9.0/go.mod h1:jCnQMLj9eUgGU7+ludSTYoZL/GGmii14RxKFj7ROgHw= +github.com/go-git/go-git/v5 v5.19.2 h1:wkfn7vOlUBu8ivAWKBWisTiwJK4jYHzTF8Ndv1LyGqY= +github.com/go-git/go-git/v5 v5.19.2/go.mod h1:QqCBE1EFN5ddFmrliLQ3/ntRCUjZU3EJuwuB/jWEHjk= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.4 h1:tG4xh9yMsRCAiodLVTxyrkzSZ9+o0L1Kg/+cPVcbP/8= +github.com/go-logr/logr v1.4.4/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= +github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE= +github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78= +github.com/go-openapi/analysis v1.0.0 h1:sNvbAGCJqUTqIAodr9IVqJMmuZas3YS9ms1dGK9yiJ4= +github.com/go-openapi/analysis v1.0.0/go.mod h1:NhYjJ57fnE+bcE7UwrJyMkhWA3Dfz7TdiBfTVAnos4Y= +github.com/go-openapi/errors v0.22.8 h1:oP7sW7TWc3wFFjrzzj0nI83H2qMBkNjNfSd+XRejk/I= +github.com/go-openapi/errors v0.22.8/go.mod h1:BuUoHcYrU6E7V9gfj1I5wLQqgtIHnup/alXZ8KdgQ0w= +github.com/go-openapi/jsonpointer v1.0.1 h1:2KxywRmNwJkT/FMBa3iRNHEaAxSJvjqoufQZy3au1Mg= +github.com/go-openapi/jsonpointer v1.0.1/go.mod h1:wI7ZYsFmbIi9nBXOZqgDaS/bqOchRGZjqxFli7FBYxY= +github.com/go-openapi/jsonreference v1.0.2 h1:oS4et8FOf3p3UQxEo4Xt0esijmBUM+F259Xl72OSZsc= +github.com/go-openapi/jsonreference v1.0.2/go.mod h1:TbUNSOo+fcorZjFaNoiSDSoaNnnZtqJtLGR1PuvE/Cs= +github.com/go-openapi/loads v0.25.3 h1:V+jKy/thXWdLJUuYC8sZX2dICyAa8M3DokF70jj34P0= +github.com/go-openapi/loads v0.25.3/go.mod h1:LgLyCSOLBL2Qnj0Ps1oo2YH8jZoEKsMeVZCc+ALIxFo= +github.com/go-openapi/spec v1.0.1 h1:lj2vdGpNDcVgwRc6qXdw6qt/KQpCtSa9tnUH6vpDPDk= +github.com/go-openapi/spec v1.0.1/go.mod h1:M//GWQGtDUAjnP37gE6fInLgaczB+FatoipV3H1fYw8= +github.com/go-openapi/strfmt v0.27.2 h1:SG32SlbwNy92s0KJiVxt2joJeFdqIYHvwrA0OU6HqzQ= +github.com/go-openapi/strfmt v0.27.2/go.mod h1:M4CKsMO0Fb8qR10+1Ra75wCKNNquy+Vj+4LWZrhTo2E= +github.com/go-openapi/swag v0.28.0 h1:xkgbOSKj6DZziNpyqRRAOt3GJGtgjgsd2RoyT30VWuw= +github.com/go-openapi/swag v0.28.0/go.mod h1:4qYnT3Cqr1p1VknOdPo70evN4rgQnAg6jwApHyxSGIg= +github.com/go-openapi/swag/cmdutils v0.28.0 h1:7TOeNtkYru1SG8Y34tDh9WBbLsMqGnptuxWiHREPZ4Q= +github.com/go-openapi/swag/cmdutils v0.28.0/go.mod h1:Sm1MVFMkF6guJJ+pQqHnQA3N0j9qALV3NxzDSv6bETM= +github.com/go-openapi/swag/conv v0.29.1 h1:AC4Eh/5c/eUDOUCzzsRC9ghmFgOSBHeRMGIngY0ZUGA= +github.com/go-openapi/swag/conv v0.29.1/go.mod h1:S1X7/ZrBEZOC0Wc8AGxjbcGS92l3WEjA7aPtpl+RaqM= +github.com/go-openapi/swag/fileutils v0.28.0 h1:Z04XWQD7R8Eq+7GnOrjovBxPPmZzsS4gt2H2GPGIViU= +github.com/go-openapi/swag/fileutils v0.28.0/go.mod h1:VvJFZLTZS0AI854gEQz5tk7dBESdLjiNUMSZ/th2ry8= +github.com/go-openapi/swag/jsonutils v0.29.1 h1:AFCxs0eQZ24/QyfhVHM2t49rMz7Vv3XCsZQI6yrNy+c= +github.com/go-openapi/swag/jsonutils v0.29.1/go.mod h1:u3+sCfJpttDpcmS5kpm0yxL6GK0eWgODsx8Yw8fcqNM= +github.com/go-openapi/swag/jsonutils/fixtures_test v0.29.1 h1:BiiXE31Bx9SfpsMmOQj5KYpUhTZBpLVriVhJDuLuY2o= +github.com/go-openapi/swag/jsonutils/fixtures_test v0.29.1/go.mod h1:julgTUKZ9/D0j6O7GKajmRs+812FWxQg/mMpGunWSjg= +github.com/go-openapi/swag/loading v0.29.1 h1:FCv5fG8UhTdDJa2R7w+5O9Ekpcbw7tt0nFWvmDKGBjc= +github.com/go-openapi/swag/loading v0.29.1/go.mod h1:N0ESuem4p2oedKal8EJhciqnJ9Q9Wmt83L1CRB3Fouw= +github.com/go-openapi/swag/mangling v0.29.1 h1:lHALtvYCdxVnRl4GrHmFPwfBTZYIObqdGNSKyu/8D6I= +github.com/go-openapi/swag/mangling v0.29.1/go.mod h1:SAop9pB7PUjQ/CGCNf/JmCKTRK+GDO+RqE9UHqC/N6s= +github.com/go-openapi/swag/netutils v0.28.0 h1:YXN6TALEi2pzts8/8GNm6T61HTAZsieukGZidap989k= +github.com/go-openapi/swag/netutils v0.28.0/go.mod h1:J+WYyFMLtvtCGqa6jLv+YNUmIKI3ZRQRrvfNDMoQoEQ= +github.com/go-openapi/swag/pools v0.29.1 h1:NRogYxdEW9SjRM4mkAOji9iefO4MRXq3p/ZJcoQbUKg= +github.com/go-openapi/swag/pools v0.29.1/go.mod h1:leDcaghjkRAhCuCRv9NfJU5f0mjoU3cT/XZObhMk3pc= +github.com/go-openapi/swag/stringutils v0.29.1 h1:1ykunK7iJQk1uOO7+oUH1ukbsK85fFCOiCFMOVSY+F0= +github.com/go-openapi/swag/stringutils v0.29.1/go.mod h1:7fSqZ+z8Qc0tOfAAK0jVa5qFGrnIlRi6n7NeGGrr1vc= +github.com/go-openapi/swag/typeutils v0.29.1 h1:Nzv9nhnlLCRBPQqfOX+7lB6Guju370or8StT+lIOf6M= +github.com/go-openapi/swag/typeutils v0.29.1/go.mod h1:hxpgDZJVBkBsi/d3MIUosafoFdE5exaQRmVp0zwu3YE= +github.com/go-openapi/swag/yamlutils v0.29.1 h1:69w3tsBajm7MR/fejLy7HD/3J68Ys1SeeZMEzZ3w2sk= +github.com/go-openapi/swag/yamlutils v0.29.1/go.mod h1:rgsp3vT/QdWzKwn43CigDwjOGIenPyTZMKnxEM8jZOA= +github.com/go-openapi/testify/enable/yaml/v2 v2.7.0 h1:wPW6YRgx3+SID1yUy/Xwa17L8kFEaEKod2VRbJDZNUs= +github.com/go-openapi/testify/enable/yaml/v2 v2.7.0/go.mod h1:mI1M88etYbc3PhgHsWQK2kwvNwW5aGFqMPbmib+SGIs= +github.com/go-openapi/testify/v2 v2.7.0 h1:bycOreEj6wfBvijg3YFogZ/sFjTCDmQnwSodSzHa3X8= +github.com/go-openapi/testify/v2 v2.7.0/go.mod h1:SgsVHtfooshd0tublTtJ50FPKhujf47YRqauXXOUxfw= +github.com/go-viper/mapstructure/v2 v2.5.0 h1:vM5IJoUAy3d7zRSVtIwQgBj7BiWtMPfmPEgAXnvj1Ro= +github.com/go-viper/mapstructure/v2 v2.5.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= +github.com/gofrs/flock v0.13.1 h1:jjREztyBeSKBZYAC+mgc1laB+xsgy4kYMf3FbKF2UBo= +github.com/gofrs/flock v0.13.1/go.mod h1:sf4BFiHwnvgxa25DlQoDqXQnwRMEOwqxRq37P6MzzmE= +github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 h1:f+oWsMOmNPc8JmEHVZIycC7hBoQxHH9pNKQORJNozsQ= +github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8/go.mod h1:wcDNUvekVysuuOpQKo3191zZyTpiI6se1N1ULghS0sw= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/gnostic-models v0.6.9 h1:MU/8wDLif2qCXZmzncUQ/BOfxWfthHi63KqpoNbWqVw= +github.com/google/gnostic-models v0.6.9/go.mod h1:CiWsm0s6BSQd1hRn8/QmxqB6BesYcbSZxsz9b0KuDBw= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/go-github/v89 v89.0.0 h1:35bEK5XoEcF3PZrlVbl9XN63f5BcJRA/UGkxeC9xPg0= +github.com/google/go-github/v89 v89.0.0/go.mod h1:QLcbU0ipeAqQuR5KSg8c2lql4Qk1EwJ2dWz/0rP4Nho= +github.com/google/go-querystring v1.2.0 h1:yhqkPbu2/OH+V9BfpCVPZkNmUXhb2gBxJArfhIxNtP0= +github.com/google/go-querystring v1.2.0/go.mod h1:8IFJqpSRITyJ8QhQ13bmbeMBDfmeEJZD5A0egEOmkqU= +github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= +github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0 h1:/Tnpcb2E0Pz/tN9s3bfEY2Q8ePCEX9iuS+cneUwncnw= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0/go.mod h1:zOBXOsUaBSjKgmH4OGzV1esUpR3oUSCPYVd2cUBjKYY= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= +github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= +github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= +github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= +github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= +github.com/kevinburke/ssh_config v1.6.0 h1:J1FBfmuVosPHf5GRdltRLhPJtJpTlMdKTBjRgTaQBFY= +github.com/kevinburke/ssh_config v1.6.0/go.mod h1:q2RIzfka+BXARoNexmF9gkxEX7DmvbW9P4hIVx2Kg4M= +github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= +github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/lufia/plan9stats v0.0.0-20260330125221-c963978e514e h1:Q6MvJtQK/iRcRtzAscm/zF23XxJlbECiGPyRicsX+Ak= +github.com/lufia/plan9stats v0.0.0-20260330125221-c963978e514e/go.mod h1:autxFIvghDt3jPTLoqZ9OZ7s9qTGNAWmYCjVFWPX/zg= +github.com/mattn/go-colorable v0.1.15 h1:+u9SLTRGnXv73cEsnsmoZBom+dMU88B2M0aDcWy0/jY= +github.com/mattn/go-colorable v0.1.15/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= +github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsReI= +github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A= +github.com/mattn/go-runewidth v0.0.27 h1:Feg/Oou5zI/wnpgDF6omIU0OokC9GxLC/WRknhVlIR0= +github.com/mattn/go-runewidth v0.0.27/go.mod h1:3qAiGCV4Koz/yuveO58qUefmUTRm8r0IGEXZ9jeHp/8= +github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= +github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= +github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= +github.com/moby/patternmatcher v0.6.1 h1:qlhtafmr6kgMIJjKJMDmMWq7WLkKIo23hsrpR3x084U= +github.com/moby/patternmatcher v0.6.1/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc= +github.com/moby/sys/atomicwriter v0.1.0 h1:kw5D/EqkBwsBFi0ss9v1VG3wIkVhzGvLklJ+w3A14Sw= +github.com/moby/sys/atomicwriter v0.1.0/go.mod h1:Ul8oqv2ZMNHOceF643P6FKPXeCmYtlQMvpizfsSoaWs= +github.com/moby/sys/sequential v0.7.0 h1:ASQNGNROJSuOO6LL6bPHbKvuZu6NU8P4ldPWk31zj/8= +github.com/moby/sys/sequential v0.7.0/go.mod h1:NfSTAp6V3fw4tmkD62PEcOKeZKquXT8VKCkf7aVR79o= +github.com/moby/term v0.5.2 h1:6qk3FJAFDs6i/q3W/pQ97SX192qKfZgGjCQqfCJkgzQ= +github.com/moby/term v0.5.2/go.mod h1:d3djjFCrjnB+fl8NJux+EJzu0msscUP+f8it8hPkFLc= +github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 h1:n6/2gBQ3RWajuToeY6ZtZTIKv2v7ThUy5KKusIT0yc0= +github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00/go.mod h1:Pm3mSP3c5uWn86xMLZ5Sa7JB9GsEZySvHYXCTK4E9q4= +github.com/morikuni/aec v1.1.0 h1:vBBl0pUnvi/Je71dsRrhMBtreIqNMYErSAbEeb8jrXQ= +github.com/morikuni/aec v1.1.0/go.mod h1:xDRgiq/iw5l+zkao76YTKzKttOp2cwPEne25HDkJnBw= +github.com/oklog/ulid/v2 v2.1.2 h1:IEclFb9JNvzYA6MW2SCxbLzcHTVsfqm3PrqGQJH5zec= +github.com/oklog/ulid/v2 v2.1.2/go.mod h1:rcEKHmBBKfef9DhnvX7y1HZBYxjXb0cP5ExxNsTT1QQ= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040= +github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M= +github.com/pborman/getopt v0.0.0-20170112200414-7148bc3a4c30/go.mod h1:85jBQOZwpVEaDAr341tbn15RS4fCAsIst0qp7i8ex1o= +github.com/pjbgf/sha1cd v0.6.0 h1:3WJ8Wz8gvDz29quX1OcEmkAlUg9diU4GxJHqs0/XiwU= +github.com/pjbgf/sha1cd v0.6.0/go.mod h1:lhpGlyHLpQZoxMv8HcgXvZEhcGs0PG/vsZnEJ7H0iCM= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 h1:o4JXh1EVt9k/+g42oCprj/FisM4qX9L3sZB3upGN2ZU= +github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= +github.com/rodaine/protogofakeit v0.1.1 h1:ZKouljuRM3A+TArppfBqnH8tGZHOwM/pjvtXe9DaXH8= +github.com/rodaine/protogofakeit v0.1.1/go.mod h1:pXn/AstBYMaSfc1/RqH3N82pBuxtWgejz1AlYpY1mI0= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= +github.com/santhosh-tekuri/jsonschema/v6 v6.0.3 h1:1EYB5IzjZawrrnELUi78f9fPu57HuXjmddZPjrls/28= +github.com/santhosh-tekuri/jsonschema/v6 v6.0.3/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU= +github.com/sergi/go-diff v1.4.0 h1:n/SP9D5ad1fORl+llWyN+D6qoUETXNZARKjyY2/KVCw= +github.com/sergi/go-diff v1.4.0/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= +github.com/shirou/gopsutil/v3 v3.24.5 h1:i0t8kL+kQTvpAYToeuiVk3TgDeKOFioZO3Ztz/iZ9pI= +github.com/shirou/gopsutil/v3 v3.24.5/go.mod h1:bsoOS1aStSs9ErQ1WWfxllSeS1K5D+U30r2NfcubMVk= +github.com/shoenig/go-m1cpu v0.2.1 h1:yqRB4fvOge2+FyRXFkXqsyMoqPazv14Yyy+iyccT2E4= +github.com/shoenig/go-m1cpu v0.2.1/go.mod h1:KkDOw6m3ZJQAPHbrzkZki4hnx+pDRR1Lo+ldA56wD5w= +github.com/shoenig/test v1.7.0 h1:eWcHtTXa6QLnBvm0jgEabMRN/uJ4DMV3M8xUGgRkZmk= +github.com/shoenig/test v1.7.0/go.mod h1:UxJ6u/x2v/TNs/LoLxBNJRV9DiwBBKYxXSyczsBHFoI= +github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w= +github.com/sirupsen/logrus v1.9.4/go.mod h1:ftWc9WdOfJ0a92nsE2jF5u5ZwH8Bv2zdeOC42RjbV2g= +github.com/skeema/knownhosts v1.3.2 h1:EDL9mgf4NzwMXCTfaxSD/o/a5fxDw/xL9nkU28JjdBg= +github.com/skeema/knownhosts v1.3.2/go.mod h1:bEg3iQAuw+jyiw+484wwFJoKSLwcfd7fqRy+N0QTiow= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.5.3 h1:jmXUvGomnU1o3W/V5h2VEradbpJDwGrzugQQvL0POH4= +github.com/stretchr/objx v0.5.3/go.mod h1:rDQraq+vQZU7Fde9LOZLr8Tax6zZvy4kuNKF+QYS+U0= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE= github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg= +github.com/tklauser/go-sysconf v0.4.0 h1:7H0uAN+7RkwWRaxhYXDLqa5V3LPrJeV8wmD9dRUgPQU= +github.com/tklauser/go-sysconf v0.4.0/go.mod h1:8mTNWyog7H+MpKijp4VmKJAd2bbYQ2zuUwkYRbUArPI= +github.com/tklauser/numcpus v0.12.0 h1:NR85qdvHA9pFse3x3weVZ0r0ST8R6l5RHbZrlRaqob4= +github.com/tklauser/numcpus v0.12.0/go.mod h1:ABHeXzJnr/qqwguhClkZKT1/8VABcYrsyUiUGobwWJg= +github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM= +github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw= +github.com/xlab/treeprint v1.2.0 h1:HzHnuAF1plUN2zGlAFHbSQP2qJ0ZAD3XF5XD7OesXRQ= +github.com/xlab/treeprint v1.2.0/go.mod h1:gj5Gd3gPdKtR1ikdDK6fnFLdmIS0X30kTTuNd/WEJu0= +github.com/yoheimuta/go-protoparser/v4 v4.14.2 h1:/P/LlX1CF9NaTWEltGcIZVvNlPbhABuAnBtAWpb3+74= +github.com/yoheimuta/go-protoparser/v4 v4.14.2/go.mod h1:AHNNnSWnb0UoL4QgHPiOAg2BniQceFscPI5X/BZNHl8= +github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= +github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= -go.opentelemetry.io/otel v1.44.0 h1:JjwHmHpA4iZ3wBxluu2fbbE7j4kqlE8jXyAyPXH7HqU= -go.opentelemetry.io/otel v1.44.0/go.mod h1:BMgjTHL9WPRlRjL2oZCBTL4whCGtXch2H4BhOPIAyYc= -go.opentelemetry.io/otel/metric v1.44.0 h1:1w0gILTcHdr3YI+ixLyjemwrVnsMURbTZFrSYCdDdmc= -go.opentelemetry.io/otel/metric v1.44.0/go.mod h1:8O7hanEPBNgEMmybD3s2VBKcgWOCsA6tzHBPODAiquo= -go.opentelemetry.io/otel/sdk v1.44.0 h1:nHYwb9lK+fJPU/dnT6s7W7Z8itMWyqrnVfbheVYrZ58= -go.opentelemetry.io/otel/sdk v1.44.0/go.mod h1:Osuydd3Se74nqjAKxid74N5eC+jfEqfTegHRnq58oK0= -go.opentelemetry.io/otel/sdk/metric v1.44.0 h1:3LlKgI+VjbVsjNRFZJZAJ30WjXC5VkNRks6si09iEfI= -go.opentelemetry.io/otel/sdk/metric v1.44.0/go.mod h1:5B5pMARnXxKhltooO4xUuCBorl65a4EpnTalObqOigA= -go.opentelemetry.io/otel/trace v1.44.0 h1:jxF5CsGYCe74MCRx2X4g7WsY/VBKRqqpNvXlX/6gtIk= -go.opentelemetry.io/otel/trace v1.44.0/go.mod h1:oLl1jrMQAVo6v3GAggN+1VH9VIz9iUSvW53sW1Q8PIE= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.71.0 h1:B2h3uqicet1CT2N5TOFhS+Gq++9i0/CLmaxvhmhtP5s= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.71.0/go.mod h1:dylvB+ZiiwMvsDij9O84Uy7SijLgHMX4mbkncds+4Sw= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.70.0 h1:LMuyCAyfalSjDyjdC65nK6N0zoTT63+E/u95X0JovZI= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.70.0/go.mod h1:085m8qbm4hgc8rZWGDEa4vmyyo2c3nPxUslYUKUIU04= +go.opentelemetry.io/otel v1.46.0 h1:FHt5/CDyVxi/8IM1CH7VE/rRgq3kLHa2mSTVMO8AWyc= +go.opentelemetry.io/otel v1.46.0/go.mod h1:Gj3SEScelsNC45tp4nSxRYlS+f5iez7W8XPMCt905kE= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.46.0 h1:OFnwLJr+pF3iHrlGSzbxyuo6/6HyBlnlN1CWEJmBVcw= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.46.0/go.mod h1:716wFneO0ov19A2beH5hjfh9AK5z/VWNAtDijp1Y0/g= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.45.0 h1:QBajQ2SrwQijzHyZbQlPsuIzpl/ll8DY6wPWsajeGcI= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.45.0/go.mod h1:08ZQLjrPLQ6R4kAXvuOvODEer5Yh4CoFvll5qB2BCI8= +go.opentelemetry.io/otel/metric v1.46.0 h1:yBnkXvgV7AXFILZc5K6IZe/CBFF3OS7BJ8ov6/lj0K8= +go.opentelemetry.io/otel/metric v1.46.0/go.mod h1:iPmdWqifKUdzziPkvvzIJXITl56fQx2mGM/DHLB3/2o= +go.opentelemetry.io/otel/sdk v1.46.0 h1:h5CNQQjEbuQXY/JfZtgt3i7HVFV3aHPO2OAwO2eTYPI= +go.opentelemetry.io/otel/sdk v1.46.0/go.mod h1:GAERFXFt5SYCEB+YiKUbMBeza6UaDH7GmGOZEfh2gSM= +go.opentelemetry.io/otel/sdk/metric v1.46.0 h1:0piZ26EG4RBfebb2jhDH6ERCYHoVWduc3kLgPCwSnSE= +go.opentelemetry.io/otel/sdk/metric v1.46.0/go.mod h1:I1PbKrdVc8Qu8HYVDNtqVIwLwjNrhsV/uFuxfwg8mO4= +go.opentelemetry.io/otel/trace v1.46.0 h1:OULy7ccdJnZtJ0UDYFOIGaCmiWzJ8Vi2G/Rsu60qs1c= +go.opentelemetry.io/otel/trace v1.46.0/go.mod h1:J7GAXweO77XSFkB/rmAqk9D6ihszhFjLU+d9WuUxDLI= +go.opentelemetry.io/proto/otlp v1.11.0 h1:5rrYs0Ykyj50sdU/JU0x8etU+LubXWb+gED6TbEdMIk= +go.opentelemetry.io/proto/otlp v1.11.0/go.mod h1:SmVizdCOAm3XBtG1g1NnOdhW6jtddT72hLMhv8VwA8E= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI= +go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU= go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= -golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8= -golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww= -golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY= -golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= -golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc= -golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38= +golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M= +golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA= +golang.org/x/exp v0.0.0-20260820142414-ca536658362e h1:01Ju2A/fZKkci4zqx0eZxw//DnRYOnBiGJG14hFBhO8= +golang.org/x/exp v0.0.0-20260820142414-ca536658362e/go.mod h1:zeBbvyFKDaLwa7CH/zI8KXt7gTl14SF7sO08Pl5jBCM= +golang.org/x/mod v0.41.0 h1:qJmnOUb4YB+FsEuM3HcWucdZASCPGhsX6uljO6pog0c= +golang.org/x/mod v0.41.0/go.mod h1:Ek9pY8RKWXwsWvd3rQiHYtMqkjSUV+s1Rj7j4H5Ur6o= +golang.org/x/net v0.60.0 h1:79p50tfZlm0J9YfoDsSi639qSXNGVwEzOPLCxM2FsYU= +golang.org/x/net v0.60.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg= +golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= +golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= +golang.org/x/term v0.46.0 h1:3+OXuTbaKDgwk8jTi3aSLHRlmWqHEUDUtxnbFigO4YE= +golang.org/x/term v0.46.0/go.mod h1:+K02xbkittuwc0Am4abfA3Fc+XRGXkvBXNO88NCXPoc= +golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI= +golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E= +golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= +golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa h1:mZHHdPZl0dbGHCflZgAq/Q468DWVFcU2whhB2KAo8fk= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= -google.golang.org/grpc v1.83.1 h1:HIO0+BEtBP6soyqvqC8sNUjZ7bTs+0hFQuFF+RAy++Y= -google.golang.org/grpc v1.83.1/go.mod h1:kDyl6SKsiHKt0uylY5gtn5cEjkrIOhQOGDgIc4JGwzQ= +google.golang.org/genproto/googleapis/api v0.0.0-20260819154853-08b0e4226688 h1:ax2KzoSRIZU/M0cIxri3pKxy99vniH1PVxWC6si/eZI= +google.golang.org/genproto/googleapis/api v0.0.0-20260819154853-08b0e4226688/go.mod h1:1RJ9BQGyNdZwkGc1eTqkErfRZ6RJyYPHZo73BZ1vQqI= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260825221802-da73d73af1c5 h1:1VUiZAXyC+zmiFYi+WLtBzr68Cj8wOofHjjrA/kkizc= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260825221802-da73d73af1c5/go.mod h1:DjtHYE8FKJLivXcBEjGwndXfIC23G0VpXiXKqG179uA= +google.golang.org/grpc v1.83.2 h1:EManeRomTObA0BU7I8vXgg/78uE5MJ9M8B39EX2WscU= +google.golang.org/grpc v1.83.2/go.mod h1:YPI1hK3kDked6iHvgX3tR0y+nX/qpMFKhPgFsokw1S8= google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc= google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/evanphx/json-patch.v4 v4.13.0 h1:czT3CmqEaQ1aanPc5SdlgQrrEIb8w/wwCvWWnfEbYzo= +gopkg.in/evanphx/json-patch.v4 v4.13.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= +gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME= +gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gotest.tools/v3 v3.5.1 h1:EENdUnS3pdur5nybKYIh2Vfgc8IUNBjxDPSjtiJcOzU= +gotest.tools/v3 v3.5.1/go.mod h1:isy3WKz7GK6uNw/sbHzfKBLvlvXwUyV06n6brMxxopU= +k8s.io/kube-openapi v0.0.0-20241212222426-2c72e554b1e7 h1:hcha5B1kVACrLujCKLbr8XWMxCxzQx42DY8QKYJrDLg= +k8s.io/kube-openapi v0.0.0-20241212222426-2c72e554b1e7/go.mod h1:GewRfANuJ70iYzvn+i4lezLDAFzvjxZYK1gn1lWcfas= +sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= +sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= +sigs.k8s.io/kustomize/api v0.21.1 h1:lzqbzvz2CSvsjIUZUBNFKtIMsEw7hVLJp0JeSIVmuJs= +sigs.k8s.io/kustomize/api v0.21.1/go.mod h1:f3wkKByTrgpgltLgySCntrYoq5d3q7aaxveSagwTlwI= +sigs.k8s.io/kustomize/kyaml v0.21.1 h1:IVlbmhC076nf6foyL6Taw4BkrLuEsXUXNpsE+ScX7fI= +sigs.k8s.io/kustomize/kyaml v0.21.1/go.mod h1:hmxADesM3yUN2vbA5z1/YTBnzLJ1dajdqpQonwBL1FQ= +sigs.k8s.io/yaml v1.5.0 h1:M10b2U7aEUY6hRtU870n2VTPgR5RZiL/I6Lcc2F4NUQ= +sigs.k8s.io/yaml v1.5.0/go.mod h1:wZs27Rbxoai4C0f8/9urLZtZtF3avA3gKvGyPdDqTO4= diff --git a/internal/config/config.go b/internal/config/config.go index dafe75e..6597b6d 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -9,8 +9,10 @@ import ( "fmt" "os" "strconv" + "strings" "time" + "github.com/codefly-dev/core/callertoken" "github.com/codefly-dev/service-warehouse/internal/backend" ) @@ -18,6 +20,12 @@ import ( type Config struct { ListenAddr string Backend backend.Config + // AuthToken is the shared secret every caller must present as the + // callertoken.MetadataKey metadata header, whitespace-trimmed: a secret delivered + // through a file or a block scalar arrives with a trailing newline that no + // client would send. Empty means the listener is anonymous, which FromEnv + // resolves only when the operator asked for it with SWH_ALLOW_ANONYMOUS. + AuthToken string } // FromEnv resolves configuration from SWH_* environment variables. @@ -61,6 +69,7 @@ func FromEnv() (Config, error) { MaxQueryBytes: maxQueryBytes, QueryTimeout: queryTimeout, }, + AuthToken: strings.TrimSpace(os.Getenv("SWH_AUTH_TOKEN")), } // Resolve the kind before anything derived from it, so an unimplemented @@ -73,9 +82,32 @@ func FromEnv() (Config, error) { if !reg.SelfContained && cfg.Backend.Database == "" && cfg.Backend.DSN == "" { return Config{}, fmt.Errorf("SWH_DATABASE (or SWH_DSN) is required for backend %q", cfg.Backend.Kind) } + // Last, so a configuration broken in more than one way reports the backend + // first: the token cannot be what stops a server that names no real backend. + if err := checkListenerIsAuthenticated(cfg.AuthToken); err != nil { + return Config{}, err + } return cfg, nil } +// checkListenerIsAuthenticated refuses to resolve a configuration that would +// run SQL against the bound database for anyone who can reach the listen port. +// The listen address says nothing about who that is, so the token is the only +// thing this server can enforce itself. This is authentication only: it asks +// whether the caller holds the deployment's token, never what that caller may +// do — that is settled before the request arrives. +func checkListenerIsAuthenticated(token string) error { + anonymous, err := envBool("SWH_ALLOW_ANONYMOUS") + if err != nil { + return err + } + return callertoken.CheckListener(token, anonymous, callertoken.Listener{ + TokenVar: "SWH_AUTH_TOKEN", + AnonymousVar: "SWH_ALLOW_ANONYMOUS", + Exposure: "can run SQL against the bound database", + }) +} + func env(key, def string) string { if v := os.Getenv(key); v != "" { return v @@ -98,6 +130,21 @@ func envInt(key string, def int64) (int64, error) { return n, nil } +// envBool parses key as a boolean, or returns false when it is unset. A +// malformed value is an error for the same reason as envInt: "ture" must not +// read as the opposite of what the operator typed. +func envBool(key string) (bool, error) { + v := os.Getenv(key) + if v == "" { + return false, nil + } + b, err := strconv.ParseBool(v) + if err != nil { + return false, fmt.Errorf("%s: %q is not a boolean (want true or false)", key, v) + } + return b, nil +} + // envDuration parses key as a Go duration, or returns def when it is unset. A // malformed value is an error for the same reason as envInt — and a unitless // number ("90") is malformed, so it can never be read as a silent zero. diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 6be65d4..552e952 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -28,16 +28,21 @@ func init() { } // clearEnv blanks every SWH_* variable so a test describes the whole -// environment rather than inheriting the developer's shell. +// environment rather than inheriting the developer's shell. The one thing it +// then sets is a token: resolution refuses an unauthenticated listener, and +// that is orthogonal to what most tests here assert. The auth tests clear it +// again to describe their own posture. func clearEnv(t *testing.T) { t.Helper() for _, k := range []string{ "SWH_LISTEN", "SWH_BACKEND", "SWH_DATABASE", "SWH_DATASET", "SWH_LOCATION", "SWH_DSN", "SWH_HOST", "SWH_PORT", "SWH_USER", "SWH_PASSWORD", "SWH_ACCOUNT", "SWH_CREDENTIALS_FILE", "SWH_MAX_QUERY_BYTES", "SWH_QUERY_TIMEOUT", + "SWH_AUTH_TOKEN", "SWH_ALLOW_ANONYMOUS", } { t.Setenv(k, "") } + t.Setenv("SWH_AUTH_TOKEN", "a-per-run-secret") } func TestSelfContainedBackendsNeedNoDatabase(t *testing.T) { @@ -138,3 +143,89 @@ func TestUnsetLimitsFallBackToDefaults(t *testing.T) { require.Zero(t, cfg.Backend.QueryTimeout) require.Zero(t, cfg.Backend.Port) } + +// TestFromEnvRefusesAnUnauthenticatedListener is the startup guard: with +// neither a token nor an explicit opt-out the server does not resolve, and the +// message names both ways out and the exposed capability. +func TestFromEnvRefusesAnUnauthenticatedListener(t *testing.T) { + clearEnv(t) + t.Setenv("SWH_AUTH_TOKEN", "") + + _, err := FromEnv() + require.Error(t, err) + for _, want := range []string{"SWH_AUTH_TOKEN", "can run SQL", "SWH_ALLOW_ANONYMOUS"} { + require.ErrorContains(t, err, want) + } +} + +// A whitespace-only token is not a token: it would install interceptors nobody +// can satisfy, or be taken for a configured credential while naming nothing. +func TestFromEnvTreatsABlankTokenAsAbsent(t *testing.T) { + clearEnv(t) + t.Setenv("SWH_AUTH_TOKEN", " \n") + + _, err := FromEnv() + require.ErrorContains(t, err, "SWH_AUTH_TOKEN is required") +} + +// A secret delivered through a file or a block scalar carries a trailing +// newline. The enforced token must be what a client can actually send, or every +// caller is refused while the server logs that auth is on. +func TestFromEnvTrimsTheToken(t *testing.T) { + clearEnv(t) + t.Setenv("SWH_AUTH_TOKEN", "s3cr3t\n") + + cfg, err := FromEnv() + require.NoError(t, err) + require.Equal(t, "s3cr3t", cfg.AuthToken) +} + +// A token together with the anonymous opt-out is the shape of a half-finished +// migration; picking either silently leaves the operator believing the other. +func TestFromEnvRejectsATokenWithTheAnonymousOptOut(t *testing.T) { + clearEnv(t) + t.Setenv("SWH_ALLOW_ANONYMOUS", "true") + + _, err := FromEnv() + require.ErrorContains(t, err, "mutually exclusive") +} + +func TestFromEnvAcceptsAnExplicitAnonymousOptOut(t *testing.T) { + clearEnv(t) + t.Setenv("SWH_AUTH_TOKEN", "") + t.Setenv("SWH_ALLOW_ANONYMOUS", "true") + + cfg, err := FromEnv() + require.NoError(t, err) + require.Empty(t, cfg.AuthToken) +} + +// A misspelled opt-out is an error, not a silent false: the refusal that +// followed would blame the missing token for a server whose operator did set the +// opt-out, and send them to provision a secret they chose not to have. +func TestFromEnvRejectsAMalformedAnonymousOptOut(t *testing.T) { + for _, value := range []string{"ture", "yes", "2"} { + t.Run(value, func(t *testing.T) { + clearEnv(t) + t.Setenv("SWH_AUTH_TOKEN", "") + t.Setenv("SWH_ALLOW_ANONYMOUS", value) + + _, err := FromEnv() + require.ErrorContains(t, err, "SWH_ALLOW_ANONYMOUS") + require.ErrorContains(t, err, "not a boolean") + }) + } +} + +// A configuration broken in more than one way reports the backend first: the +// operator fixing the token should not then meet an error that was there all +// along. +func TestFromEnvReportsTheBackendBeforeTheToken(t *testing.T) { + clearEnv(t) + t.Setenv("SWH_AUTH_TOKEN", "") + t.Setenv("SWH_BACKEND", "bigquery") + + _, err := FromEnv() + require.ErrorContains(t, err, `unknown backend kind "bigquery"`) + require.NotContains(t, err.Error(), "SWH_AUTH_TOKEN") +} diff --git a/internal/gatewayimage/image.go b/internal/gatewayimage/image.go new file mode 100644 index 0000000..5f58cb9 --- /dev/null +++ b/internal/gatewayimage/image.go @@ -0,0 +1,8 @@ +// Package gatewayimage declares the gateway's shipped platforms and role. +// Core owns subject derivation, scanning, and evidence writing. +package gatewayimage + +// Platforms is held to the publish workflow's explicit platform input by tests. +var Platforms = []string{"linux/amd64", "linux/arm64"} + +const Role = "runtime" diff --git a/main.go b/main.go new file mode 100644 index 0000000..72258d5 --- /dev/null +++ b/main.go @@ -0,0 +1,224 @@ +// Command warehouse is the codefly service agent that runs the warehouse +// gateway as a real service: a container speaking the codefly/warehouse/v0 gRPC +// contract, backed by the embedded engine locally and a cloud warehouse when +// deployed. Consumers depend on it as a gRPC service and never link a warehouse +// SDK — the gateway absorbs every backend difference. +// +// The agent mirrors the shape of the other codefly infrastructure agents +// (codefly-dev/service-object-storage, service-postgres): a Service advertises +// its configuration surface, a Runtime brings the container up for `codefly run` +// and tests, and a Builder emits the Kubernetes deployment and serves the +// image's SBOM evidence. The gateway itself is cmd/service-warehouse; this +// package is only what lets a workspace compose it. +package main + +import ( + "context" + "embed" + "fmt" + "strings" + + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + + "github.com/codefly-dev/core/agents" + "github.com/codefly-dev/core/agents/services" + "github.com/codefly-dev/core/builders" + basev0 "github.com/codefly-dev/core/generated/go/codefly/base/v0" + agentv0 "github.com/codefly-dev/core/generated/go/codefly/services/agent/v0" + "github.com/codefly-dev/core/resources" + runnersbase "github.com/codefly-dev/core/runners/base" + "github.com/codefly-dev/core/shared" + "github.com/codefly-dev/core/standards" + "github.com/codefly-dev/core/templates" +) + +// agent is loaded from the embedded agent.codefly.yaml (name + version). +var agent = shared.Must(resources.LoadFromFs[resources.Agent](shared.Embed(infoFS))) + +var requirements = builders.NewDependencies(agent.Name, + builders.NewDependency("service.codefly.yaml"), +) + +// configurationGroup is the name of the configuration group this service +// exports to its consumers and reads its own deployment values from. +const configurationGroup = "warehouse" + +// Settings are the service.codefly.yaml knobs. They are the local defaults; +// runtime configuration (an environment's values) takes precedence. Credentials +// are never settings: they arrive as runtime configuration, never here. +type Settings struct { + // Backend selects the warehouse. Create defaults new services to mem, the + // catalog-only backend. An explicitly empty setting uses the gateway's + // default, currently the unsupported DuckDB skeleton. + Backend string `yaml:"backend"` + // Database is the database the gateway is bound to (a BigQuery project, a + // Snowflake or Redshift database, an embedded engine's file). + Database string `yaml:"database"` + // Dataset is the default namespace for unqualified names. + Dataset string `yaml:"dataset"` + // WarehouseLocation is the region datasets and jobs live in, for the + // warehouses that have one. It is not named Location because services.Base + // already has a Location, the service's directory. + WarehouseLocation string `yaml:"location"` +} + +// resolved holds the effective configuration after defaults and runtime +// configuration are applied. The keys are the gateway's own SWH_* variables. +type resolved struct { + backend string + database string + dataset string + location string + // credentialsFile names a warehouse credentials file. Nothing delivers one: + // a local run would have to project it into the gateway container, and a + // deployment mounts nothing, so the Runtime and the Builder both refuse a + // configured path rather than name a file the gateway cannot open. + credentialsFile string + authToken string +} + +type Service struct { + *services.Base + *Settings + + conf resolved + + // gatewayToken is the secret every caller must present to the gateway. The + // Runtime generates one per run and hands it to consumers through the + // configuration channel as a secret value — never inside the connection + // string. It lives outside resolved because LoadConfiguration rebuilds + // resolved from the incoming configuration on every call. + gatewayToken string + + GrpcEndpoint *basev0.Endpoint +} + +func NewService() *Service { + return &Service{ + Base: services.NewServiceBase(context.Background(), agent.Of(resources.ServiceAgent)), + Settings: &Settings{}, + } +} + +func (s *Service) GetAgentInformation(ctx context.Context, _ *agentv0.AgentInformationRequest) (*agentv0.AgentInformation, error) { + readme, err := templates.ApplyTemplateFrom(ctx, shared.Embed(readmeFS), "templates/agent/README.md", s.Information) + if err != nil { + return nil, status.Error(codes.Internal, err.Error()) + } + // Validation is left unset on purpose: this agent serves image-scope SBOM + // (Builder.SBOM) without advertising ValidationCapabilities.image_sbom. A + // present Validation is authoritative for every operation, including the ones + // it omits, and advertising the capability is a separate decision from + // serving the scope: core's docs/sbom.md holds it back until every consumer + // that evaluates it runs a core whose phase bound accepts it. + // TestImageSBOMIsServedButNotAdvertised pins both halves. + return services.Advertisement{ + Backends: runnersbase.BackendSupport{Docker: true}, + ReadMe: readme, + Config: []*agentv0.ConfigurationValueDetail{ + { + Name: configurationGroup, Description: "warehouse gateway endpoint", + Fields: []*agentv0.ConfigurationValueInformation{ + {Name: "connection", Description: "grpc connection string"}, + {Name: "endpoint", Description: "host:port of the gRPC endpoint"}, + {Name: "token", Description: "shared secret sent as the x-codefly-token grpc metadata header"}, + }, + }, + }, + }.Build(), nil +} + +// resolveServingGRPCEndpoint selects the single gRPC endpoint the agent binds +// its runtime and deployment to. +func resolveServingGRPCEndpoint(ctx context.Context, endpoints []*basev0.Endpoint) (*basev0.Endpoint, error) { + grpc := resources.FindEndpointsByAPI(ctx, standards.GRPC, endpoints) + if len(grpc) == 0 { + return nil, fmt.Errorf("no grpc endpoint found") + } + return grpc[0], nil +} + +// LoadConfiguration resolves the effective configuration. Settings are the local +// defaults; runtime configuration (deployed values) takes precedence. +func (s *Service) LoadConfiguration(ctx context.Context, conf *basev0.Configuration) error { + r := resolved{ + backend: s.Backend, + database: s.Database, + dataset: s.Dataset, + location: s.WarehouseLocation, + } + if conf != nil { + for key, dst := range map[string]*string{ + "SWH_BACKEND": &r.backend, + "SWH_DATABASE": &r.database, + "SWH_DATASET": &r.dataset, + "SWH_LOCATION": &r.location, + "SWH_CREDENTIALS_FILE": &r.credentialsFile, + "SWH_AUTH_TOKEN": &r.authToken, + } { + v, err := resources.GetConfigurationValue(ctx, conf, configurationGroup, key) + if err == nil && v != "" { + *dst = v + } + } + } + // Normalized at the single boundary where it enters, for the same reason the + // gateway trims its own env read: a secret carrying a trailing newline would + // otherwise be handed to consumers in one form and enforced in another. The + // rest arrive through YAML block scalars and env files just the same, and a + // stray newline in one is not a value but a malformed manifest. + for _, field := range []*string{&r.backend, &r.database, &r.dataset, &r.location, &r.credentialsFile, &r.authToken} { + *field = strings.TrimSpace(*field) + } + s.conf = r + return nil +} + +func (s *Service) createConnectionString(address string) string { + return fmt.Sprintf("grpc://%s", address) +} + +func (s *Service) CreateConnectionConfiguration(ctx context.Context, conf *basev0.Configuration, instance *basev0.NetworkInstance) (*basev0.Configuration, error) { + defer s.Wool.Catch() + ctx = s.Wool.Inject(ctx) + + if err := s.LoadConfiguration(ctx, conf); err != nil { + return nil, s.Wool.Wrapf(err, "cannot load configuration") + } + values := []*basev0.ConfigurationValue{ + {Key: "connection", Value: s.createConnectionString(instance.Address)}, + {Key: "endpoint", Value: instance.Address}, + } + // The token travels as its own secret value: putting it in the connection + // string would leak it into every log line and manifest that carries an + // endpoint. + if s.gatewayToken != "" { + values = append(values, &basev0.ConfigurationValue{Key: "token", Value: s.gatewayToken, Secret: true}) + } + return &basev0.Configuration{ + Origin: s.Base.Unique(), + RuntimeContext: resources.RuntimeContextFromInstance(instance), + Infos: []*basev0.ConfigurationInformation{ + {Name: configurationGroup, ConfigurationValues: values}, + }, + }, nil +} + +func main() { + svc := NewService() + agents.Serve(agents.PluginRegistration{ + Agent: svc, + Runtime: NewRuntime(), + Builder: NewBuilder(), + }) +} + +//go:embed agent.codefly.yaml +var infoFS embed.FS + +//go:embed gateway-image.json +var gatewayImageLockJSON []byte + +//go:embed templates/agent +var readmeFS embed.FS diff --git a/publisher_test.go b/publisher_test.go new file mode 100644 index 0000000..4d3488d --- /dev/null +++ b/publisher_test.go @@ -0,0 +1,184 @@ +package main + +import ( + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/codefly-dev/core/resources" + "github.com/stretchr/testify/require" + "gopkg.in/yaml.v3" +) + +type publisherStep struct { + ID string `yaml:"id"` + Uses string `yaml:"uses"` + Run string `yaml:"run"` + Env map[string]string `yaml:"env"` + With map[string]any `yaml:"with"` +} + +type publisherJob struct { + Uses string `yaml:"uses"` + If string `yaml:"if"` + Permissions map[string]string `yaml:"permissions"` + Steps []publisherStep `yaml:"steps"` +} + +func loadPublisher(t *testing.T) publisherJob { + t.Helper() + var workflow struct { + Jobs map[string]publisherJob `yaml:"jobs"` + } + require.NoError(t, yaml.Unmarshal([]byte(workflowBody(t, "publish-gateway-image.yml")), &workflow)) + require.Len(t, workflow.Jobs, 1) + return workflow.Jobs["publish"] +} + +func publisherStepByID(t *testing.T, id string) publisherStep { + t.Helper() + for _, step := range loadPublisher(t).Steps { + if step.ID == id { + return step + } + } + t.Fatalf("publisher has no %s step", id) + return publisherStep{} +} + +// GHCR checks the defining repo. A job-level reusable workflow call broke the +// live object-storage release even with a caller-supplied registry token. +func TestPublisherKeepsRegistryAuthorityInServiceRepository(t *testing.T) { + job := loadPublisher(t) + require.Empty(t, job.Uses, "the publisher job must be defined in this repository") + require.Equal(t, "github.event_name == 'workflow_dispatch'", job.If) + require.Equal(t, "write", job.Permissions["packages"]) + require.Equal(t, "read", job.Permissions["contents"]) + guard, login, build, artifact := -1, -1, -1, -1 + for i, step := range job.Steps { + if strings.HasPrefix(step.Uses, "actions/checkout@") { + require.Equal(t, false, step.With["persist-credentials"]) + } + if step.ID == "default-branch" { + guard = i + } + if strings.HasPrefix(step.Uses, "docker/login-action@") { + login = i + require.Equal(t, "${{ secrets.GITHUB_TOKEN }}", step.With["password"]) + require.Equal(t, "ghcr.io", step.With["registry"]) + } + if strings.HasPrefix(step.Uses, "docker/build-push-action@") { + build = i + require.NotContains(t, step.With, "tags") + require.Contains(t, step.With["outputs"], "push-by-digest=true") + require.Contains(t, step.With["outputs"], "push=true") + require.Contains(t, step.With["labels"], "org.opencontainers.image.version=${{ steps.inputs.outputs.version }}") + } + if strings.HasPrefix(step.Uses, "actions/upload-artifact@") { + artifact = i + require.Equal(t, "release-evidence/gateway-image.json", step.With["path"]) + require.Equal(t, "error", step.With["if-no-files-found"]) + } + } + require.GreaterOrEqual(t, guard, 0) + require.Greater(t, login, guard, "refuse unmerged code before registry login") + require.Greater(t, build, login) + require.Greater(t, artifact, build) +} + +// Execute the workflow's actual guard against a local origin. The remote's +// default branch is deliberately not called main, and the supplied ref name +// lies: authority must come from the remote's HEAD and the commit's ancestry. +func TestPublisherRefusesCommitOutsideDefaultBranch(t *testing.T) { + dir := t.TempDir() + git := func(at string, args ...string) string { + t.Helper() + command := exec.Command("git", args...) + command.Dir = at + out, err := command.CombinedOutput() + require.NoError(t, err, "%s", out) + return strings.TrimSpace(string(out)) + } + git(dir, "init", "--quiet", "--initial-branch=trunk") + git(dir, "config", "commit.gpgsign", "false") + git(dir, "config", "core.hooksPath", "/dev/null") + git(dir, "config", "user.name", "publisher fixture") + git(dir, "config", "user.email", "fixture@example.invalid") + git(dir, "commit", "--quiet", "--allow-empty", "-m", "reviewed") + reviewed := git(dir, "rev-parse", "HEAD") + remote := t.TempDir() + git(remote, "init", "--bare", "--quiet", "--initial-branch=trunk") + git(dir, "remote", "add", "origin", remote) + git(dir, "push", "--quiet", "origin", "trunk") + git(dir, "switch", "--quiet", "-c", "unmerged") + git(dir, "commit", "--quiet", "--allow-empty", "-m", "unreviewed") + unmerged := git(dir, "rev-parse", "HEAD") + + guard := publisherStepByID(t, "default-branch") + require.Empty(t, guard.Env, "public git reads must receive no token") + require.NotContains(t, guard.Run, "$GH_TOKEN") + require.NotContains(t, guard.Run, "$GITHUB_TOKEN") + require.NotContains(t, guard.Run, "extraheader") + run := func(sha string) (string, error) { + command := exec.Command("bash", "-c", guard.Run) + command.Dir = dir + command.Env = append(os.Environ(), "GITHUB_SHA="+sha, "GITHUB_REF_NAME=trunk") + out, err := command.CombinedOutput() + return string(out), err + } + out, err := run(reviewed) + require.NoError(t, err, "%s", out) + out, err = run(unmerged) + require.Error(t, err, "%s", out) + require.Contains(t, out, "not reachable from refs/remotes/origin/trunk") + git(remote, "symbolic-ref", "HEAD", "refs/heads/missing") + out, err = run(reviewed) + require.Error(t, err, "%s", out) + require.Contains(t, out, "cannot read this repository's default branch") + config, err := os.ReadFile(filepath.Join(dir, ".git", "config")) + require.NoError(t, err) + require.NotContains(t, string(config), "extraheader", "authentication must not persist in the checkout") +} + +func TestPublisherValidatesVersionAndRecordsStrictLock(t *testing.T) { + run := func(step publisherStep, env ...string) (string, error) { + t.Helper() + dir := t.TempDir() + command := exec.Command("bash", "-c", step.Run) + command.Dir = dir + command.Env = append(os.Environ(), append(env, + "GITHUB_OUTPUT="+filepath.Join(dir, "outputs"), + "GITHUB_STEP_SUMMARY="+filepath.Join(dir, "summary"))...) + out, err := command.CombinedOutput() + if err != nil { + return dir, err + } + require.NotContains(t, string(out), "error") + return dir, nil + } + inputs := publisherStepByID(t, "inputs") + dir, err := run(inputs, "REPOSITORY=codefly-dev/service-warehouse", "VERSION=0.0.2") + require.NoError(t, err) + output, err := os.ReadFile(filepath.Join(dir, "outputs")) + require.NoError(t, err) + require.Contains(t, string(output), "image=ghcr.io/codefly-dev/service-warehouse\nversion=0.0.2\n") + for _, version := range []string{"", "v0.0.2", "0.0.2-rc1", "0.0.2\nimage=other"} { + _, err = run(inputs, "REPOSITORY=codefly-dev/service-warehouse", "VERSION="+version) + require.Error(t, err, "version %q must be rejected", version) + } + lock := publisherStepByID(t, "lock") + valid := "sha256:" + strings.Repeat("a", 64) + dir, err = run(lock, "IMAGE=ghcr.io/codefly-dev/service-warehouse", "DIGEST="+valid) + require.NoError(t, err) + content, err := os.ReadFile(filepath.Join(dir, "release-evidence", "gateway-image.json")) + require.NoError(t, err) + image, err := resources.ParseImageLock(content, gatewayImageName) + require.NoError(t, err) + require.Equal(t, "ghcr.io/codefly-dev/service-warehouse@"+valid, image.FullName()) + for _, digest := range []string{"", "sha256:" + strings.Repeat("A", 64), "sha256:abc"} { + _, err = run(lock, "IMAGE=ghcr.io/codefly-dev/service-warehouse", "DIGEST="+digest) + require.Error(t, err, "digest %q must be rejected", digest) + } +} diff --git a/readiness_test.go b/readiness_test.go new file mode 100644 index 0000000..bdb12a7 --- /dev/null +++ b/readiness_test.go @@ -0,0 +1,170 @@ +package main + +import ( + "context" + "net" + "testing" + "time" + + "github.com/codefly-dev/core/callertoken" + basev0 "github.com/codefly-dev/core/generated/go/codefly/base/v0" + runtimev0 "github.com/codefly-dev/core/generated/go/codefly/services/runtime/v0" + "github.com/codefly-dev/core/resources" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/credentials/insecure" + grpchealth "google.golang.org/grpc/health" + healthv1 "google.golang.org/grpc/health/grpc_health_v1" + + whv0 "github.com/codefly-dev/service-warehouse/gen/codefly/warehouse/v0" + "github.com/codefly-dev/service-warehouse/internal/backend" + "github.com/codefly-dev/service-warehouse/internal/backend/mem" + "github.com/codefly-dev/service-warehouse/internal/server" +) + +// startGateway serves the gateway's gRPC surface — the Warehouse service behind +// the token interceptors, the health service outside them — on a loopback port +// and returns its address and its health server. cmd/service-warehouse builds +// the same server from the same pieces; it is rebuilt here because that +// constructor is in a main package. +func startGateway(t *testing.T, token string) (string, *grpchealth.Server) { + t.Helper() + lis, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + health := grpchealth.NewServer() + srv := grpc.NewServer( + grpc.ChainUnaryInterceptor(callertoken.UnaryServerInterceptor(token)), + grpc.ChainStreamInterceptor(callertoken.StreamServerInterceptor(token)), + ) + healthv1.RegisterHealthServer(srv, health) + whv0.RegisterWarehouseServer(srv, server.New(mem.New(backend.Config{}))) + go func() { _ = srv.Serve(lis) }() + t.Cleanup(srv.Stop) + return lis.Addr().String(), health +} + +// runtimeAt wires a Runtime with just enough network state for WaitForReady to +// resolve address. +func runtimeAt(t *testing.T, address string, token string) *Runtime { + t.Helper() + rt := NewRuntime() + endpoint := &basev0.Endpoint{Name: "grpc", Api: "grpc"} + rt.GrpcEndpoint = endpoint + rt.Runtime.WithContext(resources.NewRuntimeContextNative()) + rt.NetworkMappings = []*basev0.NetworkMapping{{ + Endpoint: endpoint, + Instances: []*basev0.NetworkInstance{{Address: address, Access: resources.NewNativeNetworkAccess()}}, + }} + rt.gatewayToken = token + return rt +} + +// A readiness probe that swallowed its own error would retry a token mismatch +// for the whole budget and then report "not ready", pointing the operator at the +// gateway instead of at the credential. +func TestWaitForReadyReportsRejectedCredential(t *testing.T) { + address, _ := startGateway(t, "server-token") + rt := runtimeAt(t, address, "a-different-token") + + start := time.Now() + err := rt.WaitForReady(context.Background()) + + require.Error(t, err, "WaitForReady must fail when the gateway rejects the agent's credential") + require.Contains(t, err.Error(), "rejected the agent's credential") + require.Less(t, time.Since(start), 10*time.Second, "a rejected credential must not be retried as if transient") + + rt.gatewayToken = "server-token" + require.NoError(t, rt.WaitForReady(context.Background())) +} + +// The health service answers without a token, so a gateway that is up but +// reports itself not serving must not be called ready on the strength of +// Capabilities alone; once it serves, the wait ends. +func TestWaitForReadyWaitsForTheGatewayToServe(t *testing.T) { + address, health := startGateway(t, "server-token") + health.SetServingStatus("", healthv1.HealthCheckResponse_NOT_SERVING) + rt := runtimeAt(t, address, "server-token") + + go func() { + time.Sleep(1500 * time.Millisecond) + health.SetServingStatus("", healthv1.HealthCheckResponse_SERVING) + }() + start := time.Now() + require.NoError(t, rt.WaitForReady(context.Background())) + require.GreaterOrEqual(t, time.Since(start), time.Second, "ready was reported while the gateway said it was not serving") +} + +func TestProbeGatewayDistinguishesNotYetFromNever(t *testing.T) { + address, health := startGateway(t, "server-token") + conn, err := grpc.NewClient(address, grpc.WithTransportCredentials(insecureCredentials()), callertoken.DialOption("server-token")) + require.NoError(t, err) + defer conn.Close() + healthClient, warehouse := healthv1.NewHealthClient(conn), whv0.NewWarehouseClient(conn) + + reason, final := probeGateway(context.Background(), healthClient, warehouse) + require.Empty(t, reason) + require.NoError(t, final) + + health.SetServingStatus("", healthv1.HealthCheckResponse_NOT_SERVING) + reason, final = probeGateway(context.Background(), healthClient, warehouse) + require.Contains(t, reason, "NOT_SERVING") + require.NoError(t, final, "a gateway that is not serving yet is worth retrying") +} + +func TestWaitForReadyReturnsOnCancellation(t *testing.T) { + rt := runtimeAt(t, "127.0.0.1:1", "token") // nothing listens here + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(1500 * time.Millisecond) + cancel() + }() + + start := time.Now() + err := rt.WaitForReady(ctx) + + require.Error(t, err) + require.Less(t, time.Since(start), 10*time.Second, "cancellation must end the wait, not the 60s budget") +} + +func insecureCredentials() credentials.TransportCredentials { return insecure.NewCredentials() } + +// The CLI judges Result, not legacy Status. A successful readiness assertion +// must not decode as UNKNOWN, and a failed assertion must report FAILED. +func TestRuntimeTestReportsRunLevelOutcome(t *testing.T) { + address, health := startGateway(t, "server-token") + for _, tc := range []struct { + name string + serving healthv1.HealthCheckResponse_ServingStatus + want runtimev0.TestRunResult_State + passed, failed int32 + }{ + {"serving", healthv1.HealthCheckResponse_SERVING, runtimev0.TestRunResult_PASSED, 1, 0}, + {"not serving", healthv1.HealthCheckResponse_NOT_SERVING, runtimev0.TestRunResult_FAILED, 0, 1}, + } { + t.Run(tc.name, func(t *testing.T) { + health.SetServingStatus("", tc.serving) + rt := runtimeAt(t, address, "server-token") + budget := 5 * time.Second + if tc.failed > 0 { + budget = 200 * time.Millisecond + } + ctx, cancel := context.WithTimeout(context.Background(), budget) + defer cancel() + response, err := rt.Test(ctx, &runtimev0.TestRequest{}) + require.NoError(t, err) + require.Equal(t, tc.want, response.GetResult().GetState()) + require.EqualValues(t, 1, response.GetCounts().GetTotal()) + require.Equal(t, tc.passed, response.GetCounts().GetPassed()) + require.Equal(t, tc.failed, response.GetCounts().GetFailed()) + require.Equal(t, tc.passed, response.GetTestsPassed()) + require.Equal(t, tc.failed, response.GetTestsFailed()) + if tc.failed > 0 { + require.Equal(t, runtimev0.TestStatus_ERROR, response.GetStatus().GetState()) + require.Contains(t, response.GetResult().GetMessage(), "not ready") + } else { + require.Equal(t, runtimev0.TestStatus_SUCCESS, response.GetStatus().GetState()) + } + }) + } +} diff --git a/release_gate_test.go b/release_gate_test.go new file mode 100644 index 0000000..8ed82fe --- /dev/null +++ b/release_gate_test.go @@ -0,0 +1,59 @@ +package main + +import ( + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestUnpublishedImageExceptionEndsAtFirstLockOrRelease(t *testing.T) { + script, err := filepath.Abs(".github/scripts/allow-unpublished-image.sh") + require.NoError(t, err) + for _, scenario := range []string{"unpublished", "previous lock", "release tag", "shallow history"} { + t.Run(scenario, func(t *testing.T) { + dir := t.TempDir() + git := func(args ...string) string { + t.Helper() + command := exec.Command("git", args...) + command.Dir = dir + output, err := command.CombinedOutput() + require.NoError(t, err, "%s", output) + return string(output) + } + git("init", "--quiet") + git("config", "commit.gpgsign", "false") + git("config", "tag.gpgsign", "false") + git("config", "core.hooksPath", "/dev/null") + git("config", "user.name", "gate fixture") + git("config", "user.email", "fixture@example.invalid") + commit := func(digest string) { + t.Helper() + require.NoError(t, os.WriteFile(filepath.Join(dir, "gateway-image.json"), []byte(`{"name":"registry.example/warehouse","digest":"`+digest+`"}`), 0600)) + git("add", "gateway-image.json") + git("commit", "--quiet", "-m", "fixture") + } + commit("") + switch scenario { + case "previous lock": + commit("sha256:" + strings.Repeat("a", 64)) + commit("") + case "release tag": + git("tag", "v0.0.2") // This tag exists only in the disposable test repository. + case "shallow history": + require.NoError(t, os.WriteFile(filepath.Join(dir, ".git", "shallow"), []byte(git("rev-parse", "HEAD")), 0600)) + } + command := exec.Command("bash", script) + command.Dir = dir + output, err := command.CombinedOutput() + if scenario == "unpublished" { + require.NoError(t, err, "%s", output) + } else { + require.Error(t, err, "%s", output) + } + }) + } +} diff --git a/runtime.go b/runtime.go new file mode 100644 index 0000000..2fa673a --- /dev/null +++ b/runtime.go @@ -0,0 +1,361 @@ +package main + +import ( + "context" + "fmt" + "os" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials/insecure" + healthv1 "google.golang.org/grpc/health/grpc_health_v1" + "google.golang.org/grpc/status" + + basev0 "github.com/codefly-dev/core/generated/go/codefly/base/v0" + runtimev0 "github.com/codefly-dev/core/generated/go/codefly/services/runtime/v0" + + "github.com/codefly-dev/core/agents/services" + "github.com/codefly-dev/core/callertoken" + "github.com/codefly-dev/core/resources" + dockerrun "github.com/codefly-dev/core/runners/dockerrun" + "github.com/codefly-dev/core/wool" + + whv0 "github.com/codefly-dev/service-warehouse/gen/codefly/warehouse/v0" +) + +// gatewayContainerPort is the port the gateway listens on inside its container. +// It is handed to the gateway as SWH_LISTEN and to the Kubernetes templates as +// the container port; TestGatewayContainerPortIsTheGatewaysDefault holds it to +// the gateway's own default and to the Dockerfile's EXPOSE. +const gatewayContainerPort = 9465 + +// gatewayContainerUser is the unprivileged uid:gid the gateway runs as. The +// published image already declares it, but SWH_GATEWAY_IMAGE can name another +// one; setting the user explicitly makes the guarantee hold for whichever image +// is run. +const gatewayContainerUser = "65532:65532" + +// readinessBudget bounds the whole wait for the gateway in Start and Test: one overall +// deadline, so a slow attempt spends the budget instead of extending it. +const readinessBudget = 60 * time.Second + +// readinessProbeTimeout bounds a single attempt, and readinessPollInterval paces +// the retries between them. +const ( + readinessProbeTimeout = 5 * time.Second + readinessPollInterval = time.Second +) + +// containerEnvironment is the part of dockerrun.DockerEnvironment the Runtime +// owns: something it can identify and tear down. Teardown is the path that has +// to keep working when Docker does not, so it is reachable through this seam. +type containerEnvironment interface { + ContainerID() (string, error) + Shutdown(context.Context) error +} + +type Runtime struct { + *services.DefaultRuntime + *Service + + gatewayEnv containerEnvironment +} + +func NewRuntime() *Runtime { + service := NewService() + return &Runtime{ + DefaultRuntime: services.NewDefaultRuntime(service.Runtime), + Service: service, + } +} + +func (s *Runtime) Load(ctx context.Context, req *runtimev0.LoadRequest) (*runtimev0.LoadResponse, error) { + defer s.Wool.Catch() + return s.Runtime.LoadService(ctx, req, services.RuntimeLoad{ + Settings: s.Settings, + Requirements: requirements, + ResolveEndpoints: func(ctx context.Context, endpoints []*basev0.Endpoint) error { + endpoint, err := resolveServingGRPCEndpoint(ctx, endpoints) + if err != nil { + return s.Wool.Wrapf(err, "cannot find grpc endpoint") + } + s.GrpcEndpoint = endpoint + return nil + }, + }) +} + +func (s *Runtime) Init(ctx context.Context, req *runtimev0.InitRequest) (*runtimev0.InitResponse, error) { + defer s.Wool.Catch() + ctx = s.Wool.Inject(ctx) + + s.Runtime.LogInitRequest(req) + s.Runtime.WithContext(req.GetRuntimeContext()) + s.NetworkMappings = req.ProposedNetworkMappings + + w := s.Wool.In("runtime::init") + configuration := req.GetConfiguration() + + net, err := resources.FindNetworkMapping(ctx, s.NetworkMappings, s.GrpcEndpoint) + if err != nil { + return s.Runtime.InitError(err) + } + instance, err := resources.FindNetworkInstanceInNetworkMappings(ctx, s.NetworkMappings, s.GrpcEndpoint, s.Runtime.NetworkAccess()) + if err != nil { + return s.Runtime.InitError(err) + } + + if err = s.LoadConfiguration(ctx, configuration); err != nil { + return s.Runtime.InitError(err) + } + // Refuse before any container exists: a path the gateway cannot open must + // fail here with its own cause, not as a gateway that comes up and cannot + // authenticate to its warehouse. + if s.conf.credentialsFile != "" { + return s.Runtime.InitError(s.Wool.NewError("SWH_CREDENTIALS_FILE cannot be delivered to a local run: "+ + "nothing here projects %s into the gateway container, which would name a path it cannot open. "+ + "A local run is for the embedded engine, which needs no credential; "+ + "a cloud warehouse is reached from a deployment", s.conf.credentialsFile)) + } + + s.gatewayToken, err = s.resolveGatewayToken() + if err != nil { + return s.Runtime.InitError(err) + } + + if err = s.startGateway(ctx, uint16(instance.Port)); err != nil { + s.rollbackInit(ctx) + return s.Runtime.InitError(err) + } + + for _, inst := range net.Instances { + conf, errConn := s.CreateConnectionConfiguration(ctx, configuration, inst) + if errConn != nil { + return s.Runtime.InitError(errConn) + } + s.Runtime.RuntimeConfigurations = append(s.Runtime.RuntimeConfigurations, conf) + } + + w.Debug("init successful") + return s.Runtime.InitResponse() +} + +// resolveGatewayToken returns the secret the gateway will enforce and consumers +// will present. A configured token is honored so an operator who pins one is not +// silently overridden; otherwise a fresh one is generated per run, which is what +// stands between the all-interface published port and SQL against the bound +// database, and which revokes every token a previous run handed out. +// +// Generating a new token changes the gateway container's environment, and +// dockerrun fingerprints the environment, so each run deliberately recreates the +// container rather than reattaching to the previous one. Making the token stable +// to recover that reattach would silently give up the revocation property. +func (s *Runtime) resolveGatewayToken() (string, error) { + if s.conf.authToken != "" { + return s.conf.authToken, nil + } + return callertoken.Generate() +} + +// startGateway runs the gateway container, mapping the assigned host port onto +// the gateway's listen port and handing it the resolved configuration. A value +// nobody set is not passed, so the gateway's own default decides it. +func (s *Runtime) startGateway(ctx context.Context, hostPort uint16) error { + image, _, err := effectiveGatewayImage() + if err != nil { + return err + } + + runner, err := dockerrun.NewDockerHeadlessEnvironment(ctx, image, s.UniqueWithWorkspace()+"-gateway") + if err != nil { + return s.Wool.Wrapf(err, "cannot create gateway environment") + } + // Own the environment before starting it: a container that is created and + // then exits still has to be removed, and only Shutdown does that. + s.gatewayEnv = runner + runner.WithOutput(os.Stdout) + runner.WithUser(gatewayContainerUser) + runner.WithPortMapping(ctx, hostPort, gatewayContainerPort) + // Consumers running in their own container reach the gateway through + // host.docker.internal (the Container network instance the agent advertises), + // which resolves to the bridge gateway on Linux — unreachable if the port is + // bound only to 127.0.0.1. Publish on all interfaces; what makes that safe is + // SWH_AUTH_TOKEN below, not the binding. + runner.WithPublicPorts() + envs := []*resources.EnvironmentVariable{ + resources.Env("SWH_LISTEN", fmt.Sprintf(":%d", gatewayContainerPort)), + resources.Env("SWH_AUTH_TOKEN", s.gatewayToken), + } + for _, value := range []struct{ name, value string }{ + {"SWH_BACKEND", s.conf.backend}, + {"SWH_DATABASE", s.conf.database}, + {"SWH_DATASET", s.conf.dataset}, + {"SWH_LOCATION", s.conf.location}, + } { + if value.value != "" { + envs = append(envs, resources.Env(value.name, value.value)) + } + } + runner.WithEnvironmentVariables(ctx, envs...) + if err = runner.Init(ctx); err != nil { + return s.Wool.Wrapf(err, "cannot start gateway") + } + return nil +} + +// teardown releases everything the Runtime owns. An environment that fails to +// shut down keeps its reference so a later attempt can retry it. +func (s *Runtime) teardown(ctx context.Context) error { + if s.gatewayEnv == nil { + return nil + } + if err := s.gatewayEnv.Shutdown(ctx); err != nil { + return s.Wool.Wrapf(err, "cannot shut down gateway") + } + s.gatewayEnv = nil + return nil +} + +// rollbackInit releases what Init already owns after a failed startup. The +// startup error is what the caller reports, so a teardown failure is logged +// rather than substituted for it. +func (s *Runtime) rollbackInit(ctx context.Context) { + if err := s.teardown(ctx); err != nil { + s.Wool.Warn("cannot fully roll back after failed init", wool.ErrField(err)) + } +} + +func (s *Runtime) Start(ctx context.Context, req *runtimev0.StartRequest) (*runtimev0.StartResponse, error) { + defer s.Wool.Catch() + ctx = s.Wool.Inject(ctx) + if err := s.WaitForReady(ctx); err != nil { + return s.Runtime.StartError(err) + } + return s.Runtime.StartResponse() +} + +// WaitForReady polls the gateway until its health service reports it serving +// and it has accepted the agent's credential. +// +// This is liveness plus the credential, and says nothing about the warehouse: +// the gateway does not probe its backend and no RPC reports that, so a gateway +// whose warehouse is unreachable is still ready here and fails the first query +// that needs it (README, "Authentication"). Capabilities is the second probe +// because it is the cheapest call that is not exempt from authentication — the +// health service carries no token — so a mismatched token surfaces here as +// itself rather than at a consumer's first call. +// +// The gateway is addressed through the NATIVE network view. The agent is a host +// process whatever runtime context the service itself was given, and a +// container-only name resolves to nothing from here. +func (s *Runtime) WaitForReady(ctx context.Context) error { + instance, err := resources.FindNetworkInstanceInNetworkMappings(ctx, s.NetworkMappings, s.GrpcEndpoint, resources.NewNativeNetworkAccess()) + if err != nil { + return s.Wool.Wrapf(err, "cannot find network instance") + } + address := instance.Address + s.Wool.Debug("waiting for warehouse gateway", wool.Field("address", address)) + + conn, err := grpc.NewClient(address, + grpc.WithTransportCredentials(insecure.NewCredentials()), + callertoken.DialOption(s.gatewayToken)) + if err != nil { + return s.Wool.Wrapf(err, "cannot create gateway client for %s", address) + } + defer func() { _ = conn.Close() }() + health := healthv1.NewHealthClient(conn) + warehouse := whv0.NewWarehouseClient(conn) + + start := time.Now() + budget, cancel := context.WithTimeout(ctx, readinessBudget) + defer cancel() + ticker := time.NewTicker(readinessPollInterval) + defer ticker.Stop() + + lastErr := "no attempt completed" +poll: + for { + if budget.Err() != nil { + break + } + + attempt, attemptCancel := context.WithTimeout(budget, readinessProbeTimeout) + lastErr, err = probeGateway(attempt, health, warehouse) + attemptCancel() + if err != nil { + return s.Wool.Wrapf(err, "warehouse gateway rejected the agent's credential") + } + if lastErr == "" { + return nil + } + + select { + case <-budget.Done(): + break poll + case <-ticker.C: + } + } + return s.Wool.NewError("warehouse gateway at %s not ready after %s: %s", + address, time.Since(start).Round(time.Millisecond), lastErr) +} + +// probeGateway makes one readiness attempt. A non-empty reason means the gateway +// is not ready yet and the attempt is worth repeating; a non-nil error is final. +// A rejected credential is not a gateway that is still coming up: retrying +// cannot change the answer, and spending the budget on it reports the failure as +// readiness instead of as the mismatched token it is. +func probeGateway(ctx context.Context, health healthv1.HealthClient, warehouse whv0.WarehouseClient) (reason string, final error) { + res, err := health.Check(ctx, &healthv1.HealthCheckRequest{}) + if err != nil { + return err.Error(), nil + } + if res.GetStatus() != healthv1.HealthCheckResponse_SERVING { + return fmt.Sprintf("health status %s", res.GetStatus()), nil + } + if _, err = warehouse.Capabilities(ctx, &whv0.CapabilitiesRequest{}); err != nil { + if status.Code(err) == codes.Unauthenticated { + return "", err + } + return err.Error(), nil + } + return "", nil +} + +func (s *Runtime) Stop(ctx context.Context, req *runtimev0.StopRequest) (*runtimev0.StopResponse, error) { + defer s.Wool.Catch() + return s.Runtime.StopResponse() +} + +func (s *Runtime) Destroy(ctx context.Context, req *runtimev0.DestroyRequest) (*runtimev0.DestroyResponse, error) { + defer s.Wool.Catch() + ctx = s.Wool.Inject(ctx) + if err := s.teardown(ctx); err != nil { + return s.Runtime.DestroyError(err) + } + return s.Runtime.DestroyResponse() +} + +func (s *Runtime) Test(ctx context.Context, req *runtimev0.TestRequest) (*runtimev0.TestResponse, error) { + defer s.Wool.Catch() + // Conformance calls Init then Test without Start. Creating a container is + // not proof that it stayed alive or accepts the credential we handed out. + readinessErr := s.WaitForReady(s.Wool.Inject(ctx)) + passed, failed := int32(1), int32(0) + state := runtimev0.TestRunResult_PASSED + var failures []string + if readinessErr != nil { + passed, failed = 0, 1 + state = runtimev0.TestRunResult_FAILED + failures = []string{readinessErr.Error()} + } + response, err := s.Runtime.TestResponseWithResults(1, passed, failed, 0, 0, failures, readinessErr) + if err != nil { + return nil, err + } + // CLI 0.1.185 judges the run-level verdict; legacy status/counters alone + // decode as UNKNOWN. Keep both representations consistent for older hosts. + response.Result = &runtimev0.TestRunResult{State: state, Message: response.GetStatus().GetMessage()} + response.Counts = &runtimev0.TestCounts{Total: 1, Passed: passed, Failed: failed} + return response, nil +} diff --git a/runtime_e2e_test.go b/runtime_e2e_test.go new file mode 100644 index 0000000..87093ed --- /dev/null +++ b/runtime_e2e_test.go @@ -0,0 +1,286 @@ +//go:build e2e + +// This file drives the agent Runtime through its real lifecycle — Load → Init → +// Start — standing the gateway container up exactly as `codefly run` does, then +// calls it over the advertised gRPC endpoint with the credential the agent +// handed out. Run with: +// +// docker build -t service-warehouse:e2e . +// SWH_GATEWAY_IMAGE=service-warehouse:e2e go test -tags e2e -run TestRuntimeEndToEnd . +// +// The gateway image is a prerequisite because the published one only exists once +// a release has run; the test points the Runtime at the locally built image +// through the same SWH_GATEWAY_IMAGE override the Runtime already honors. +// +// It asks the gateway for the catalog-only backend: that is the one a container +// can serve with no warehouse, so what is under test is the agent, the image and +// the credential, not a warehouse. +package main + +import ( + "context" + "fmt" + "os" + "os/exec" + "path" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/codefly-dev/core/callertoken" + basev0 "github.com/codefly-dev/core/generated/go/codefly/base/v0" + builderv0 "github.com/codefly-dev/core/generated/go/codefly/services/builder/v0" + runtimev0 "github.com/codefly-dev/core/generated/go/codefly/services/runtime/v0" + "github.com/codefly-dev/core/network" + "github.com/codefly-dev/core/resources" + "github.com/codefly-dev/core/shared" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/status" + + whv0 "github.com/codefly-dev/service-warehouse/gen/codefly/warehouse/v0" +) + +func TestRuntimeEndToEnd(t *testing.T) { + if os.Getenv(gatewayImageOverrideEnv) == "" { + t.Skipf("%s not set; skipping agent-runtime end-to-end test", gatewayImageOverrideEnv) + } + ctx := context.Background() + rt, networkMappings, runtimeContext := loadedRuntime(t, ctx) + + // Register teardown before Init: a failure after the container exists must + // still clean it up. Destroy nil-checks the environment, so it is safe even + // if Init failed before starting anything. The flag keeps this safety net + // from running a second Shutdown after the explicit teardown asserted below. + destroyed := false + defer func() { + if !destroyed { + _, _ = rt.Destroy(context.Background(), &runtimev0.DestroyRequest{}) + } + }() + + init, err := rt.Init(ctx, &runtimev0.InitRequest{ + RuntimeContext: runtimeContext, + ProposedNetworkMappings: networkMappings, + }) + require.NoError(t, err) + // Init reports a lifecycle failure in its status, not as a returned error: + // the contract hands the CLI a diagnostic rather than a transport fault. + require.Equal(t, runtimev0.InitStatus_READY, init.GetStatus().GetState(), init.GetStatus().GetMessage()) + + // This is the conformance lifecycle: Create → Load → Init → Test, no Start + // and no backend override. It must exercise the generated service defaults. + require.Equal(t, "mem", rt.Settings.Backend) + tested, err := rt.Test(ctx, &runtimev0.TestRequest{}) + require.NoError(t, err) + require.Equal(t, runtimev0.TestStatus_SUCCESS, tested.GetStatus().GetState(), tested.GetStatus().GetMessage()) + require.EqualValues(t, 1, tested.GetTestsPassed()) + require.Equal(t, runtimev0.TestRunResult_PASSED, tested.GetResult().GetState()) + require.EqualValues(t, 1, tested.GetCounts().GetPassed()) + + start, err := rt.Start(ctx, &runtimev0.StartRequest{}) + require.NoError(t, err) + require.Equal(t, runtimev0.StartStatus_STARTED, start.GetStatus().GetState(), start.GetStatus().GetMessage()) + + // A consumer running in its own container reaches the gateway through + // host.docker.internal, which resolves to the bridge gateway on Linux — a + // 127.0.0.1-bound port is unreachable there. It must publish on all + // interfaces; the token checked below is what keeps that from being an open + // door. + gatewayID, err := rt.gatewayEnv.ContainerID() + require.NoError(t, err) + requirePublishedOnAllInterfaces(t, gatewayID, gatewayContainerPort) + + conf, err := resources.ExtractConfiguration(init.RuntimeConfigurations, resources.NewRuntimeContextNative()) + require.NoError(t, err) + endpoint, err := resources.GetConfigurationValue(ctx, conf, configurationGroup, "endpoint") + require.NoError(t, err) + require.NotEmpty(t, endpoint) + connection, err := resources.GetConfigurationValue(ctx, conf, configurationGroup, "connection") + require.NoError(t, err) + token, err := resources.GetConfigurationValue(ctx, conf, configurationGroup, "token") + require.NoError(t, err) + require.NotEmpty(t, token, "the runtime must hand consumers a gateway credential") + require.NotContains(t, endpoint, token, "endpoint must not carry the credential") + require.NotContains(t, connection, token, "connection string must not carry the credential") + requireTokenIsSecret(t, conf, token) + + requireUnauthenticatedPeerIsRefused(t, endpoint) + + // What the gateway was started with is what the agent resolved: the backend + // it was told, and no variable for a value nobody set. + env, err := exec.Command("docker", "inspect", "-f", `{{join .Config.Env "\n"}}`, gatewayID).Output() + require.NoError(t, err) + gatewayEnv := strings.Split(string(env), "\n") + require.Contains(t, gatewayEnv, "SWH_BACKEND=mem") + require.Contains(t, gatewayEnv, fmt.Sprintf("SWH_LISTEN=:%d", gatewayContainerPort)) + for _, variable := range gatewayEnv { + require.False(t, strings.HasPrefix(variable, "SWH_DATABASE="), "a value nobody set must be left to the gateway: %s", variable) + } + + conn, err := grpc.NewClient(endpoint, + grpc.WithTransportCredentials(insecure.NewCredentials()), + callertoken.DialOption(token)) + require.NoError(t, err) + defer conn.Close() + capabilities, err := whv0.NewWarehouseClient(conn).Capabilities(ctx, &whv0.CapabilitiesRequest{}) + require.NoError(t, err) + require.NotNil(t, capabilities) + + // Teardown owns exactly what it started. + _, err = rt.Destroy(context.Background(), &runtimev0.DestroyRequest{}) + require.NoError(t, err) + destroyed = true + require.Error(t, exec.Command("docker", "inspect", gatewayID).Run(), "container %s survived teardown", gatewayID) +} + +// requireTokenIsSecret asserts the gateway credential is carried as a +// secret-marked configuration value, so nothing downstream logs or renders it. +func requireTokenIsSecret(t *testing.T, conf *basev0.Configuration, token string) { + t.Helper() + for _, info := range conf.GetInfos() { + for _, value := range info.GetConfigurationValues() { + if value.GetKey() == "token" { + require.Equal(t, token, value.GetValue()) + require.True(t, value.GetSecret(), "gateway token must be marked secret") + return + } + } + } + t.Fatal("no token value in the runtime configuration") +} + +// requireUnauthenticatedPeerIsRefused dials the published gateway port the way +// any other host on the network could — no credential at all, then a plausible +// wrong one — and asserts the gateway refuses it. This is the acceptance case +// for the all-interface binding asserted above: reachable, but useless without +// the session token. +func requireUnauthenticatedPeerIsRefused(t *testing.T, endpoint string) { + t.Helper() + peers := map[string][]grpc.DialOption{ + "no credentials": nil, + "wrong token": {callertoken.DialOption("0000000000000000000000000000000000000000000000000000000000000000")}, + } + for peerName, extra := range peers { + conn, err := grpc.NewClient(endpoint, + append(extra, grpc.WithTransportCredentials(insecure.NewCredentials()))...) + require.NoError(t, err) + t.Run(peerName, func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + _, err := whv0.NewWarehouseClient(conn).Capabilities(ctx, &whv0.CapabilitiesRequest{}) + require.Error(t, err) + require.Equal(t, codes.Unauthenticated, status.Code(err), "got %v", err) + }) + _ = conn.Close() + } +} + +// loadedRuntime scaffolds a service through the Builder and returns a Runtime +// loaded against it, with the network mappings and runtime context Init needs. +// +// The native runtime context puts the test process on the host, so both the +// Runtime's readiness probe and the test reach the gateway at localhost:. +func loadedRuntime(t *testing.T, ctx context.Context) (*Runtime, []*basev0.NetworkMapping, *basev0.RuntimeContext) { + t.Helper() + + // Keep all persistent test data outside the developer's real Codefly home. + codeflyHome, err := filepath.EvalSymlinks(t.TempDir()) + require.NoError(t, err) + t.Setenv(resources.CodeflyHomeEnv, codeflyHome) + workspace := &resources.Workspace{Name: "test"} + tmpDir := t.TempDir() + serviceName := fmt.Sprintf("svc-%v", time.Now().UnixMilli()) + service := resources.Service{Name: serviceName, Version: "test-me"} + require.NoError(t, service.SaveAtDir(ctx, path.Join(tmpDir, "mod", service.Name))) + + identity := &basev0.ServiceIdentity{ + Name: service.Name, + Version: service.Version, + Module: "mod", + Workspace: workspace.Name, + WorkspacePath: tmpDir, + RelativeToWorkspace: fmt.Sprintf("mod/%s", service.Name), + } + + builder := NewBuilder() + _, err = builder.Load(ctx, &builderv0.LoadRequest{ + DisableCatch: true, + Identity: identity, + CreationMode: &builderv0.CreationMode{Communicate: false}, + }) + require.NoError(t, err) + _, err = builder.Create(ctx, &builderv0.CreateRequest{}) + require.NoError(t, err) + + rt := NewRuntime() + networkManager, err := network.NewRuntimeManager(ctx, nil) + require.NoError(t, err) + networkManager.WithTemporaryPorts() + env := resources.LocalEnvironment() + + _, err = rt.Load(ctx, &runtimev0.LoadRequest{ + Identity: identity, + Environment: shared.Must(env.Proto()), + DisableCatch: true, + }) + require.NoError(t, err) + require.Equal(t, 1, len(rt.Endpoints)) + + runtimeContext := resources.NewRuntimeContextNative() + networkMappings, err := networkManager.GenerateNetworkMappings(ctx, env, workspace, rt.Identity, rt.Endpoints, runtimeContext) + require.NoError(t, err) + + return rt, networkMappings, runtimeContext +} + +// requirePublishedOnAllInterfaces asserts the container's port is published on +// 0.0.0.0, not 127.0.0.1 — the difference between reachable and refused from a +// consumer container on a Linux bridge network. +// +// Docker can publish a single port under more than one host binding (an IPv6 +// "::" entry beside the IPv4 "0.0.0.0" one), and their order is not guaranteed, +// so it ranges over every binding rather than trusting index 0. +func requirePublishedOnAllInterfaces(t *testing.T, containerID string, containerPort int) { + t.Helper() + out, err := exec.Command("docker", "inspect", "-f", + fmt.Sprintf(`{{range (index .NetworkSettings.Ports "%d/tcp")}}{{.HostIp}} {{end}}`, containerPort), + containerID).Output() + require.NoError(t, err) + hostIPs := strings.Fields(string(out)) + require.NotEmpty(t, hostIPs, "container port %d has no published host binding", containerPort) + require.NotContains(t, hostIPs, "127.0.0.1", "container port %d must not be published loopback-only", containerPort) + require.Contains(t, hostIPs, "0.0.0.0", "container port %d must publish on all interfaces", containerPort) +} + +// A gateway can die after Init returns. Test must reject that dead gateway +// even though container creation succeeded and Start was never called. +func TestRuntimeEndToEndRejectsDeadGateway(t *testing.T) { + if os.Getenv(gatewayImageOverrideEnv) == "" { + t.Skipf("set %s to a locally built gateway image", gatewayImageOverrideEnv) + } + ctx := context.Background() + rt, mappings, runtimeContext := loadedRuntime(t, ctx) + t.Cleanup(func() { _, _ = rt.Destroy(context.Background(), &runtimev0.DestroyRequest{}) }) + init, err := rt.Init(ctx, &runtimev0.InitRequest{ + RuntimeContext: runtimeContext, ProposedNetworkMappings: mappings, + }) + require.NoError(t, err) + require.Equal(t, runtimev0.InitStatus_READY, init.GetStatus().GetState(), init.GetStatus().GetMessage()) + id, err := rt.gatewayEnv.ContainerID() + require.NoError(t, err) + require.NoError(t, exec.Command("docker", "stop", id).Run()) + probe, cancel := context.WithTimeout(ctx, 3*time.Second) + defer cancel() + result, err := rt.Test(probe, &runtimev0.TestRequest{}) + require.NoError(t, err) + require.Equal(t, runtimev0.TestStatus_ERROR, result.GetStatus().GetState()) + require.Equal(t, runtimev0.TestRunResult_FAILED, result.GetResult().GetState()) + require.EqualValues(t, 1, result.GetCounts().GetFailed()) + require.Contains(t, result.GetResult().GetMessage(), "not ready") + require.Contains(t, result.GetStatus().GetMessage(), "not ready") + require.NotContains(t, result.GetStatus().GetMessage(), "no attempt completed") +} diff --git a/sbom_e2e_test.go b/sbom_e2e_test.go new file mode 100644 index 0000000..8cd8503 --- /dev/null +++ b/sbom_e2e_test.go @@ -0,0 +1,97 @@ +//go:build e2e + +// This file drives the agent's image-SBOM path against a real gateway image, +// inventorying it exactly as a release does and checking the result against the +// shared coverage contract. Run with: +// +// SWH_GATEWAY_IMAGE= go test -tags e2e -run TestImageSBOM . +// +// It needs syft on PATH: an image the local daemon holds and no registry serves +// is only reachable by a scanner with Docker access, which the managed +// containerized scanner deliberately does not have. +// +// Writing the evidence out is the release command's job (cmd/image-sbom), not +// this test's. +package main + +import ( + "context" + "os" + "os/exec" + "strings" + "testing" + + "github.com/codefly-dev/core/agents/services/sbom" + builderv0 "github.com/codefly-dev/core/generated/go/codefly/services/builder/v0" + "github.com/stretchr/testify/require" +) + +func TestImageSBOMInventoriesTheGatewayImage(t *testing.T) { + override := os.Getenv(gatewayImageOverrideEnv) + if override == "" { + t.Skipf("set %s to a locally built gateway image", gatewayImageOverrideEnv) + } + if _, err := exec.LookPath("syft"); err != nil { + t.Skip("syft is required to inventory an image held only by the local daemon") + } + builder := newSBOMBuilder(t) + subjects, err := builder.imageSubjects(context.Background()) + require.NoError(t, err) + + // The daemon resolves the override to its own image ID, so that ID is what a + // scan of it binds evidence to and what the subject has to name. + require.Len(t, subjects, 1, "the single platform the daemon holds") + require.Equal(t, sbom.SourceDockerDaemon, sbom.SourceOf(subjects[0])) + require.Equal(t, override, subjects[0].GetReference()) + require.NotEmpty(t, subjects[0].GetPlatform(), "core reads the platform from the daemon image") + id, err := sbom.LocalImageID(context.Background(), override) + require.NoError(t, err) + require.Equal(t, id, subjects[0].GetDigest()) + require.NoError(t, sbom.RequirePinned(subjects[0])) + + resp, err := builder.SBOM(context.Background(), &builderv0.SBOMRequest{Scope: builderv0.SBOMScope_SBOM_SCOPE_IMAGE}) + require.NoError(t, err) + require.Equal(t, builderv0.SBOMStatus_COMPLETE, resp.GetState().GetState(), resp.GetState().GetMessage()) + require.NoError(t, sbom.ValidateCoverage(testService, subjects, resp)) + + require.Len(t, resp.GetImages(), 1) + evidence := resp.GetImages()[0] + + // The subject named the local image ID before the scan ran; evidence that + // resolved to any other identity describes an image nobody asked for. + require.Equal(t, subjects[0].GetDigest(), evidence.GetDigest()) + require.True(t, strings.HasPrefix(evidence.GetDigest(), "sha256:")) + require.Equal(t, subjects[0].GetPlatform(), evidence.GetPlatform()) + require.NotEmpty(t, evidence.GetSha256(), "evidence carries no document checksum") + + var goModules, osPackages int + var libc bool + for _, component := range evidence.GetBom().GetComponents() { + switch purl := component.GetPurl(); { + case strings.HasPrefix(purl, "pkg:golang/"): + goModules++ + case isOSPackage(purl): + osPackages++ + libc = libc || strings.HasPrefix(purl, "pkg:deb/debian/libc6@") + } + } + require.True(t, libc, "inventory must contain Debian libc6: the gateway dynamically links glibc") + require.NotZero(t, goModules, "inventory found no Go modules: the gateway's application dependencies are missing") + require.NotZero(t, osPackages, "inventory found no OS packages: an image inventory is not satisfied by application dependencies alone") + t.Logf("inventoried %s on %s (%d Go modules, %d OS packages)", evidence.GetDigest(), evidence.GetPlatform(), goModules, osPackages) +} + +// osPackageTypes are the package-URL namespaces an OS inventory reports under. +// Most of what syft finds in an image carries no package URL at all, so counting +// everything that is not a Go module would let an image with no OS inventory +// pass as covered. +var osPackageTypes = []string{"pkg:deb/", "pkg:apk/", "pkg:rpm/"} + +func isOSPackage(purl string) bool { + for _, prefix := range osPackageTypes { + if strings.HasPrefix(purl, prefix) { + return true + } + } + return false +} diff --git a/sbom_test.go b/sbom_test.go new file mode 100644 index 0000000..0334a94 --- /dev/null +++ b/sbom_test.go @@ -0,0 +1,543 @@ +package main + +import ( + "context" + "fmt" + "os" + "os/exec" + "path/filepath" + "slices" + "sort" + "strings" + "testing" + + "github.com/codefly-dev/core/agents/services/sbom" + basev0 "github.com/codefly-dev/core/generated/go/codefly/base/v0" + agentv0 "github.com/codefly-dev/core/generated/go/codefly/services/agent/v0" + builderv0 "github.com/codefly-dev/core/generated/go/codefly/services/builder/v0" + "github.com/codefly-dev/core/resources" + "github.com/stretchr/testify/require" + "gopkg.in/yaml.v3" + + "github.com/codefly-dev/service-warehouse/internal/gatewayimage" +) + +// testService is the identity newSBOMBuilder loads, and therefore the identity +// every subject it produces carries. +const testService = "module/warehouse" + +// pinnedReference is a digest-pinned image reference of the shape a release +// scans. It stands in for the published image so these tests mean the same thing +// before the first image is published and after. +const pinnedReference = registry + "/service-warehouse@" + digest + +// newSBOMBuilder returns a Builder wired for a headless SBOM call. +func newSBOMBuilder(t *testing.T) *Builder { + t.Helper() + builder := NewBuilder() + identity := &basev0.ServiceIdentity{ + Workspace: "workspace", Module: "module", Name: "warehouse", Version: "1.2.3", + WorkspacePath: t.TempDir(), RelativeToWorkspace: ".", + } + require.NoError(t, builder.Base.HeadlessLoad(context.Background(), identity)) + return builder +} + +func TestImageSubjectsCoverEveryPublishedPlatform(t *testing.T) { + withPublishedLock(t) + t.Setenv(gatewayImageOverrideEnv, "") + subjects, err := newSBOMBuilder(t).imageSubjects(context.Background()) + require.NoError(t, err) + + require.Len(t, subjects, len(gatewayimage.Platforms), "one subject per shipped platform") + covered := map[string]bool{} + for _, subject := range subjects { + require.Equal(t, sbom.SourceRegistry, sbom.SourceOf(subject)) + require.NoError(t, sbom.RequirePinned(subject)) + require.Equal(t, pinnedReference, subject.GetReference()) + require.Equal(t, gatewayimage.Role, subject.GetRole()) + require.Equal(t, testService, subject.GetService()) + covered[subject.GetPlatform()] = true + } + for _, platform := range gatewayimage.Platforms { + require.True(t, covered[platform], "no subject covers shipped platform %s", platform) + } +} + +// The registry derivation is core's, and it refuses what the evidence contract +// refuses: a tag serves whatever was pushed to it last, so a subject naming one +// says nothing about the image that was built. +func TestImageSubjectsRefuseAReferenceThatIsNotPinned(t *testing.T) { + for _, reference := range []string{ + registry + "/service-warehouse", + registry + "/service-warehouse:latest", + registry + "/service-warehouse:0.0.1", + } { + _, err := sbom.ImageSubjects(context.Background(), sbom.PublishedImage{Service: testService, Role: gatewayimage.Role, Reference: reference, Platforms: gatewayimage.Platforms}, false) + require.Error(t, err, reference) + } +} + +// What the Builder enumerates for itself is the pinned image gateway-image.json +// records, however many platforms that is. While the lock records no image there +// is nothing to enumerate, and saying so is the only honest answer. +func TestBuilderEnumeratesThePublishedImageOrRefuses(t *testing.T) { + t.Setenv(gatewayImageOverrideEnv, "") + builder := newSBOMBuilder(t) + + subjects, err := builder.imageSubjects(context.Background()) + if _, lockErr := publishedGatewayImage(); lockErr != nil { + require.ErrorIs(t, lockErr, resources.ErrImageUnpublished) + require.ErrorIs(t, err, resources.ErrImageUnpublished) + return + } + require.NoError(t, err) + published, err := publishedGatewayImage() + require.NoError(t, err) + require.Len(t, subjects, len(gatewayimage.Platforms)) + for _, subject := range subjects { + require.Equal(t, published.FullName(), subject.GetReference()) + require.Equal(t, testService, subject.GetService()) + } +} + +func TestImageEvidenceSatisfiesTheCoverageContract(t *testing.T) { + subjects, err := sbom.ImageSubjects(context.Background(), sbom.PublishedImage{Service: testService, Role: gatewayimage.Role, Reference: pinnedReference, Platforms: gatewayimage.Platforms}, false) + require.NoError(t, err) + + // The expectation is built from the platforms the release pipeline actually + // publishes, never from gatewayimage: deriving both sides from the code + // under test compares it against itself, and would validate however many + // platforms it dropped. + expected := expectedFromReleasePipeline(t, pinnedReference) + + require.NoError(t, sbom.ValidateCoverage(testService, expected, imageResponse(evidenceFor(subjects))), + "evidence for every published platform must validate as coverage") + require.Error(t, sbom.ValidateCoverage(testService, expected, imageResponse(evidenceFor(subjects[:1]))), + "evidence omitting a published platform must not validate as coverage") +} + +// The daemon resolves whatever reference it is handed to its own image ID, so a +// subject for a local image has to carry that ID: evidence bound to an identity +// the subject never named is not coverage of it. Resolving it can fail — the +// override may name an image the daemon does not hold — and reporting that is +// the only alternative to producing a subject pinned to nothing. +// +// The resolved case needs a real image and is asserted by the e2e inventory. +func TestGatewayOverrideMustBeHeldByTheDaemon(t *testing.T) { + const absent = "swh-gateway-unit-test-absent:none" + t.Setenv(gatewayImageOverrideEnv, absent) + + subjects, err := newSBOMBuilder(t).imageSubjects(context.Background()) + require.Error(t, err, "got %d subjects for an image the daemon does not hold", len(subjects)) + require.Contains(t, err.Error(), absent, "the error must name the override that could not be resolved") +} + +func TestImageScopeReportsFailureRatherThanCoverage(t *testing.T) { + for name, setup := range map[string]func(*testing.T){ + "an override that is not an image": func(t *testing.T) { + t.Setenv(gatewayImageOverrideEnv, "not a valid image reference") + }, + "a lock that records no image": func(t *testing.T) { + if _, err := publishedGatewayImage(); err == nil { + t.Skip("gateway-image.json records a published image") + } + t.Setenv(gatewayImageOverrideEnv, "") + }, + } { + t.Run(name, func(t *testing.T) { + setup(t) + resp, err := newSBOMBuilder(t).SBOM(context.Background(), &builderv0.SBOMRequest{Scope: builderv0.SBOMScope_SBOM_SCOPE_IMAGE}) + require.NoError(t, err) + + require.Equal(t, builderv0.SBOMStatus_ERROR, resp.GetState().GetState()) + require.Equal(t, builderv0.SBOMScope_SBOM_SCOPE_IMAGE, resp.GetScope(), "the failure is attributed to the image request") + require.Empty(t, resp.GetImages(), "a failed scan carries no inventory") + require.Error(t, sbom.ValidateCoverage(testService, nil, resp), "a failed scan must not validate as coverage") + }) + } +} + +func TestSourceScopeKeepsTheExistingInventory(t *testing.T) { + t.Setenv(gatewayImageOverrideEnv, "") + subjects, err := sbom.ImageSubjects(context.Background(), sbom.PublishedImage{Service: testService, Role: gatewayimage.Role, Reference: pinnedReference, Platforms: gatewayimage.Platforms}, false) + require.NoError(t, err) + + // A cancelled context stops the scan before it reaches the network, which is + // enough to tell the two paths apart: the image path attributes even its + // failures to SBOM_SCOPE_IMAGE, so an unspecified scope that does not come + // back image-scoped was answered by the source inventory. + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + resp, err := newSBOMBuilder(t).SBOM(ctx, &builderv0.SBOMRequest{}) + require.NoError(t, err) + + require.NotEqual(t, builderv0.SBOMScope_SBOM_SCOPE_IMAGE, resp.GetScope(), "an unspecified scope must not be answered with image evidence") + require.Error(t, sbom.ValidateCoverage(testService, subjects, resp), "a source inventory must not validate as image coverage") +} + +// Serving the image scope is what lets the agent answer; advertising +// ValidationCapabilities.image_sbom is what gets the phase scheduled against +// every consumer, and core's docs/sbom.md holds that back until each of them +// runs a core that can represent it. A present Validation is also authoritative +// for every operation it omits, so setting one to advertise this capability +// would declare test unsupported for a service that has no suite to run. Flipping +// this is a rollout decision, which is why a test, not a comment, marks it. +func TestImageSBOMIsServedButNotAdvertised(t *testing.T) { + info, err := NewService().GetAgentInformation(context.Background(), &agentv0.AgentInformationRequest{}) + require.NoError(t, err) + require.Nil(t, info.GetValidation(), "advertising image_sbom is a fleet rollout decision, not this agent's") + + // And it is served: an explicit subject is scanned rather than answered as + // unsupported, so the failure here is the scanner's, not a missing + // implementation. + subject := &builderv0.ImageSubject{Reference: "not-pinned", Service: testService, Role: gatewayimage.Role} + resp, err := newSBOMBuilder(t).SBOM(context.Background(), &builderv0.SBOMRequest{ + Scope: builderv0.SBOMScope_SBOM_SCOPE_IMAGE, + Subjects: []*builderv0.ImageSubject{subject}, + }) + require.NoError(t, err) + require.Equal(t, builderv0.SBOMStatus_ERROR, resp.GetState().GetState(), resp.GetState().GetMessage()) + require.NotEqual(t, builderv0.SBOMStatus_UNSUPPORTED, resp.GetState().GetState()) +} + +func TestShippedPlatformsMatchTheReleasePipeline(t *testing.T) { + published := append([]string(nil), publishedPlatforms(t)...) + declared := append([]string(nil), gatewayimage.Platforms...) + // Compare as sets: the pipeline and the code have to name the same + // platforms, but neither owns the order the other writes them in. + sort.Strings(published) + sort.Strings(declared) + + require.Equal(t, published, declared, + "the pipeline publishes %v but evidence covers %v; a platform shipped without a subject ships uninventoried", published, declared) +} + +// The release once looped over a platform list written into the workflow and +// called the scanner itself, which is a second list to keep in sync and a +// second scanner to drift from the contract. Evidence comes from the release +// command, which reads the same platform list the agent does. +func TestReleaseEvidenceComesFromTheSharedCommand(t *testing.T) { + body := workflowBody(t, "release.yml") + + require.Contains(t, body, "cmd/image-sbom", "the release workflow does not generate evidence through cmd/image-sbom") + require.NotContains(t, body, "for platform in", "the release workflow loops over its own platform list") + require.NotContains(t, workflowBody(t, "publish-gateway-image.yml"), "for platform in", + "the publish workflow loops over its own platform list; the platforms it builds are read from its build-push step") +} + +// expectedFromReleasePipeline builds the coverage expectation from the publish +// workflow, independently of the code that produces the evidence. +func expectedFromReleasePipeline(t *testing.T, reference string) []*builderv0.ImageSubject { + t.Helper() + platforms := publishedPlatforms(t) + expected := make([]*builderv0.ImageSubject, 0, len(platforms)) + for _, platform := range platforms { + expected = append(expected, &builderv0.ImageSubject{ + Reference: reference, + Platform: platform, + Role: gatewayimage.Role, + Service: testService, + }) + } + return expected +} + +// publishedPlatforms reads the actual build step in this repository's publisher. +func publishedPlatforms(t *testing.T) []string { + t.Helper() + var workflow struct { + Jobs map[string]struct { + Steps []struct { + Uses string `yaml:"uses"` + With struct { + Platforms string `yaml:"platforms"` + } `yaml:"with"` + } `yaml:"steps"` + } `yaml:"jobs"` + } + require.NoError(t, yaml.Unmarshal([]byte(workflowBody(t, "publish-gateway-image.yml")), &workflow)) + var published []string + for _, job := range workflow.Jobs { + for _, step := range job.Steps { + if !strings.HasPrefix(step.Uses, "docker/build-push-action@") { + continue + } + for _, platform := range strings.Split(step.With.Platforms, ",") { + if platform = strings.TrimSpace(platform); platform != "" { + published = append(published, platform) + } + } + } + } + require.NotEmpty(t, published, "the publisher must declare the platforms evidence covers") + return published +} + +func workflowBody(t *testing.T, name string) string { + t.Helper() + data, err := os.ReadFile(filepath.Join(".github", "workflows", name)) + require.NoError(t, err) + return string(data) +} + +// releaseStep is one step of the release workflow, kept in file order so a test +// can assert what happens before what. +type releaseStep struct { + ID string `yaml:"id"` + Name string `yaml:"name"` + Uses string `yaml:"uses"` + Run string `yaml:"run"` + Env map[string]string `yaml:"env"` +} + +type releaseJob struct { + Needs any `yaml:"needs"` + Uses string `yaml:"uses"` + Steps []releaseStep `yaml:"steps"` +} + +func releaseJobs(t *testing.T) map[string]releaseJob { + t.Helper() + var workflow struct { + Jobs map[string]releaseJob `yaml:"jobs"` + } + require.NoError(t, yaml.Unmarshal([]byte(workflowBody(t, "release.yml")), &workflow)) + return workflow.Jobs +} + +// releaseStepRunning returns the one release step whose script contains marker. +func releaseStepRunning(t *testing.T, marker string) releaseStep { + t.Helper() + var found []releaseStep + for _, job := range releaseJobs(t) { + for _, step := range job.Steps { + if strings.Contains(step.Run, marker) { + found = append(found, step) + } + } + } + require.Len(t, found, 1, "want exactly one release step running %q", marker) + return found[0] +} + +// stepIndex is where in the release job order marker first runs, counting the +// jobs in name order so a step added to another job shifts every index the same +// way on each run: Go randomises map iteration. +func stepIndex(t *testing.T, marker string) int { + t.Helper() + jobs := releaseJobs(t) + names := make([]string, 0, len(jobs)) + for name := range jobs { + names = append(names, name) + } + sort.Strings(names) + index := 0 + for _, name := range names { + for _, step := range jobs[name].Steps { + if strings.Contains(step.Run, marker) { + return index + } + index++ + } + } + return -1 +} + +// The release used to push the tags consumers resolve and inventory the image +// afterwards. A failure anywhere in the evidence step then turned the run red +// with a pullable, uninventoried image already published — and a red run reads +// as "nothing shipped", so nobody goes looking for it. The image is now pushed +// by digest before the tag exists, and the tags are created only once evidence +// for that digest exists. +func TestReleasePublishesTagsOnlyAfterEvidence(t *testing.T) { + for jobName, job := range releaseJobs(t) { + for _, step := range job.Steps { + require.NotContains(t, step.Uses, "docker/build-push-action", + "%s builds the gateway image; the digest the agent pins cannot come from the tag that ships it, so the image is published before the tag", jobName) + } + } + evidence, publish := stepIndex(t, "cmd/image-sbom"), stepIndex(t, "imagetools create") + require.GreaterOrEqual(t, evidence, 0, "the release workflow does not generate image evidence") + require.GreaterOrEqual(t, publish, 0, "the release workflow never creates the tags consumers resolve") + require.Greater(t, publish, evidence, "release tags are created before evidence: a failed scan would leave a pullable image with no evidence") +} + +// The gateway image the release tags is the one gateway-image.json records, and +// the agent runs that digest rather than a tag. Reading it from anywhere else — +// resolving :latest, or rebuilding — publishes tags for bytes the released +// agent never names. +func TestReleaseTagsTheRecordedDigest(t *testing.T) { + publish := releaseStepRunning(t, "imagetools create") + require.Contains(t, publish.Run, `"$IMAGE@$DIGEST"`, "the tag-publishing step does not create its tags from a recorded digest") + + locked := releaseStepRunning(t, "gateway-image.json") + require.Contains(t, locked.Run, "org.opencontainers.image.version", + "the release accepts the recorded digest without checking it was built for this version; a lock left unrefreshed publishes the previous release's image under the new tag") +} + +// The release must refuse the tree as it is merged: gateway-image.json records +// no digest until the first image is published, and `jq -e` on its own passes an +// empty string. This runs the workflow's own guard against that file, not a +// description of it. It stops before any registry call, so it needs no network. +func TestReleaseRefusesALockThatRecordsNoImage(t *testing.T) { + if _, err := exec.LookPath("jq"); err != nil { + t.Skip("jq is required to run the release's lock check") + } + if _, err := publishedGatewayImage(); err == nil { + t.Skip("gateway-image.json records a published image; this guards the state before one exists") + } + locked := releaseStepRunning(t, "gateway-image.json") + lock, err := os.ReadFile("gateway-image.json") + require.NoError(t, err) + + dir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(dir, "gateway-image.json"), lock, 0o600)) + command := exec.Command("bash", "-c", locked.Run) + command.Dir = dir + command.Env = append(os.Environ(), "VERSION=0.0.1", "IMAGE="+registry+"/service-warehouse") + out, err := command.CombinedOutput() + require.Error(t, err, "the release accepted a lock that records no image:\n%s", out) + require.Contains(t, string(out), "no gateway image has been published") +} + +// The published image tag is derived from the git tag while the agent reports +// the version of the embedded agent.codefly.yaml. Nothing reconciles them at run +// time, and a tag one release ahead of the file makes a consumer pinned to one +// version run another. This runs the workflow's own guard against the real file +// rather than trusting that it is present. +func TestReleaseGuardRejectsTagThatDisagreesWithAgentVersion(t *testing.T) { + guard := releaseStepRunning(t, "agent.codefly.yaml") + + run := func(version string) error { + command := exec.Command("bash", "-c", guard.Run) + command.Env = append(os.Environ(), "VERSION="+version) + return command.Run() + } + + require.NoError(t, run(agent.Version), "guard rejected the version this repo actually carries") + require.Error(t, run(agent.Version+"9"), "guard accepted a tag that disagrees with agent.codefly.yaml") +} + +// needs is a job's `needs`, which GitHub accepts as one job name or a list. +func needs(job releaseJob) []string { + switch value := job.Needs.(type) { + case string: + return []string{value} + case []any: + names := make([]string, 0, len(value)) + for _, name := range value { + names = append(names, fmt.Sprint(name)) + } + return names + } + return nil +} + +// runsAfter reports whether job cannot start until earlier has succeeded. +func runsAfter(jobs map[string]releaseJob, job, earlier string) bool { + seen := map[string]bool{} + queue := needs(jobs[job]) + for len(queue) > 0 { + next := queue[0] + queue = queue[1:] + if next == earlier { + return true + } + if !seen[next] { + seen[next] = true + queue = append(queue, needs(jobs[next])...) + } + } + return false +} + +// A `v*` tag must start one workflow, because separate workflows cannot order or +// gate each other: either could fail while the other published, and the harmful +// direction is the release becoming `latest` while no registry serves the image +// its agent names. Everything a tag publishes hangs off one chain, ordered by +// the dependency that actually exists. +func TestATagPublishesThroughOneOrderedWorkflow(t *testing.T) { + entries, err := os.ReadDir(filepath.Join(".github", "workflows")) + require.NoError(t, err) + var onTag []string + for _, entry := range entries { + if !entry.Type().IsRegular() || (filepath.Ext(entry.Name()) != ".yml" && filepath.Ext(entry.Name()) != ".yaml") { + continue + } + var workflow struct { + On map[string]any `yaml:"on"` + } + require.NoError(t, yaml.Unmarshal([]byte(workflowBody(t, entry.Name())), &workflow), entry.Name()) + push, ok := workflow.On["push"].(map[string]any) + if !ok { + continue + } + // A push trigger covers branches and tags alike: naming only branches + // drops tags, while naming neither filter leaves both live. + _, tags := push["tags"] + _, tagsIgnore := push["tags-ignore"] + _, branches := push["branches"] + _, branchesIgnore := push["branches-ignore"] + if tags || tagsIgnore || (!branches && !branchesIgnore) { + onTag = append(onTag, entry.Name()) + } + } + require.Equal(t, []string{"release.yml"}, onTag, "a tag push must start one workflow") + + jobs := releaseJobs(t) + var releases, images, gates []string + for name, job := range jobs { + if strings.Contains(job.Uses, "go-service-release.yml") { + releases = append(releases, name) + } + for _, step := range job.Steps { + if strings.Contains(step.Run, "imagetools create") { + images = append(images, name) + } + if strings.Contains(step.Run, "go test") { + gates = append(gates, name) + } + } + } + require.Len(t, releases, 1, "a tag ships one release") + require.NotEmpty(t, images) + require.NotEmpty(t, gates) + // Every image the tag publishes has to exist before the release names it, + // and has to be gated on the suite. Quantifying over the sets keeps a job + // added later from changing which pair happens to be checked. + for _, image := range images { + require.True(t, runsAfter(jobs, releases[0], image), + "job %q does not wait for %q: the release can ship an agent whose gateway image no registry resolves", releases[0], image) + require.True(t, slices.ContainsFunc(gates, func(gate string) bool { return runsAfter(jobs, image, gate) }), + "job %q waits for none of %v: a tag whose tests fail still publishes an image tagged :latest", image, gates) + } +} + +// imageResponse wraps evidence in the complete image-scope response an agent +// returns. +func imageResponse(images []*builderv0.ImageSBOM) *builderv0.SBOMResponse { + return &builderv0.SBOMResponse{ + State: &builderv0.SBOMStatus{State: builderv0.SBOMStatus_COMPLETE}, + Scope: builderv0.SBOMScope_SBOM_SCOPE_IMAGE, + Images: images, + } +} + +// evidenceFor builds one digest-bound inventory per subject, each with a +// distinct digest so coverage is matched per platform. +func evidenceFor(subjects []*builderv0.ImageSubject) []*builderv0.ImageSBOM { + images := make([]*builderv0.ImageSBOM, 0, len(subjects)) + for i, subject := range subjects { + images = append(images, &builderv0.ImageSBOM{ + Digest: fmt.Sprintf("sha256:%064d", i), + Platform: subject.GetPlatform(), + Subjects: []*builderv0.ImageSubject{subject}, + Bom: &agentv0.Bom{Components: []*agentv0.Component{{Name: "libc", Version: "2.36"}}}, + Tool: "syft", + Sha256: fmt.Sprintf("%064d", i), + }) + } + return images +} diff --git a/templates/agent/README.md.tmpl b/templates/agent/README.md.tmpl new file mode 100644 index 0000000..77a7fa8 --- /dev/null +++ b/templates/agent/README.md.tmpl @@ -0,0 +1,17 @@ +# warehouse + +The warehouse gateway is a codefly/warehouse/v0 gRPC service in front of a SQL +warehouse. A generated service declares `backend: mem`, the catalog-only backend +that needs no warehouse credentials. Set `backend` in service.codefly.yaml to +choose another engine; removing it falls back to the unsupported DuckDB skeleton. +See https://github.com/codefly-dev/service-warehouse for current backend support. + +A local run starts the gateway image with a fresh shared secret and hands it to +consumers as the secret `token` value of the `warehouse` configuration group. +Both Start and Test wait for health and authenticated Capabilities. + +Deployment requires an environment-managed Kubernetes secret reference for +SWH_AUTH_TOKEN. Bind the same secret key to each consumer's warehouse.token +configuration. The agent renders a same-namespace ingress NetworkPolicy and a +dedicated ServiceAccount; a policy-capable CNI is required. See the repository +README for both secret bindings and workload-identity setup. diff --git a/templates/deployment/kustomize/base/deployment.yaml.tmpl b/templates/deployment/kustomize/base/deployment.yaml.tmpl new file mode 100644 index 0000000..35607dc --- /dev/null +++ b/templates/deployment/kustomize/base/deployment.yaml.tmpl @@ -0,0 +1,117 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{.Name}} + namespace: {{.Namespace}} +spec: + replicas: 1 + selector: + matchLabels: + app: {{.Name}} + template: + metadata: + labels: + app: {{.Name}} + spec: + automountServiceAccountToken: false + securityContext: + runAsNonRoot: true + runAsUser: 65532 + runAsGroup: 65532 + seccompProfile: + type: RuntimeDefault + terminationGracePeriodSeconds: 30 + containers: + - name: warehouse + image: {{ .Image }} + imagePullPolicy: IfNotPresent + securityContext: + allowPrivilegeEscalation: false + runAsNonRoot: true + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + seccompProfile: + type: RuntimeDefault + ports: + - name: grpc + containerPort: {{ .Deployment.Parameters.ContainerPort }} + env: + # The service identity the CLI binds the environment's service + # configuration onto: per environment, after this render, it + # replaces the SWH_* literals below by name on the container that + # declares this (and adds the ones not rendered here). + - name: CODEFLY__SERVICE + value: "{{ .Name }}" + - name: SWH_LISTEN + value: ":{{ .Deployment.Parameters.ContainerPort }}" +{{- if .Deployment.Parameters.Backend }} + - name: SWH_BACKEND + value: "{{ .Deployment.Parameters.Backend }}" +{{- end }} +{{- if .Deployment.Parameters.Database }} + - name: SWH_DATABASE + value: "{{ .Deployment.Parameters.Database }}" +{{- end }} +{{- if .Deployment.Parameters.Dataset }} + - name: SWH_DATASET + value: "{{ .Deployment.Parameters.Dataset }}" +{{- end }} +{{- if .Deployment.Parameters.Location }} + - name: SWH_LOCATION + value: "{{ .Deployment.Parameters.Location }}" +{{- end }} + - name: SWH_ALLOW_ANONYMOUS + value: "false" +{{- range $key, $ref := .Deployment.SecretReferences }} + - name: {{ $key }} + valueFrom: + secretKeyRef: + name: {{ $ref.Name }} + key: {{ $ref.Key }} +{{- end }} +{{- /* No SWH_CREDENTIALS_FILE: nothing here mounts a credentials file, so a path +would name a file the pod does not have. A deployed cloud warehouse authenticates +through a workload identity bound to the ServiceAccount below, or through a file +an overlay outside this agent mounts and patches the variable for; Deploy rejects +a configured path rather than emit one. Kept as a template comment: rendering it +as YAML would ship that prose into every manifest. */}} +{{- if not .Restricted }} + envFrom: + - secretRef: + name: {{.Name}}-secret +{{- end }} + resources: + requests: + cpu: 50m + memory: 128Mi + limits: + cpu: "1" + memory: 1Gi + # The gateway's gRPC health service reports that the process is serving + # and nothing more: it does not probe the warehouse, so there is no + # service to name here that would attest warehouse access, and all + # three probes check the overall ("") service. A warehouse outage is + # reported per request instead, and does not drain or restart replicas + # that share one warehouse and one credential set. + # + # The grpc probe handler requires Kubernetes >= 1.24 (GA in 1.27); on + # older API servers the field is pruned and the Deployment is rejected + # for specifying no handler. + startupProbe: + grpc: + port: {{ .Deployment.Parameters.ContainerPort }} + periodSeconds: 2 + failureThreshold: 30 + readinessProbe: + grpc: + port: {{ .Deployment.Parameters.ContainerPort }} + periodSeconds: 5 + timeoutSeconds: 3 + livenessProbe: + grpc: + port: {{ .Deployment.Parameters.ContainerPort }} + periodSeconds: 30 + timeoutSeconds: 5 + failureThreshold: 3 diff --git a/templates/deployment/kustomize/base/kustomization.yaml.tmpl b/templates/deployment/kustomize/base/kustomization.yaml.tmpl new file mode 100644 index 0000000..fab5a9c --- /dev/null +++ b/templates/deployment/kustomize/base/kustomization.yaml.tmpl @@ -0,0 +1,10 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: +{{- if not .Restricted }} + - namespace.yaml +{{- end }} + - deployment.yaml + - service.yaml + - networkpolicy.yaml diff --git a/templates/deployment/kustomize/base/namespace.yaml.tmpl b/templates/deployment/kustomize/base/namespace.yaml.tmpl new file mode 100644 index 0000000..f24694a --- /dev/null +++ b/templates/deployment/kustomize/base/namespace.yaml.tmpl @@ -0,0 +1,8 @@ +{{- if not .Restricted }} +apiVersion: v1 +kind: Namespace +metadata: + name: {{.Namespace}} + labels: + app.kubernetes.io/managed-by: codefly +{{- end }} diff --git a/templates/deployment/kustomize/base/networkpolicy.yaml.tmpl b/templates/deployment/kustomize/base/networkpolicy.yaml.tmpl new file mode 100644 index 0000000..98adccc --- /dev/null +++ b/templates/deployment/kustomize/base/networkpolicy.yaml.tmpl @@ -0,0 +1,17 @@ +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{.Name}} + namespace: {{.Namespace}} +spec: + podSelector: + matchLabels: + app: {{.Name}} + policyTypes: + - Ingress + ingress: + - from: + - podSelector: {} + ports: + - protocol: TCP + port: {{ .Deployment.Parameters.ContainerPort }} diff --git a/templates/deployment/kustomize/base/service.yaml.tmpl b/templates/deployment/kustomize/base/service.yaml.tmpl new file mode 100644 index 0000000..7944a40 --- /dev/null +++ b/templates/deployment/kustomize/base/service.yaml.tmpl @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{.Name}} + namespace: {{.Namespace}} +spec: + selector: + app: {{.Name}} + ports: + - name: grpc + port: {{ .Deployment.Parameters.ServicePort }} + targetPort: {{ .Deployment.Parameters.ContainerPort }} diff --git a/templates/deployment/kustomize/overlays/environment/kustomization.yaml.tmpl b/templates/deployment/kustomize/overlays/environment/kustomization.yaml.tmpl new file mode 100644 index 0000000..904871f --- /dev/null +++ b/templates/deployment/kustomize/overlays/environment/kustomization.yaml.tmpl @@ -0,0 +1,8 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - ../../base +{{- if not .Restricted }} + - secret.yaml +{{- end }} diff --git a/templates/deployment/kustomize/overlays/environment/secret.yaml.tmpl b/templates/deployment/kustomize/overlays/environment/secret.yaml.tmpl new file mode 100644 index 0000000..7ff4522 --- /dev/null +++ b/templates/deployment/kustomize/overlays/environment/secret.yaml.tmpl @@ -0,0 +1,12 @@ +{{- if not .Restricted }} +apiVersion: v1 +kind: Secret +metadata: + name: {{.Name}}-secret + namespace: {{.Namespace}} +type: Opaque +data: +{{- range $key, $value := .SecretMap }} + {{ $key }}: "{{ $value }}" +{{- end }} +{{- end }} diff --git a/templates/factory/GETTING_STARTED.md.tmpl b/templates/factory/GETTING_STARTED.md.tmpl new file mode 100644 index 0000000..03486ad --- /dev/null +++ b/templates/factory/GETTING_STARTED.md.tmpl @@ -0,0 +1,12 @@ +# Getting started + +This service runs the warehouse gateway with `backend: mem`: the catalog-only +backend needs no warehouse credentials and does not execute queries. The agent +generates the caller token locally and delivers it to consumers. Its test step +checks that the gateway is serving and accepts that token. + +Deployment requires an environment-managed Kubernetes secret reference for +`SWH_AUTH_TOKEN`. Bind the same secret key to every consumer's `warehouse.token` +configuration. The rendered NetworkPolicy allows only same-namespace ingress; +a policy-capable CNI is required. Anonymous access is only an explicit opt-in +when running the gateway directly for local development. diff --git a/templates/factory/configurations/local/warehouse.secret.env b/templates/factory/configurations/local/warehouse.secret.env new file mode 100644 index 0000000..75fb6b8 --- /dev/null +++ b/templates/factory/configurations/local/warehouse.secret.env @@ -0,0 +1,7 @@ +# A local run needs no secrets. The agent generates the gateway's SWH_AUTH_TOKEN +# per run and hands it to consumers as the secret "token" value of the warehouse +# configuration group — nothing to set here. +# For deployment, bind SWH_AUTH_TOKEN and each consumer's warehouse.token to +# the same environment-managed Kubernetes secret key (see agent README). +# Deployed environments reach a cloud warehouse through a workload identity, not +# a credential file.