diff --git a/.github/workflows/this-test-modules.yaml b/.github/workflows/this-test-modules.yaml index 744e482c..1130a93b 100644 --- a/.github/workflows/this-test-modules.yaml +++ b/.github/workflows/this-test-modules.yaml @@ -188,5 +188,5 @@ jobs: # secrets (SOPS_AGE_KEY, registry creds) unavailable here. Add a module # once tests/smoke/.sh exists and runs without secrets. - name: smoke test ${{ matrix.module }} - if: contains(fromJSON('["ansible", "cosign", "crossplane", "kyverno", "sops", "terraform"]'), matrix.module) + if: contains(fromJSON('["ansible", "cosign", "crossplane", "kyverno", "sops", "techdocs", "terraform"]'), matrix.module) run: ./tests/smoke/${{ matrix.module }}.sh diff --git a/README.md b/README.md index cc3ddac4..1da28963 100644 --- a/README.md +++ b/README.md @@ -29,6 +29,7 @@ modular collection of Dagger building blocks for infrastructure, containers, sec | [OCI](./oci/README.md) | OCI Artifact Management | Flux push, batch push, local TLS registry testing | | [Flux](./flux/README.md) | Flux GitOps Artifacts | Build/push OCI artifacts, batch push, registry auth | | [Slidev](./slidev/README.md) | Presentations / Decks | Scaffold, dev server, static build from a single `slides.md` | +| [TechDocs](./techdocs/README.md) | Backstage TechDocs | Build docs in CI, publish to S3/MinIO for `builder: external` | | [Python](./python/README.md) | Python Development | Lint (ruff), tests (pytest), security (bandit), Docker build | | [Linting](./linting/README.md) | Linting | YAML, Markdown, pre-commit hooks | | [Templating](./templating/README.md) | Go Templating | Render Go text/template from file or URL with YAML/CLI vars | diff --git a/Taskfile.yaml b/Taskfile.yaml index 2f53ee8b..f3da20c9 100644 --- a/Taskfile.yaml +++ b/Taskfile.yaml @@ -460,6 +460,11 @@ tasks: cmds: - ./tests/smoke/sops.sh + test-techdocs: + desc: Test techdocs (builds the fixture, no bucket or credentials needed) + cmds: + - ./tests/smoke/techdocs.sh + test-cosign: desc: Test cosign verification and signing refusals (public registries, no credentials needed) cmds: diff --git a/techdocs/.gitattributes b/techdocs/.gitattributes new file mode 100644 index 00000000..3a454933 --- /dev/null +++ b/techdocs/.gitattributes @@ -0,0 +1,4 @@ +/dagger.gen.go linguist-generated +/internal/dagger/** linguist-generated +/internal/querybuilder/** linguist-generated +/internal/telemetry/** linguist-generated diff --git a/techdocs/.gitignore b/techdocs/.gitignore new file mode 100644 index 00000000..773338bf --- /dev/null +++ b/techdocs/.gitignore @@ -0,0 +1,5 @@ +/dagger.gen.go +/internal/dagger +/internal/querybuilder +/internal/telemetry +/.env diff --git a/techdocs/README.md b/techdocs/README.md new file mode 100644 index 00000000..aa3d315c --- /dev/null +++ b/techdocs/README.md @@ -0,0 +1,66 @@ +# techdocs + +Builds a repository's [Backstage TechDocs](https://backstage.io/docs/features/techdocs/) +site in CI and publishes it to S3-compatible storage (MinIO). Backstage then +runs with `techdocs.builder: external` and an `awsS3` publisher: it only reads +the finished HTML and never runs mkdocs in its own pod. + +Tools (pinned, Renovate-managed in `container.go`): `mkdocs-techdocs-core`, +`@techdocs/cli`, Node.js from nodejs.org (checked against `SHASUMS256.txt`) on +`python:3.13-slim`. + +## Functions + +| Function | What it does | +|---|---| +| `generate` | `techdocs-cli generate --no-docker` on `--src` (mkdocs.yml in `--source-dir`, default `.`); returns the site directory | +| `entity` | `//` of the entity in `catalog-info.yaml` that carries `backstage.io/techdocs-ref` (namespace defaults to `default`) | +| `publish` | `techdocs-cli publish --publisher-type awsS3 --awsS3ForcePathStyle` of a site to `////`; removes stale files of the previous publish | +| `generate-and-publish` | all of the above; `--entity` is read from the catalog file unless given | + +`publish` never comes from cache: an upload is a side effect. + +## Usage + +```bash +# Build only (e.g. in a PR), export the site to look at it +dagger call -m github.com/stuttgart-things/dagger/techdocs@ \ + generate --src . export --path ./site + +# Which path will Backstage read? +dagger call -m github.com/stuttgart-things/dagger/techdocs@ \ + entity --src . + +# Build and publish (main branch) +dagger call -m github.com/stuttgart-things/dagger/techdocs@ \ + generate-and-publish \ + --src . \ + --bucket techdocs \ + --endpoint https://minio.example.com \ + --access-key env:TECHDOCS_S3_ACCESS_KEY \ + --secret-key env:TECHDOCS_S3_SECRET_KEY \ + --ca-cert ./ca.crt # only for an endpoint signed by a private CA +``` + +The S3 user needs `s3:ListBucket` and `s3:GetBucketLocation` on the bucket and +`s3:GetObject`, `s3:PutObject`, `s3:DeleteObject` on its objects (delete is +used to remove stale pages). + +## Backstage side + +```yaml +techdocs: + builder: external + publisher: + type: awsS3 + awsS3: + bucketName: techdocs + endpoint: https://minio.example.com + s3ForcePathStyle: true + region: us-east-1 + credentials: + accessKeyId: ${TECHDOCS_S3_ACCESS_KEY_ID} + secretAccessKey: ${TECHDOCS_S3_SECRET_ACCESS_KEY} +``` + +Backstage needs read access only. diff --git a/techdocs/container.go b/techdocs/container.go new file mode 100644 index 00000000..a75aa908 --- /dev/null +++ b/techdocs/container.go @@ -0,0 +1,77 @@ +package main + +import ( + "context" + "dagger/techdocs/internal/dagger" + "fmt" + "strings" +) + +// Pinned tool versions. +// renovate: datasource=pypi depName=mkdocs-techdocs-core +const techdocsCoreVersion = "1.7.1" + +// renovate: datasource=npm depName=@techdocs/cli +const techdocsCliVersion = "1.12.0" + +// Node.js is downloaded from nodejs.org and checked against the release's +// SHASUMS256.txt instead of installed with apt, so the version does not +// depend on the day of the run. The tarball is .tar.gz: the slim python +// image has no xz. +// renovate: datasource=node-version depName=node +const nodeVersion = "22.23.3" + +// The base image, pinned by digest. It brings python3 + pip for mkdocs; +// node comes from nodejs.org above. Renovate moves the digest. +// renovate: datasource=docker depName=python +const baseImageTag = "3.13-slim@sha256:70729b46c69b4f1e97c4822c1af3df53a1476cf5ddc6c087c0c10bc3a5678c2f" + +const defaultBaseImage = "python:" + baseImageTag + +func (m *Techdocs) container(ctx context.Context) (*dagger.Container, error) { + if m.BaseImage == "" { + m.BaseImage = defaultBaseImage + } + + arch, err := nodeArch(ctx) + if err != nil { + return nil, err + } + + nodeURL := fmt.Sprintf("https://nodejs.org/dist/v%s/", nodeVersion) + tarball := fmt.Sprintf("node-v%s-linux-%s.tar.gz", nodeVersion, arch) + + return dag.Container(). + From(m.BaseImage). + WithMountedFile("/tmp/dl/"+tarball, dag.HTTP(nodeURL+tarball)). + WithMountedFile("/tmp/dl/SHASUMS256.txt", dag.HTTP(nodeURL+"SHASUMS256.txt")). + WithWorkdir("/tmp/dl"). + WithExec([]string{"sh", "-c", + `grep " ` + tarball + `$" SHASUMS256.txt | sha256sum -c - && ` + + `tar -xzf ` + tarball + ` -C /usr/local --strip-components=1 ` + + `--exclude='*/CHANGELOG.md' --exclude='*/README.md' --exclude='*/LICENSE'`, + }). + WithWorkdir("/"). + WithExec([]string{"pip", "install", "--no-cache-dir", "--root-user-action=ignore", + "mkdocs-techdocs-core==" + techdocsCoreVersion}). + WithExec([]string{"npm", "install", "--global", "--no-fund", "--no-audit", + "@techdocs/cli@" + techdocsCliVersion}), nil +} + +// nodeArch maps the engine's platform (e.g. linux/arm64) to the architecture +// name used in the Node.js release assets. +func nodeArch(ctx context.Context) (string, error) { + platform, err := dag.DefaultPlatform(ctx) + if err != nil { + return "", fmt.Errorf("failed to read engine platform: %w", err) + } + + switch arch := strings.Split(strings.TrimPrefix(string(platform), "linux/"), "/")[0]; arch { + case "amd64": + return "x64", nil + case "arm64": + return "arm64", nil + default: + return "", fmt.Errorf("unsupported platform %q: node is downloaded for linux/amd64 or linux/arm64", platform) + } +} diff --git a/techdocs/dagger.json b/techdocs/dagger.json new file mode 100644 index 00000000..dd111c4b --- /dev/null +++ b/techdocs/dagger.json @@ -0,0 +1,7 @@ +{ + "name": "techdocs", + "engineVersion": "v0.21.9", + "sdk": { + "source": "go" + } +} diff --git a/techdocs/entity.go b/techdocs/entity.go new file mode 100644 index 00000000..63a9bd17 --- /dev/null +++ b/techdocs/entity.go @@ -0,0 +1,83 @@ +package main + +import ( + "context" + "dagger/techdocs/internal/dagger" + "errors" + "fmt" + "io" + "strings" + + "gopkg.in/yaml.v3" +) + +const techdocsRefAnnotation = "backstage.io/techdocs-ref" + +type catalogEntity struct { + Kind string `yaml:"kind"` + Metadata struct { + Name string `yaml:"name"` + Namespace string `yaml:"namespace"` + Annotations map[string]string `yaml:"annotations"` + } `yaml:"metadata"` +} + +// Entity returns // of the entity in catalogFile that +// carries the backstage.io/techdocs-ref annotation -- the path Backstage +// looks up the docs under. The namespace defaults to "default". +func (m *Techdocs) Entity( + ctx context.Context, + // Repository directory containing catalogFile + src *dagger.Directory, + // Catalog file inside src to read the entity from + // +optional + // +default="catalog-info.yaml" + catalogFile string, +) (string, error) { + content, err := src.File(catalogFile).Contents(ctx) + if err != nil { + return "", fmt.Errorf("reading %s: %w", catalogFile, err) + } + return entityRef(content) +} + +// entityRef picks the entity with a techdocs-ref annotation out of a +// (possibly multi-document) catalog file. +func entityRef(content string) (string, error) { + dec := yaml.NewDecoder(strings.NewReader(content)) + var withDocs []catalogEntity + for { + var e catalogEntity + err := dec.Decode(&e) + if errors.Is(err, io.EOF) { + break + } + if err != nil { + return "", fmt.Errorf("parsing catalog file: %w", err) + } + if _, ok := e.Metadata.Annotations[techdocsRefAnnotation]; ok { + withDocs = append(withDocs, e) + } + } + + switch len(withDocs) { + case 0: + return "", fmt.Errorf("no entity with a %s annotation; pass --entity", techdocsRefAnnotation) + case 1: + default: + return "", fmt.Errorf("%d entities carry %s; pass --entity to pick one", len(withDocs), techdocsRefAnnotation) + } + + e := withDocs[0] + if e.Kind == "" || e.Metadata.Name == "" { + return "", errors.New("the entity with techdocs-ref has no kind or metadata.name") + } + ns := e.Metadata.Namespace + if ns == "" { + ns = "default" + } + // Backstage looks docs up lower-cased (unless + // legacyUseCaseSensitiveTripletPaths); techdocs-cli publish lower-cases + // too, so this only keeps the log line honest. + return strings.ToLower(fmt.Sprintf("%s/%s/%s", ns, e.Kind, e.Metadata.Name)), nil +} diff --git a/techdocs/entity_test.go b/techdocs/entity_test.go new file mode 100644 index 00000000..942a93c9 --- /dev/null +++ b/techdocs/entity_test.go @@ -0,0 +1,74 @@ +package main + +import "testing" + +func TestEntityRef(t *testing.T) { + tests := []struct { + name string + content string + want string + wantErr bool + }{ + { + name: "single component, default namespace", + content: `apiVersion: backstage.io/v1alpha1 +kind: Component +metadata: + name: flux + annotations: + backstage.io/techdocs-ref: dir:. +`, + want: "default/component/flux", + }, + { + name: "multi-document, only one with docs", + content: `apiVersion: backstage.io/v1alpha1 +kind: System +metadata: + name: homerun2 +--- +apiVersion: backstage.io/v1alpha1 +kind: Component +metadata: + name: homerun2-k8s-pitcher + namespace: Platform + annotations: + backstage.io/techdocs-ref: url:https://github.com/stuttgart-things/homerun2-k8s-pitcher/tree/main +`, + want: "platform/component/homerun2-k8s-pitcher", + }, + { + name: "no techdocs-ref", + content: `kind: Component +metadata: + name: x +`, + wantErr: true, + }, + { + name: "two entities with techdocs-ref", + content: `kind: Component +metadata: + name: a + annotations: {backstage.io/techdocs-ref: dir:.} +--- +kind: Component +metadata: + name: b + annotations: {backstage.io/techdocs-ref: dir:.} +`, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := entityRef(tt.content) + if (err != nil) != tt.wantErr { + t.Fatalf("err = %v, wantErr %v", err, tt.wantErr) + } + if got != tt.want { + t.Errorf("got %q, want %q", got, tt.want) + } + }) + } +} diff --git a/techdocs/go.mod b/techdocs/go.mod new file mode 100644 index 00000000..5a07509d --- /dev/null +++ b/techdocs/go.mod @@ -0,0 +1,58 @@ +module dagger/techdocs + +go 1.26.7 + +require ( + github.com/99designs/gqlgen v0.17.90 + github.com/Khan/genqlient v0.8.1 + github.com/dagger/otel-go v1.43.0 + github.com/vektah/gqlparser/v2 v2.5.37 + go.opentelemetry.io/otel v1.46.0 + go.opentelemetry.io/otel/trace v1.46.0 + golang.org/x/sync v0.22.0 + gopkg.in/yaml.v3 v3.0.1 +) + +require ( + github.com/cenkalti/backoff/v5 v5.0.3 // indirect + github.com/kr/text v0.2.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.17.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.17.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.46.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.46.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.46.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.46.0 // indirect + go.opentelemetry.io/otel/log v0.17.0 // indirect + go.opentelemetry.io/otel/metric v1.46.0 // indirect + go.opentelemetry.io/otel/sdk v1.46.0 // indirect + go.opentelemetry.io/otel/sdk/log v0.17.0 // indirect + go.opentelemetry.io/otel/sdk/metric v1.46.0 // indirect + go.opentelemetry.io/proto/otlp v1.11.0 // indirect + google.golang.org/grpc v1.83.1 // indirect +) + +require ( + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/dagger/querybuilder v0.0.0-20260402040506-574a5e81cb59 + github.com/go-logr/logr v1.4.4 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0 // indirect + github.com/sosodev/duration v1.4.0 // indirect + go.opentelemetry.io/auto/sdk v1.2.1 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.46.0 // indirect + golang.org/x/net v0.58.0 // indirect + golang.org/x/sys v0.47.0 // indirect + golang.org/x/text v0.41.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20260819154853-08b0e4226688 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260819154853-08b0e4226688 // indirect + google.golang.org/protobuf v1.36.12 // indirect +) + +replace go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc => go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.16.0 + +replace go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp => go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.16.0 + +replace go.opentelemetry.io/otel/log => go.opentelemetry.io/otel/log v0.16.0 + +replace go.opentelemetry.io/otel/sdk/log => go.opentelemetry.io/otel/sdk/log v0.16.0 diff --git a/techdocs/go.sum b/techdocs/go.sum new file mode 100644 index 00000000..9ccd34c1 --- /dev/null +++ b/techdocs/go.sum @@ -0,0 +1,103 @@ +github.com/99designs/gqlgen v0.17.90 h1:wSv6blm/PoplU6QoNw83EcQpNtC0HX3/+44vITJOzpk= +github.com/99designs/gqlgen v0.17.90/go.mod h1:GqYrEwYsqCG8VaOsq2kJUCUKwAE1T+u2i+Nj7NtXiVI= +github.com/Khan/genqlient v0.8.1 h1:wtOCc8N9rNynRLXN3k3CnfzheCUNKBcvXmVv5zt6WCs= +github.com/Khan/genqlient v0.8.1/go.mod h1:R2G6DzjBvCbhjsEajfRjbWdVglSH/73kSivC9TLWVjU= +github.com/agnivade/levenshtein v1.2.1 h1:EHBY3UOn1gwdy/VbFwgo4cxecRznFk7fKWN1KOX7eoM= +github.com/agnivade/levenshtein v1.2.1/go.mod h1:QVVI16kDrtSuwcpd0p1+xMC6Z/VfhtCyDIjcwga4/DU= +github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM= +github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/dagger/otel-go v1.43.0 h1:AYCnAamWmxtSxigWPTgC+8EWqiWPcDZEegh8y05gdJ8= +github.com/dagger/otel-go v1.43.0/go.mod h1:83CTuXi70zcx1kaym5buqmb7RNzg1E9dEiQSFyLbLdU= +github.com/dagger/querybuilder v0.0.0-20260402040506-574a5e81cb59 h1:g6vfdGRyz6fAjfHz5FyYPZgHy8qcQ31fHrBl1iCOzxw= +github.com/dagger/querybuilder v0.0.0-20260402040506-574a5e81cb59/go.mod h1:jsdUJeYzcbyK1j/EqMGPrQgNYxl/Zfg06vvM9C/xXxs= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.4 h1:tG4xh9yMsRCAiodLVTxyrkzSZ9+o0L1Kg/+cPVcbP/8= +github.com/go-logr/logr v1.4.4/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0 h1:/Tnpcb2E0Pz/tN9s3bfEY2Q8ePCEX9iuS+cneUwncnw= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0/go.mod h1:zOBXOsUaBSjKgmH4OGzV1esUpR3oUSCPYVd2cUBjKYY= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= +github.com/sosodev/duration v1.4.0 h1:35ed0KiVFriGHHzZZJaZLgmTEEICIyt8Sx0RQfj9IjE= +github.com/sosodev/duration v1.4.0/go.mod h1:RQIBBX0+fMLc/D9+Jb/fwvVmo0eZvDDEERAikUR6SDg= +github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE= +github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg= +github.com/vektah/gqlparser/v2 v2.5.37 h1:jbb1Ilv+xBklV6653tKb4oVUupPNTLb5LmrnBKVI12Y= +github.com/vektah/gqlparser/v2 v2.5.37/go.mod h1:9O4Ox6Ngd3Y12bMD3w6i3CRQXh8W1oC1q0m6olCymDM= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/otel v1.46.0 h1:FHt5/CDyVxi/8IM1CH7VE/rRgq3kLHa2mSTVMO8AWyc= +go.opentelemetry.io/otel v1.46.0/go.mod h1:Gj3SEScelsNC45tp4nSxRYlS+f5iez7W8XPMCt905kE= +go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.16.0 h1:ZVg+kCXxd9LtAaQNKBxAvJ5NpMf7LpvEr4MIZqb0TMQ= +go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.16.0/go.mod h1:hh0tMeZ75CCXrHd9OXRYxTlCAdxcXioWHFIpYw2rZu8= +go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.16.0 h1:djrxvDxAe44mJUrKataUbOhCKhR3F8QCyWucO16hTQs= +go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.16.0/go.mod h1:dt3nxpQEiSoKvfTVxp3TUg5fHPLhKtbcnN3Z1I1ePD0= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.46.0 h1:qkDYCAFiZXLcs1L4aY+tP2wguQ4kURANqHOQMA2et2s= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.46.0/go.mod h1:tkipS4DRzmpAmvg+Gw4++O1IdDq6TVDnvnYU6cmbQVs= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.46.0 h1:AP23h/mFgb/lc7tdck1Kfn9qxsM8TAeNPCU5C3pzaps= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.46.0/go.mod h1:K4EqCe1b4kGk5WR690ntg9LaBfsPoV32FwthbyoptuA= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.46.0 h1:OFnwLJr+pF3iHrlGSzbxyuo6/6HyBlnlN1CWEJmBVcw= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.46.0/go.mod h1:716wFneO0ov19A2beH5hjfh9AK5z/VWNAtDijp1Y0/g= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.46.0 h1:w53CDeOA/Kurp7yRsegSr6pbbr759dOvJ+yNmWM6Hxs= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.46.0/go.mod h1:BOmGMCbAtvcJiSJ+hLuhgPLdDbimnraSl8irz3iY8sY= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.46.0 h1:KrC1YrQeSt46ITMWAbgQx1M1eV1/1TKzttrBzymPmss= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.46.0/go.mod h1:zDSEzoEqsOrgBeGvH66KRgxh90VonFyJqBHA0Pk3+rM= +go.opentelemetry.io/otel/log v0.16.0 h1:DeuBPqCi6pQwtCK0pO4fvMB5eBq6sNxEnuTs88pjsN4= +go.opentelemetry.io/otel/log v0.16.0/go.mod h1:rWsmqNVTLIA8UnwYVOItjyEZDbKIkMxdQunsIhpUMes= +go.opentelemetry.io/otel/metric v1.46.0 h1:yBnkXvgV7AXFILZc5K6IZe/CBFF3OS7BJ8ov6/lj0K8= +go.opentelemetry.io/otel/metric v1.46.0/go.mod h1:iPmdWqifKUdzziPkvvzIJXITl56fQx2mGM/DHLB3/2o= +go.opentelemetry.io/otel/metric/x v0.68.0 h1:TA/cBT23D3MnxYPwHL7YFOdYGdx0A0v+s7Mzotpd1dU= +go.opentelemetry.io/otel/metric/x v0.68.0/go.mod h1:agudOmvWhwUTjgibWDzxD2PoWYnpw5Ht5jISYOD2Hd4= +go.opentelemetry.io/otel/sdk v1.46.0 h1:h5CNQQjEbuQXY/JfZtgt3i7HVFV3aHPO2OAwO2eTYPI= +go.opentelemetry.io/otel/sdk v1.46.0/go.mod h1:GAERFXFt5SYCEB+YiKUbMBeza6UaDH7GmGOZEfh2gSM= +go.opentelemetry.io/otel/sdk/log v0.16.0 h1:e/b4bdlQwC5fnGtG3dlXUrNOnP7c8YLVSpSfEBIkTnI= +go.opentelemetry.io/otel/sdk/log v0.16.0/go.mod h1:JKfP3T6ycy7QEuv3Hj8oKDy7KItrEkus8XJE6EoSzw4= +go.opentelemetry.io/otel/sdk/log/logtest v0.16.0 h1:/XVkpZ41rVRTP4DfMgYv1nEtNmf65XPPyAdqV90TMy4= +go.opentelemetry.io/otel/sdk/log/logtest v0.16.0/go.mod h1:iOOPgQr5MY9oac/F5W86mXdeyWZGleIx3uXO98X2R6Y= +go.opentelemetry.io/otel/sdk/metric v1.46.0 h1:0piZ26EG4RBfebb2jhDH6ERCYHoVWduc3kLgPCwSnSE= +go.opentelemetry.io/otel/sdk/metric v1.46.0/go.mod h1:I1PbKrdVc8Qu8HYVDNtqVIwLwjNrhsV/uFuxfwg8mO4= +go.opentelemetry.io/otel/trace v1.46.0 h1:OULy7ccdJnZtJ0UDYFOIGaCmiWzJ8Vi2G/Rsu60qs1c= +go.opentelemetry.io/otel/trace v1.46.0/go.mod h1:J7GAXweO77XSFkB/rmAqk9D6ihszhFjLU+d9WuUxDLI= +go.opentelemetry.io/proto/otlp v1.11.0 h1:5rrYs0Ykyj50sdU/JU0x8etU+LubXWb+gED6TbEdMIk= +go.opentelemetry.io/proto/otlp v1.11.0/go.mod h1:SmVizdCOAm3XBtG1g1NnOdhW6jtddT72hLMhv8VwA8E= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= +go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= +golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To= +golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU= +golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= +golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= +golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8= +golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M= +gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= +gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= +google.golang.org/genproto/googleapis/api v0.0.0-20260819154853-08b0e4226688 h1:ax2KzoSRIZU/M0cIxri3pKxy99vniH1PVxWC6si/eZI= +google.golang.org/genproto/googleapis/api v0.0.0-20260819154853-08b0e4226688/go.mod h1:1RJ9BQGyNdZwkGc1eTqkErfRZ6RJyYPHZo73BZ1vQqI= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260819154853-08b0e4226688 h1:cYNAzI2sUwhmCcoj9TxvihSrqsxt6uIkj3rDRhSDmW4= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260819154853-08b0e4226688/go.mod h1:DjtHYE8FKJLivXcBEjGwndXfIC23G0VpXiXKqG179uA= +google.golang.org/grpc v1.83.1 h1:HIO0+BEtBP6soyqvqC8sNUjZ7bTs+0hFQuFF+RAy++Y= +google.golang.org/grpc v1.83.1/go.mod h1:kDyl6SKsiHKt0uylY5gtn5cEjkrIOhQOGDgIc4JGwzQ= +google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc= +google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/techdocs/main.go b/techdocs/main.go new file mode 100644 index 00000000..7e0bf584 --- /dev/null +++ b/techdocs/main.go @@ -0,0 +1,171 @@ +// A Dagger module for Backstage TechDocs +// +// Builds a repository's TechDocs site in CI and publishes it to S3-compatible +// storage (MinIO), so Backstage runs with `techdocs.builder: external` and +// only reads the finished HTML instead of running mkdocs in its own pod. +// +// The site is built with `techdocs-cli generate --no-docker` (mkdocs + +// mkdocs-techdocs-core) and uploaded with `techdocs-cli publish +// --publisher-type awsS3` to ////, the path +// Backstage's awsS3 publisher reads. +// +// Functions: +// - Generate: builds the site of a source directory +// - Entity: reads // from catalog-info.yaml +// - Publish: uploads a generated site for an entity +// - GenerateAndPublish: all of the above in one call + +package main + +import ( + "context" + "dagger/techdocs/internal/dagger" + "fmt" + "time" +) + +type Techdocs struct { + BaseImage string +} + +// Generate builds the TechDocs site of src (the directory holding mkdocs.yml) +// and returns it. +func (m *Techdocs) Generate( + ctx context.Context, + // Repository (or docs) directory containing mkdocs.yml + src *dagger.Directory, + // Directory of mkdocs.yml inside src + // +optional + // +default="." + sourceDir string, +) (*dagger.Directory, error) { + ctr, err := m.container(ctx) + if err != nil { + return nil, err + } + + // techdocs-cli rewrites mkdocs.yml (it adds the techdocs-core plugin if it + // is missing), so src is copied in rather than mounted read-only. + return ctr. + WithDirectory("/src", src). + WithWorkdir("/src"). + WithExec([]string{ + "techdocs-cli", "generate", + "--no-docker", + "--source-dir", sourceDir, + "--output-dir", "/site", + "--verbose", + }). + Directory("/site"), nil +} + +// Publish uploads a site built by Generate to an S3-compatible bucket, under +// the path Backstage reads for entity. Stale files of a previous publish of +// the same entity are removed. +func (m *Techdocs) Publish( + ctx context.Context, + // Site directory returned by Generate + site *dagger.Directory, + // Entity as //, e.g. default/component/flux + entity string, + // Bucket Backstage reads (techdocs.publisher.awsS3.bucketName) + bucket string, + // S3 endpoint, e.g. https://minio.example.com + endpoint string, + // S3 access key of a user that may write to bucket + accessKey *dagger.Secret, + // S3 secret key + secretKey *dagger.Secret, + // Region; MinIO accepts any value + // +optional + // +default="us-east-1" + region string, + // PEM CA certificate for an endpoint signed by a private CA + // +optional + caCert *dagger.File, +) (string, error) { + ctr, err := m.container(ctx) + if err != nil { + return "", err + } + + if caCert != nil { + ctr = ctr. + WithMountedFile("/etc/ssl/custom/ca.crt", caCert). + WithEnvVariable("NODE_EXTRA_CA_CERTS", "/etc/ssl/custom/ca.crt") + } + + return ctr. + WithMountedDirectory("/site", site). + WithSecretVariable("AWS_ACCESS_KEY_ID", accessKey). + WithSecretVariable("AWS_SECRET_ACCESS_KEY", secretKey). + WithEnvVariable("AWS_REGION", region). + // An upload is a side effect: a cached run would skip it after the + // bucket was emptied or another commit published in between. + WithEnvVariable("TECHDOCS_PUBLISH_AT", time.Now().UTC().Format(time.RFC3339Nano)). + WithExec([]string{ + "techdocs-cli", "publish", + "--publisher-type", "awsS3", + "--storage-name", bucket, + "--entity", entity, + "--directory", "/site", + "--awsEndpoint", endpoint, + // MinIO serves buckets by path, not by virtual host. + "--awsS3ForcePathStyle", + }). + // techdocs-cli logs to stderr. + CombinedOutput(ctx) +} + +// GenerateAndPublish builds the site of src and publishes it. The entity is +// read from catalog-info.yaml unless given. +func (m *Techdocs) GenerateAndPublish( + ctx context.Context, + // Repository directory containing mkdocs.yml and catalog-info.yaml + src *dagger.Directory, + // Bucket Backstage reads (techdocs.publisher.awsS3.bucketName) + bucket string, + // S3 endpoint, e.g. https://minio.example.com + endpoint string, + // S3 access key of a user that may write to bucket + accessKey *dagger.Secret, + // S3 secret key + secretKey *dagger.Secret, + // Entity as //; read from catalogFile if empty + // +optional + entity string, + // Catalog file inside src to read the entity from + // +optional + // +default="catalog-info.yaml" + catalogFile string, + // Directory of mkdocs.yml inside src + // +optional + // +default="." + sourceDir string, + // Region; MinIO accepts any value + // +optional + // +default="us-east-1" + region string, + // PEM CA certificate for an endpoint signed by a private CA + // +optional + caCert *dagger.File, +) (string, error) { + if entity == "" { + var err error + entity, err = m.Entity(ctx, src, catalogFile) + if err != nil { + return "", err + } + } + + site, err := m.Generate(ctx, src, sourceDir) + if err != nil { + return "", fmt.Errorf("generate: %w", err) + } + + out, err := m.Publish(ctx, site, entity, bucket, endpoint, accessKey, secretKey, region, caCert) + if err != nil { + return "", fmt.Errorf("publish %s: %w", entity, err) + } + return fmt.Sprintf("published %s to %s/%s\n%s", entity, endpoint, bucket, out), nil +} diff --git a/tests/smoke/techdocs.sh b/tests/smoke/techdocs.sh new file mode 100755 index 00000000..0f6843ff --- /dev/null +++ b/tests/smoke/techdocs.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash +# Smoke test for the techdocs module. +# +# `dagger functions` and `dagger develop` pass whether or not the container can +# actually build docs: node is downloaded and checked against SHASUMS256.txt, +# and mkdocs-techdocs-core and @techdocs/cli are installed at pinned versions, +# all at runtime. This builds the fixture under tests/techdocs and reads its +# entity, which needs neither secrets nor a bucket. publish is not covered +# here: it needs S3 credentials. +set -euo pipefail + +cd "$(git rev-parse --show-toplevel)" + +MODULE=techdocs +FIXTURE=tests/techdocs + +fail=0 + +# expect +expect() { + local desc=$1 needle=$2 out + shift 2 + if ! out=$(dagger call -m "./${MODULE}" "$@" 2>&1); then + echo "FAIL: ${desc} (call failed)" + echo "${out}" + fail=1 + elif ! grep -qF -- "${needle}" <<<"${out}"; then + echo "FAIL: ${desc} (missing: ${needle})" + echo "${out}" + fail=1 + else + echo "OK: ${desc}" + fi +} + +# The entity Backstage would read the docs under: the Component carrying +# techdocs-ref, not the System before it. +expect "entity from a multi-document catalog-info.yaml" \ + "default/component/techdocs-smoke" \ + entity --src "${FIXTURE}" + +# techdocs_metadata.json is what Backstage reads first; without it the page +# stays empty even if the HTML is there. +expect "generate writes techdocs_metadata.json" \ + '"site_name": "techdocs-smoke"' \ + generate --src "${FIXTURE}" file --path techdocs_metadata.json contents + +expect "generate renders the page" \ + "techdocs smoke test" \ + generate --src "${FIXTURE}" file --path index.html contents + +exit "${fail}" diff --git a/tests/techdocs/catalog-info.yaml b/tests/techdocs/catalog-info.yaml new file mode 100644 index 00000000..8e0eec44 --- /dev/null +++ b/tests/techdocs/catalog-info.yaml @@ -0,0 +1,16 @@ +--- +apiVersion: backstage.io/v1alpha1 +kind: System +metadata: + name: techdocs-smoke-system +--- +apiVersion: backstage.io/v1alpha1 +kind: Component +metadata: + name: techdocs-smoke + annotations: + backstage.io/techdocs-ref: dir:. +spec: + type: library + lifecycle: experimental + owner: stuttgart-things diff --git a/tests/techdocs/docs/index.md b/tests/techdocs/docs/index.md new file mode 100644 index 00000000..8b023a1c --- /dev/null +++ b/tests/techdocs/docs/index.md @@ -0,0 +1,3 @@ +# techdocs smoke test + +Fixture for `tests/smoke/techdocs.sh`. diff --git a/tests/techdocs/mkdocs.yml b/tests/techdocs/mkdocs.yml new file mode 100644 index 00000000..5cdeedad --- /dev/null +++ b/tests/techdocs/mkdocs.yml @@ -0,0 +1,5 @@ +site_name: techdocs-smoke +nav: + - Home: index.md +plugins: + - techdocs-core