feat(techdocs): build Backstage TechDocs in CI and publish them to S3/MinIO

.github/workflows/this-test-modules.yaml

@@ -188,5 +188,5 @@ jobs:
       # secrets (SOPS_AGE_KEY, registry creds) unavailable here. Add a module
       # once tests/smoke/<module>.sh exists and runs without secrets.
       - name: smoke test ${{ matrix.module }}
-        if: contains(fromJSON('["ansible", "cosign", "crossplane", "kyverno", "sops", "terraform"]'), matrix.module)
+        if: contains(fromJSON('["ansible", "cosign", "crossplane", "kyverno", "sops", "techdocs", "terraform"]'), matrix.module)
         run: ./tests/smoke/${{ matrix.module }}.sh

README.md

@@ -29,6 +29,7 @@ modular collection of Dagger building blocks for infrastructure, containers, sec
 | [OCI](./oci/README.md)             | OCI Artifact Management         | Flux push, batch push, local TLS registry testing |
 | [Flux](./flux/README.md)           | Flux GitOps Artifacts           | Build/push OCI artifacts, batch push, registry auth |
 | [Slidev](./slidev/README.md)       | Presentations / Decks           | Scaffold, dev server, static build from a single `slides.md` |
+| [TechDocs](./techdocs/README.md)   | Backstage TechDocs              | Build docs in CI, publish to S3/MinIO for `builder: external` |
 | [Python](./python/README.md)       | Python Development              | Lint (ruff), tests (pytest), security (bandit), Docker build |
 | [Linting](./linting/README.md)     | Linting                         | YAML, Markdown, pre-commit hooks |
 | [Templating](./templating/README.md) | Go Templating                 | Render Go text/template from file or URL with YAML/CLI vars |

Taskfile.yaml

@@ -460,6 +460,11 @@ tasks:
     cmds:
       - ./tests/smoke/sops.sh
 
+  test-techdocs:
+    desc: Test techdocs (builds the fixture, no bucket or credentials needed)
+    cmds:
+      - ./tests/smoke/techdocs.sh
+
   test-cosign:
     desc: Test cosign verification and signing refusals (public registries, no credentials needed)
     cmds:

techdocs/.gitattributes

@@ -0,0 +1,4 @@
+/dagger.gen.go linguist-generated
+/internal/dagger/** linguist-generated
+/internal/querybuilder/** linguist-generated
+/internal/telemetry/** linguist-generated

techdocs/.gitignore

@@ -0,0 +1,5 @@
+/dagger.gen.go
+/internal/dagger
+/internal/querybuilder
+/internal/telemetry
+/.env

techdocs/README.md

@@ -0,0 +1,66 @@
+# techdocs
+
+Builds a repository's [Backstage TechDocs](https://backstage.io/docs/features/techdocs/)
+site in CI and publishes it to S3-compatible storage (MinIO). Backstage then
+runs with `techdocs.builder: external` and an `awsS3` publisher: it only reads
+the finished HTML and never runs mkdocs in its own pod.
+
+Tools (pinned, Renovate-managed in `container.go`): `mkdocs-techdocs-core`,
+`@techdocs/cli`, Node.js from nodejs.org (checked against `SHASUMS256.txt`) on
+`python:3.13-slim`.
+
+## Functions
+
+| Function | What it does |
+|---|---|
+| `generate` | `techdocs-cli generate --no-docker` on `--src` (mkdocs.yml in `--source-dir`, default `.`); returns the site directory |
+| `entity` | `<namespace>/<kind>/<name>` of the entity in `catalog-info.yaml` that carries `backstage.io/techdocs-ref` (namespace defaults to `default`) |
+| `publish` | `techdocs-cli publish --publisher-type awsS3 --awsS3ForcePathStyle` of a site to `<bucket>/<namespace>/<kind>/<name>/`; removes stale files of the previous publish |
+| `generate-and-publish` | all of the above; `--entity` is read from the catalog file unless given |
+
+`publish` never comes from cache: an upload is a side effect.
+
+## Usage
+
+```bash
+# Build only (e.g. in a PR), export the site to look at it
+dagger call -m github.com/stuttgart-things/dagger/techdocs@<version> \
+  generate --src . export --path ./site
+
+# Which path will Backstage read?
+dagger call -m github.com/stuttgart-things/dagger/techdocs@<version> \
+  entity --src .
+
+# Build and publish (main branch)
+dagger call -m github.com/stuttgart-things/dagger/techdocs@<version> \
+  generate-and-publish \
+  --src . \
+  --bucket techdocs \
+  --endpoint https://minio.example.com \
+  --access-key env:TECHDOCS_S3_ACCESS_KEY \
+  --secret-key env:TECHDOCS_S3_SECRET_KEY \
+  --ca-cert ./ca.crt   # only for an endpoint signed by a private CA
+```
+
+The S3 user needs `s3:ListBucket` and `s3:GetBucketLocation` on the bucket and
+`s3:GetObject`, `s3:PutObject`, `s3:DeleteObject` on its objects (delete is
+used to remove stale pages).
+
+## Backstage side
+
+```yaml
+techdocs:
+  builder: external
+  publisher:
+    type: awsS3
+    awsS3:
+      bucketName: techdocs
+      endpoint: https://minio.example.com
+      s3ForcePathStyle: true
+      region: us-east-1
+      credentials:
+        accessKeyId: ${TECHDOCS_S3_ACCESS_KEY_ID}
+        secretAccessKey: ${TECHDOCS_S3_SECRET_ACCESS_KEY}
+```
+
+Backstage needs read access only.

techdocs/container.go

@@ -0,0 +1,77 @@
+package main
+
+import (
+	"context"
+	"dagger/techdocs/internal/dagger"
+	"fmt"
+	"strings"
+)
+
+// Pinned tool versions.
+// renovate: datasource=pypi depName=mkdocs-techdocs-core
+const techdocsCoreVersion = "1.7.1"
+
+// renovate: datasource=npm depName=@techdocs/cli
+const techdocsCliVersion = "1.12.0"
+
+// Node.js is downloaded from nodejs.org and checked against the release's
+// SHASUMS256.txt instead of installed with apt, so the version does not
+// depend on the day of the run. The tarball is .tar.gz: the slim python
+// image has no xz.
+// renovate: datasource=node-version depName=node
+const nodeVersion = "22.23.3"
+
+// The base image, pinned by digest. It brings python3 + pip for mkdocs;
+// node comes from nodejs.org above. Renovate moves the digest.
+// renovate: datasource=docker depName=python
+const baseImageTag = "3.13-slim@sha256:70729b46c69b4f1e97c4822c1af3df53a1476cf5ddc6c087c0c10bc3a5678c2f"
+
+const defaultBaseImage = "python:" + baseImageTag
+
+func (m *Techdocs) container(ctx context.Context) (*dagger.Container, error) {
+	if m.BaseImage == "" {
+		m.BaseImage = defaultBaseImage
+	}
+
+	arch, err := nodeArch(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	nodeURL := fmt.Sprintf("https://nodejs.org/dist/v%s/", nodeVersion)
+	tarball := fmt.Sprintf("node-v%s-linux-%s.tar.gz", nodeVersion, arch)
+
+	return dag.Container().
+		From(m.BaseImage).
+		WithMountedFile("/tmp/dl/"+tarball, dag.HTTP(nodeURL+tarball)).
+		WithMountedFile("/tmp/dl/SHASUMS256.txt", dag.HTTP(nodeURL+"SHASUMS256.txt")).
+		WithWorkdir("/tmp/dl").
+		WithExec([]string{"sh", "-c",
+			`grep " ` + tarball + `$" SHASUMS256.txt | sha256sum -c - && ` +
+				`tar -xzf ` + tarball + ` -C /usr/local --strip-components=1 ` +
+				`--exclude='*/CHANGELOG.md' --exclude='*/README.md' --exclude='*/LICENSE'`,
+		}).
+		WithWorkdir("/").
+		WithExec([]string{"pip", "install", "--no-cache-dir", "--root-user-action=ignore",
+			"mkdocs-techdocs-core==" + techdocsCoreVersion}).
+		WithExec([]string{"npm", "install", "--global", "--no-fund", "--no-audit",
+			"@techdocs/cli@" + techdocsCliVersion}), nil
+}
+
+// nodeArch maps the engine's platform (e.g. linux/arm64) to the architecture
+// name used in the Node.js release assets.
+func nodeArch(ctx context.Context) (string, error) {
+	platform, err := dag.DefaultPlatform(ctx)
+	if err != nil {
+		return "", fmt.Errorf("failed to read engine platform: %w", err)
+	}
+
+	switch arch := strings.Split(strings.TrimPrefix(string(platform), "linux/"), "/")[0]; arch {
+	case "amd64":
+		return "x64", nil
+	case "arm64":
+		return "arm64", nil
+	default:
+		return "", fmt.Errorf("unsupported platform %q: node is downloaded for linux/amd64 or linux/arm64", platform)
+	}
+}

techdocs/dagger.json

@@ -0,0 +1,7 @@
+{
+  "name": "techdocs",
+  "engineVersion": "v0.21.9",
+  "sdk": {
+    "source": "go"
+  }
+}

techdocs/entity.go

@@ -0,0 +1,83 @@
+package main
+
+import (
+	"context"
+	"dagger/techdocs/internal/dagger"
+	"errors"
+	"fmt"
+	"io"
+	"strings"
+
+	"gopkg.in/yaml.v3"
+)
+
+const techdocsRefAnnotation = "backstage.io/techdocs-ref"
+
+type catalogEntity struct {
+	Kind     string `yaml:"kind"`
+	Metadata struct {
+		Name        string            `yaml:"name"`
+		Namespace   string            `yaml:"namespace"`
+		Annotations map[string]string `yaml:"annotations"`
+	} `yaml:"metadata"`
+}
+
+// Entity returns <namespace>/<kind>/<name> of the entity in catalogFile that
+// carries the backstage.io/techdocs-ref annotation -- the path Backstage
+// looks up the docs under. The namespace defaults to "default".
+func (m *Techdocs) Entity(
+	ctx context.Context,
+	// Repository directory containing catalogFile
+	src *dagger.Directory,
+	// Catalog file inside src to read the entity from
+	// +optional
+	// +default="catalog-info.yaml"
+	catalogFile string,
+) (string, error) {
+	content, err := src.File(catalogFile).Contents(ctx)
+	if err != nil {
+		return "", fmt.Errorf("reading %s: %w", catalogFile, err)
+	}
+	return entityRef(content)
+}
+
+// entityRef picks the entity with a techdocs-ref annotation out of a
+// (possibly multi-document) catalog file.
+func entityRef(content string) (string, error) {
+	dec := yaml.NewDecoder(strings.NewReader(content))
+	var withDocs []catalogEntity
+	for {
+		var e catalogEntity
+		err := dec.Decode(&e)
+		if errors.Is(err, io.EOF) {
+			break
+		}
+		if err != nil {
+			return "", fmt.Errorf("parsing catalog file: %w", err)
+		}
+		if _, ok := e.Metadata.Annotations[techdocsRefAnnotation]; ok {
+			withDocs = append(withDocs, e)
+		}
+	}
+
+	switch len(withDocs) {
+	case 0:
+		return "", fmt.Errorf("no entity with a %s annotation; pass --entity", techdocsRefAnnotation)
+	case 1:
+	default:
+		return "", fmt.Errorf("%d entities carry %s; pass --entity to pick one", len(withDocs), techdocsRefAnnotation)
+	}
+
+	e := withDocs[0]
+	if e.Kind == "" || e.Metadata.Name == "" {
+		return "", errors.New("the entity with techdocs-ref has no kind or metadata.name")
+	}
+	ns := e.Metadata.Namespace
+	if ns == "" {
+		ns = "default"
+	}
+	// Backstage looks docs up lower-cased (unless
+	// legacyUseCaseSensitiveTripletPaths); techdocs-cli publish lower-cases
+	// too, so this only keeps the log line honest.
+	return strings.ToLower(fmt.Sprintf("%s/%s/%s", ns, e.Kind, e.Metadata.Name)), nil
+}

techdocs/entity_test.go

@@ -0,0 +1,74 @@
+package main
+
+import "testing"
+
+func TestEntityRef(t *testing.T) {
+	tests := []struct {
+		name    string
+		content string
+		want    string
+		wantErr bool
+	}{
+		{
+			name: "single component, default namespace",
+			content: `apiVersion: backstage.io/v1alpha1
+kind: Component
+metadata:
+  name: flux
+  annotations:
+    backstage.io/techdocs-ref: dir:.
+`,
+			want: "default/component/flux",
+		},
+		{
+			name: "multi-document, only one with docs",
+			content: `apiVersion: backstage.io/v1alpha1
+kind: System
+metadata:
+  name: homerun2
+---
+apiVersion: backstage.io/v1alpha1
+kind: Component
+metadata:
+  name: homerun2-k8s-pitcher
+  namespace: Platform
+  annotations:
+    backstage.io/techdocs-ref: url:https://github.com/stuttgart-things/homerun2-k8s-pitcher/tree/main
+`,
+			want: "platform/component/homerun2-k8s-pitcher",
+		},
+		{
+			name: "no techdocs-ref",
+			content: `kind: Component
+metadata:
+  name: x
+`,
+			wantErr: true,
+		},
+		{
+			name: "two entities with techdocs-ref",
+			content: `kind: Component
+metadata:
+  name: a
+  annotations: {backstage.io/techdocs-ref: dir:.}
+---
+kind: Component
+metadata:
+  name: b
+  annotations: {backstage.io/techdocs-ref: dir:.}
+`,
+			wantErr: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := entityRef(tt.content)
+			if (err != nil) != tt.wantErr {
+				t.Fatalf("err = %v, wantErr %v", err, tt.wantErr)
+			}
+			if got != tt.want {
+				t.Errorf("got %q, want %q", got, tt.want)
+			}
+		})
+	}
+}