diff --git a/README.md b/README.md index b75b924..805e917 100644 --- a/README.md +++ b/README.md @@ -28,6 +28,18 @@ bot-prod/ bot-rooms/ mariadb/ legacy AKS folders, disabled Keep Auth behind the tunnel and ensure Cloudflare does not remove the header. Cluster-internal IdP calls carry no visitor address and share Auth's bounded token budget of 60 requests per minute. +- **Backend exposure:** the backend is in-cluster only except its declared + `/public` prefix, routed on `api.polinetwork.org/public/...` by one + IngressRoute with a path whitelist, body limit and rate limits + (`apps/backend/public-route.yaml`; `tests/` enforces it). New public routes + are added in the backend, never as new Ingresses here. +- **Regulations editor:** `regulations-collab` (backend image, `backend` + namespace) publishes only its websocket on `collab.polinetwork.org`. + `latex-worker` runs untrusted LaTeX with no egress, a read-only root and a + namespace quota. Both wait for the `regulations-secrets` Kustomization, + which is Ready only once the Key Vault entries `regulations-internal-secret` + and `regulations-worker-secret` exist; the worker also waits for its image. + Until then nothing of theirs runs; afterwards they start by themselves. - **Secrets:** `ExternalSecret`s sync from Azure Key Vault (`kv-pn-apps`, `kv-pn-infra`). No secrets live in Git. - **Images:** PoliNetwork apps run `ghcr.io/polinetworkorg/:latest`. The diff --git a/apps/backend/deployment.yaml b/apps/backend/deployment.yaml index 27a8699..349fc2d 100644 --- a/apps/backend/deployment.yaml +++ b/apps/backend/deployment.yaml @@ -99,6 +99,75 @@ spec: secretKeyRef: name: azure-kv key: oauth-private-jwk + # Regulations editor: every variable is optional and the feature + # stays off without them. Empty values (TODO) are pending from + # Lorenzo; the backend reads env with emptyStringAsUndefined. The + # secrets come from apps/regulations-secrets; env from a Secret is + # read at container start, so restart the backend once after they + # first sync. + - name: REGULATIONS_INTERNAL_SECRET + valueFrom: + secretKeyRef: + name: regulations-core + key: internal-secret + optional: true + - name: REGULATIONS_COLLAB_PUBLIC_URL # handed to browsers in collab tickets + value: "wss://collab.polinetwork.org" + - name: REGULATIONS_COLLAB_INTERNAL_URL + value: "http://regulations-collab.backend.svc.cluster.local:4101" + - name: REGULATIONS_WORKER_URL + value: "http://latex-worker.latex-worker.svc.cluster.local:4200" + - name: REGULATIONS_WORKER_SECRET + valueFrom: + secretKeyRef: + name: regulations-core + key: worker-secret + optional: true + - name: REGULATIONS_GITHUB_MODE + value: "app" + # GitHub App webhook: POST https://api.polinetwork.org/public/webhooks/github + # (public-route.yaml), verified with this secret. Polling is only a slow + # reconciliation for missed deliveries. + - name: REGULATIONS_GITHUB_WEBHOOK_SECRET + valueFrom: + secretKeyRef: + name: regulations-github + key: webhook-secret + optional: true + - name: REGULATIONS_GITHUB_APP_ID + value: "" # TODO: GitHub App ID + - name: REGULATIONS_GITHUB_PRIVATE_KEY + valueFrom: + secretKeyRef: + name: regulations-github + key: private-key + optional: true + - name: REGULATIONS_GITHUB_REPO + value: "PoliNetworkOrg/legal" + - name: REGULATIONS_GITHUB_BASE + value: "main" + - name: REGULATIONS_LLM_PROVIDER + value: "" # TODO: "anthropic" (the only provider implemented) once the key exists + - name: REGULATIONS_LLM_MODEL + value: "" # TODO + - name: REGULATIONS_LLM_API_KEY + valueFrom: + secretKeyRef: + name: regulations-llm + key: api-key + optional: true + - name: REGULATIONS_TELEGRAM_CHAT_ID + value: "" # TODO: Soci group chat id + - name: REGULATIONS_TELEGRAM_TOPIC_ID + value: "" # TODO: Soci group topic id + - name: REGULATIONS_EMAIL_SENDER + value: "noreply@polinetwork.org" # TODO: confirm the final sender + - name: REGULATIONS_EMAIL_TO + value: "soci@polinetwork.org" + - name: REGULATIONS_EMAIL_REPLY_TO + value: "direttivo@polinetwork.org" + - name: REGULATIONS_DIRETTIVO_EMAIL + value: "direttivo@polinetwork.org" startupProbe: tcpSocket: port: 3000 diff --git a/apps/backend/kustomization.yaml b/apps/backend/kustomization.yaml index 726e901..9d146fd 100644 --- a/apps/backend/kustomization.yaml +++ b/apps/backend/kustomization.yaml @@ -7,4 +7,5 @@ resources: - namespace.yaml - external-secret.yaml - deployment.yaml + - public-route.yaml - network-policy.yaml diff --git a/apps/backend/network-policy.yaml b/apps/backend/network-policy.yaml index 72c9570..6d7739a 100644 --- a/apps/backend/network-policy.yaml +++ b/apps/backend/network-policy.yaml @@ -2,7 +2,10 @@ # dashboard and the website (tRPC, esign agent proxy), the Telegram bot (tRPC # and its socket), the IdP (access-changed events on /internal/events) and # Kuvasz (uptime monitoring). -# Nothing reaches it from outside the cluster since the Ingress was removed. +# From outside the cluster, Traefik reaches it only through the backend-public +# IngressRoute (public-route.yaml), which matches the declared /public prefix +# and nothing else; a NetworkPolicy cannot filter paths, so that route is the +# guard. apiVersion: networking.k8s.io/v1 kind: NetworkPolicy metadata: @@ -45,6 +48,12 @@ spec: podSelector: matchLabels: app: kuvasz + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: kube-system + podSelector: + matchLabels: + app.kubernetes.io/name: traefik ports: - port: 3000 protocol: TCP diff --git a/apps/backend/public-route.yaml b/apps/backend/public-route.yaml new file mode 100644 index 0000000..8144eb8 --- /dev/null +++ b/apps/backend/public-route.yaml @@ -0,0 +1,80 @@ +# The backend's only public surface: its declared /public prefix (regulations +# spec, "Public backend routes"), on api.polinetwork.org through the tunnel. +# Every other path, on any host, stays in-cluster only (#28): this is the only +# router that targets backend-service, and the test suite enforces it. +# +# The match is a whitelist: after Traefik's own path sanitizing, the path must +# be /public followed by plain segments of letters, digits, "-" and "_". Dots, +# "%" (encoded characters), empty segments and a trailing slash never match, +# so /public/../api or /public/%2e%2e/api cannot reach another backend route. +# The backend checks again, authenticates every route itself (GitHub HMAC) and +# keeps its own body and rate limits. Bun resolves ".." and "%2e%2e" before the +# app sees the path, so the backend can't spot an escape from the path alone: +# public-ingress-marker stamps every request from this route, overwriting any +# client value, and the backend lets a stamped request reach only its declared +# /public routes. +apiVersion: traefik.io/v1alpha1 +kind: IngressRoute +metadata: + name: backend-public +spec: + entryPoints: [web] + routes: + - kind: Rule + match: Host(`api.polinetwork.org`) && PathPrefix(`/public/`) && PathRegexp(`^/public(/[A-Za-z0-9_-]+)+$`) + middlewares: + - name: public-rate-limit-total + - name: public-rate-limit-client + - name: public-body-limit + - name: public-ingress-marker + services: + - name: backend-service + port: 3000 +--- +# GitHub webhook payloads are a few tens of KiB; refuse anything over 1 MiB +# with 413 before it reaches the backend. +apiVersion: traefik.io/v1alpha1 +kind: Middleware +metadata: + name: public-body-limit +spec: + buffering: + maxRequestBodyBytes: 1048576 + memRequestBodyBytes: 1048576 +--- +# Per visitor: Cloudflare sets Cf-Connecting-Ip at the edge (see README, +# Traffic); the TCP peer is always a cloudflared Pod. +apiVersion: traefik.io/v1alpha1 +kind: Middleware +metadata: + name: public-rate-limit-client +spec: + rateLimit: + average: 1 + period: 1s + burst: 20 + sourceCriterion: + requestHeaderName: Cf-Connecting-Ip +--- +# Whole prefix: every request shares the api.polinetwork.org bucket. +apiVersion: traefik.io/v1alpha1 +kind: Middleware +metadata: + name: public-rate-limit-total +spec: + rateLimit: + average: 5 + period: 1s + burst: 50 + sourceCriterion: + requestHost: true +--- +# Set, never appended: Traefik replaces whatever the client sent. +apiVersion: traefik.io/v1alpha1 +kind: Middleware +metadata: + name: public-ingress-marker +spec: + headers: + customRequestHeaders: + X-PoliNetwork-Ingress: "public" diff --git a/apps/latex-worker/deployment.yaml b/apps/latex-worker/deployment.yaml new file mode 100644 index 0000000..bd6b175 --- /dev/null +++ b/apps/latex-worker/deployment.yaml @@ -0,0 +1,133 @@ +apiVersion: v1 +kind: Service +metadata: + name: latex-worker +spec: + selector: + app: latex-worker + ports: + - name: http + port: 4200 + targetPort: 4200 +--- +# Compiles regulation drafts with xelatex for the backend and collab. It runs untrusted +# LaTeX, so it has no network egress, no service account token, a read-only +# root and a size-limited tmpfs workdir. The worker runs each xelatex in a clean +# environment it builds itself: no shell escape, openin_any/openout_any paranoid, +# caches under WORKER_TMP_DIR, and an address-space prlimit. One pod with a small internal pool: +# warm per-draft workdirs (.aux) live in its memory, and the node has 2 vCPUs. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: latex-worker +spec: + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: + app: latex-worker + template: + metadata: + labels: + app: latex-worker + spec: + automountServiceAccountToken: false + enableServiceLinks: false + securityContext: + runAsNonRoot: true + # A uid no other workload uses (not the image's 1000): the worker caps + # each xelatex with prlimit --nproc, and RLIMIT_NPROC counts every task + # of the uid on the node, so sharing 1000 with Bun and Node apps made + # xelatex unable to fork xdvipdfmx (tested: compiles fail at ~90 tasks). + runAsUser: 10200 + runAsGroup: 10200 + fsGroup: 10200 + seccompProfile: + type: RuntimeDefault + containers: + - name: latex-worker + image: ghcr.io/polinetworkorg/latex-worker:latest + imagePullPolicy: IfNotPresent + ports: + - containerPort: 4200 + env: + - name: NODE_ENV + value: "production" + - name: TZ + value: "Europe/Rome" + - name: WORKER_PORT + value: "4200" + - name: WORKER_SECRET + valueFrom: + secretKeyRef: + name: regulations + key: worker-secret + - name: WORKER_WORK_DIR + value: "/work" + - name: WORKER_TMP_DIR + value: "/tmp" + # Parallel xelatex runs; a third only contends for the same 1.5 CPUs. + - name: WORKER_CONCURRENCY + value: "2" + - name: WORKER_TIMEOUT_MS + value: "45000" + # Per-xelatex prlimit; real usage is ~200Mi per run, see resources. + - name: WORKER_MEMORY_LIMIT_MB + value: "1024" + # Warm workdirs kept on /work (~8Mi each with the RRI). + - name: WORKER_MAX_WORKDIRS + value: "40" + startupProbe: + httpGet: + path: /health + port: 4200 + periodSeconds: 2 + failureThreshold: 30 + readinessProbe: + httpGet: + path: /health + port: 4200 + periodSeconds: 5 + timeoutSeconds: 2 + failureThreshold: 3 + livenessProbe: + httpGet: + path: /health + port: 4200 + periodSeconds: 10 + timeoutSeconds: 2 + failureThreshold: 3 + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: [ALL] + # Measured on the RRI (19 pages) in this sandbox: a cold latexmk takes + # 1.8 s, a warm pass 0.4 s, 3 parallel passes peak at 650Mi including + # tmpfs. Memory-backed volumes count against the memory limit, so the + # limit covers a full 512Mi /work too. CPU stays below the node's + # 2 vCPUs so a compile burst cannot starve the other workloads. + resources: + requests: + cpu: "250m" + memory: "768Mi" + ephemeral-storage: "64Mi" + limits: + cpu: "1500m" + memory: "1536Mi" + ephemeral-storage: "256Mi" + volumeMounts: + - name: work + mountPath: /work + - name: tmp + mountPath: /tmp + volumes: + - name: work + emptyDir: + medium: Memory + sizeLimit: 512Mi + - name: tmp + emptyDir: + medium: Memory + sizeLimit: 64Mi diff --git a/apps/latex-worker/kustomization.yaml b/apps/latex-worker/kustomization.yaml new file mode 100644 index 0000000..04d23cf --- /dev/null +++ b/apps/latex-worker/kustomization.yaml @@ -0,0 +1,10 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +# The namespace and the worker secret belong to apps/regulations-secrets. +namespace: latex-worker + +resources: + - limits.yaml + - deployment.yaml + - network-policy.yaml diff --git a/apps/latex-worker/limits.yaml b/apps/latex-worker/limits.yaml new file mode 100644 index 0000000..b392017 --- /dev/null +++ b/apps/latex-worker/limits.yaml @@ -0,0 +1,38 @@ +# Caps the namespace so a runaway compile cannot starve the single node: no +# container above the worker's own limits, and no more than one worker's worth +# of resources in total (the Deployment uses Recreate, never two pods at once). +apiVersion: v1 +kind: LimitRange +metadata: + name: latex-worker +spec: + limits: + - type: Container + max: + cpu: "1500m" + memory: "1536Mi" + ephemeral-storage: "256Mi" + default: + cpu: "500m" + memory: "512Mi" + ephemeral-storage: "128Mi" + defaultRequest: + cpu: "50m" + memory: "128Mi" + ephemeral-storage: "64Mi" +--- +apiVersion: v1 +kind: ResourceQuota +metadata: + name: latex-worker +spec: + hard: + pods: "2" + requests.cpu: "500m" + requests.memory: "768Mi" + requests.ephemeral-storage: "128Mi" + limits.cpu: "1500m" + limits.memory: "1536Mi" + limits.ephemeral-storage: "256Mi" + services.loadbalancers: "0" + services.nodeports: "0" diff --git a/apps/latex-worker/network-policy.yaml b/apps/latex-worker/network-policy.yaml new file mode 100644 index 0000000..dd5f05e --- /dev/null +++ b/apps/latex-worker/network-policy.yaml @@ -0,0 +1,28 @@ +# Only the backend and regulations-collab may call the worker, and the worker may call nothing: not +# even DNS. Compiles get every input in the request. +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: latex-worker + namespace: latex-worker +spec: + podSelector: {} + policyTypes: [Ingress, Egress] + ingress: + - from: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: backend + podSelector: + matchLabels: + app: backend + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: backend + podSelector: + matchLabels: + app: regulations-collab + ports: + - port: 4200 + protocol: TCP + egress: [] diff --git a/apps/regulations-collab/deployment.yaml b/apps/regulations-collab/deployment.yaml new file mode 100644 index 0000000..a00360b --- /dev/null +++ b/apps/regulations-collab/deployment.yaml @@ -0,0 +1,139 @@ +apiVersion: v1 +kind: Service +metadata: + name: regulations-collab +spec: + selector: + app: regulations-collab + ports: + - name: websocket + port: 4100 + targetPort: 4100 + - name: internal + port: 4101 + targetPort: 4101 +--- +# Hocuspocus server for the regulations editor: the backend image with another +# entrypoint, so it always runs the backend's digest (same ResourceSet input). +# Its Flux Kustomization waits for regulations-secrets, so this Pod is created +# only once REGULATIONS_INTERNAL_SECRET and REGULATIONS_WORKER_SECRET exist. +# Port 4100 is the editors' websocket, published on collab.polinetwork.org; it +# accepts only single-use tickets from regulations.collabTicket. Port 4101 is +# the internal API, for the backend only. Exactly one replica, replaced with +# Recreate: Hocuspocus keeps open documents in memory, so two pods would split +# the editors of a draft between two copies. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: regulations-collab +spec: + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: + app: regulations-collab + template: + metadata: + labels: + app: regulations-collab + spec: + automountServiceAccountToken: false + enableServiceLinks: false + securityContext: + runAsNonRoot: true + # The image's "bun" user. + runAsUser: 1000 + runAsGroup: 1000 + seccompProfile: + type: RuntimeDefault + containers: + - name: regulations-collab + image: ghcr.io/polinetworkorg/backend:latest + imagePullPolicy: IfNotPresent + command: ["bun", "run", "collab.js"] + ports: + - containerPort: 4100 + - containerPort: 4101 + env: + - name: NODE_ENV + value: "production" + - name: TZ + value: "Europe/Rome" + - name: TMPDIR + value: "/tmp" + - name: COLLAB_PORT + value: "4100" + - name: COLLAB_INTERNAL_PORT + value: "4101" + # Match the worker's WORKER_CONCURRENCY: more would only queue there. + - name: REGULATIONS_COMPILE_CONCURRENCY + value: "2" + - name: COLLAB_SNAPSHOT_MINUTES + value: "10" + - name: DB_NAME + value: "polinetwork_backend" + - name: DB_HOST + value: "postgres-service.postgres.svc.cluster.local" + - name: DB_PORT + value: "5432" + - name: DB_USER + valueFrom: + secretKeyRef: + name: azure-kv + key: db-user + - name: DB_PASS + valueFrom: + secretKeyRef: + name: azure-kv + key: db-pass + # Required: collab refuses to start without it. + - name: REGULATIONS_INTERNAL_SECRET + valueFrom: + secretKeyRef: + name: regulations-core + key: internal-secret + - name: REGULATIONS_WORKER_URL + value: "http://latex-worker.latex-worker.svc.cluster.local:4200" + - name: REGULATIONS_WORKER_SECRET + valueFrom: + secretKeyRef: + name: regulations-core + key: worker-secret + optional: true + startupProbe: + tcpSocket: + port: 4100 + periodSeconds: 2 + failureThreshold: 30 + readinessProbe: + tcpSocket: + port: 4100 + periodSeconds: 5 + timeoutSeconds: 2 + failureThreshold: 3 + livenessProbe: + tcpSocket: + port: 4100 + periodSeconds: 10 + timeoutSeconds: 2 + failureThreshold: 3 + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: [ALL] + resources: + requests: + cpu: "20m" + memory: "128Mi" + limits: + memory: "512Mi" + volumeMounts: + - name: tmp + mountPath: /tmp + volumes: + - name: tmp + emptyDir: + medium: Memory + sizeLimit: 32Mi diff --git a/apps/regulations-collab/ingress.yaml b/apps/regulations-collab/ingress.yaml new file mode 100644 index 0000000..7f0efa7 --- /dev/null +++ b/apps/regulations-collab/ingress.yaml @@ -0,0 +1,24 @@ +# The regulations editor's websocket, the only public path into this +# namespace. Cloudflare Tunnel sends every HTTP hostname to Traefik, which +# routes by host and passes the Upgrade through. Only the websocket port is +# published; the backend API and collab's internal port stay in-cluster. No +# Cloudflare Access: the single-use ticket authenticates each connection. +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: regulations-collab + annotations: + traefik.ingress.kubernetes.io/router.entrypoints: web +spec: + ingressClassName: traefik + rules: + - host: collab.polinetwork.org + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: regulations-collab + port: + number: 4100 diff --git a/apps/regulations-collab/kustomization.yaml b/apps/regulations-collab/kustomization.yaml new file mode 100644 index 0000000..3c00a89 --- /dev/null +++ b/apps/regulations-collab/kustomization.yaml @@ -0,0 +1,10 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +# The backend namespace itself belongs to apps/backend. +namespace: backend + +resources: + - deployment.yaml + - ingress.yaml + - network-policy.yaml diff --git a/apps/regulations-collab/network-policy.yaml b/apps/regulations-collab/network-policy.yaml new file mode 100644 index 0000000..74fa97c --- /dev/null +++ b/apps/regulations-collab/network-policy.yaml @@ -0,0 +1,64 @@ +# regulations-collab faces the internet: Traefik reaches the websocket port +# only, the backend the internal API, and collab itself may reach only DNS, +# PostgreSQL and the LaTeX worker. +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: regulations-collab + namespace: backend +spec: + podSelector: + matchLabels: + app: regulations-collab + policyTypes: [Ingress, Egress] + ingress: + - from: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: kube-system + podSelector: + matchLabels: + app.kubernetes.io/name: traefik + ports: + - port: 4100 + protocol: TCP + - from: + - podSelector: + matchLabels: + app: backend + ports: + - port: 4101 + protocol: TCP + egress: + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: kube-system + podSelector: + matchLabels: + k8s-app: kube-dns + ports: + - port: 53 + protocol: UDP + - port: 53 + protocol: TCP + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: postgres + podSelector: + matchLabels: + app: postgres + ports: + - port: 5432 + protocol: TCP + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: latex-worker + podSelector: + matchLabels: + app: latex-worker + ports: + - port: 4200 + protocol: TCP diff --git a/apps/regulations-secrets/external-secrets.yaml b/apps/regulations-secrets/external-secrets.yaml new file mode 100644 index 0000000..1bde27b --- /dev/null +++ b/apps/regulations-secrets/external-secrets.yaml @@ -0,0 +1,91 @@ +# Regulations editor secrets, from kv-pn-apps. The Flux Kustomization +# (clusters/k3s/apps/regulations-secrets.yaml) is Ready only once the two core +# entries exist; regulations-collab and latex-worker wait for it, so nothing +# that needs them runs, fails or alerts before Lorenzo creates them. The GitHub +# and LLM entries are optional and gate nothing: the backend reads them with +# optional refs. +# Backend <-> regulations-collab, and the bearer token both send to latex-worker. +apiVersion: external-secrets.io/v1 +kind: ExternalSecret +metadata: + name: regulations-core + namespace: backend +spec: + refreshInterval: 1h + secretStoreRef: + kind: ClusterSecretStore + name: kv-pn-apps + target: + name: regulations-core + creationPolicy: Owner + deletionPolicy: Retain + data: + - secretKey: internal-secret + remoteRef: + key: regulations-internal-secret + - secretKey: worker-secret + remoteRef: + key: regulations-worker-secret +--- +# The same worker token, for the worker to check. +apiVersion: external-secrets.io/v1 +kind: ExternalSecret +metadata: + name: regulations + namespace: latex-worker +spec: + refreshInterval: 1h + secretStoreRef: + kind: ClusterSecretStore + name: kv-pn-apps + target: + name: regulations + creationPolicy: Owner + deletionPolicy: Retain + data: + - secretKey: worker-secret + remoteRef: + key: regulations-worker-secret +--- +# The GitHub App on PoliNetworkOrg/legal: its PEM key and webhook secret. +apiVersion: external-secrets.io/v1 +kind: ExternalSecret +metadata: + name: regulations-github + namespace: backend +spec: + refreshInterval: 1h + secretStoreRef: + kind: ClusterSecretStore + name: kv-pn-apps + target: + name: regulations-github + creationPolicy: Owner + deletionPolicy: Retain + data: + - secretKey: private-key + remoteRef: + key: regulations-github-app-private-key + - secretKey: webhook-secret + remoteRef: + key: regulations-github-webhook-secret +--- +# The LLM provider key for the differences summary. +apiVersion: external-secrets.io/v1 +kind: ExternalSecret +metadata: + name: regulations-llm + namespace: backend +spec: + refreshInterval: 1h + secretStoreRef: + kind: ClusterSecretStore + name: kv-pn-apps + target: + name: regulations-llm + creationPolicy: Owner + deletionPolicy: Retain + data: + - secretKey: api-key + remoteRef: + key: regulations-llm-api-key diff --git a/apps/regulations-secrets/kustomization.yaml b/apps/regulations-secrets/kustomization.yaml new file mode 100644 index 0000000..185387e --- /dev/null +++ b/apps/regulations-secrets/kustomization.yaml @@ -0,0 +1,7 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +# Explicit namespaces: backend (owned by apps/backend) and latex-worker. +resources: + - namespace.yaml + - external-secrets.yaml diff --git a/apps/regulations-secrets/namespace.yaml b/apps/regulations-secrets/namespace.yaml new file mode 100644 index 0000000..2eb31e2 --- /dev/null +++ b/apps/regulations-secrets/namespace.yaml @@ -0,0 +1,11 @@ +# latex-worker's namespace lives here so its secret can sync before the worker +# image exists. +apiVersion: v1 +kind: Namespace +metadata: + name: latex-worker + annotations: + kustomize.toolkit.fluxcd.io/prune: disabled + labels: + pod-security.kubernetes.io/enforce: restricted + pod-security.kubernetes.io/enforce-version: latest diff --git a/clusters/k3s/apps/backend.yaml b/clusters/k3s/apps/backend.yaml index b0871b5..0001b3c 100644 --- a/clusters/k3s/apps/backend.yaml +++ b/clusters/k3s/apps/backend.yaml @@ -1,5 +1,6 @@ -# Generates the backend Kustomization with the current digest of -# ghcr.io/polinetworkorg/backend:latest (provider backend-image). +# Generates the backend and regulations-collab Kustomizations with the current +# digest of ghcr.io/polinetworkorg/backend:latest (provider backend-image), so +# collab always runs the backend's image. apiVersion: fluxcd.controlplane.io/v1 kind: ResourceSet metadata: @@ -40,3 +41,30 @@ spec: - name: ghcr.io/polinetworkorg/backend newTag: << inputs.backend_image.tag | quote >> digest: << inputs.backend_image.digest | quote >> + # Hocuspocus for the regulations editor, from the same image. It waits for + # regulations-secrets (its required secrets) and for the backend (which runs + # the database migrations); before then nothing of it is applied. + - apiVersion: kustomize.toolkit.fluxcd.io/v1 + kind: Kustomization + metadata: + name: regulations-collab + namespace: flux-system + annotations: + fluxcd.controlplane.io/prune: disabled + spec: + dependsOn: + - name: regulations-secrets + - name: backend + interval: 30m + retryInterval: 2m + timeout: 5m + prune: true + wait: true + sourceRef: + kind: GitRepository + name: flux-system + path: ./apps/regulations-collab + images: + - name: ghcr.io/polinetworkorg/backend + newTag: << inputs.backend_image.tag | quote >> + digest: << inputs.backend_image.digest | quote >> diff --git a/clusters/k3s/apps/kustomization.yaml b/clusters/k3s/apps/kustomization.yaml index 1faf6c2..9fbc6a7 100644 --- a/clusters/k3s/apps/kustomization.yaml +++ b/clusters/k3s/apps/kustomization.yaml @@ -10,10 +10,12 @@ resources: - bot-ts.yaml - influxdb.yaml - kuvasz.yaml + - latex-worker.yaml - live-audience.yaml - monitoring.yaml - polinetcc.yaml - postgres.yaml - redis.yaml + - regulations-secrets.yaml - vaultwarden.yaml - web.yaml diff --git a/clusters/k3s/apps/latex-worker.yaml b/clusters/k3s/apps/latex-worker.yaml new file mode 100644 index 0000000..51b4082 --- /dev/null +++ b/clusters/k3s/apps/latex-worker.yaml @@ -0,0 +1,58 @@ +# Unlike the other image providers (infrastructure/image-automation), this one +# lives here, under the root sync, which does not wait for health: until the +# image exists and its package is public, the provider is the only thing that +# reports an error, and infrastructure-image-automation stays Ready. +apiVersion: fluxcd.controlplane.io/v1 +kind: ResourceSetInputProvider +metadata: + name: latex-worker-image + namespace: flux-system + annotations: + fluxcd.controlplane.io/reconcileEvery: "6h" +spec: + type: OCIArtifactTag + url: oci://ghcr.io/polinetworkorg/latex-worker + filter: + includeTag: '^latest$' + limit: 1 +--- +# Generates the latex-worker Kustomization with the current digest of +# ghcr.io/polinetworkorg/latex-worker:latest (provider latex-worker-image). +# Nothing is deployed until that image is pullable (the backend repository +# publishes it, and the package is public) and regulations-secrets is Ready. +apiVersion: fluxcd.controlplane.io/v1 +kind: ResourceSet +metadata: + name: latex-worker + namespace: flux-system +spec: + inputStrategy: + name: Permute + inputsFrom: + - kind: ResourceSetInputProvider + name: latex-worker-image + resources: + - apiVersion: kustomize.toolkit.fluxcd.io/v1 + kind: Kustomization + metadata: + name: latex-worker + namespace: flux-system + annotations: + # Never garbage-collect the app if the provider exports no inputs. + fluxcd.controlplane.io/prune: disabled + spec: + dependsOn: + - name: regulations-secrets + interval: 30m + retryInterval: 2m + timeout: 5m + prune: true + wait: true + sourceRef: + kind: GitRepository + name: flux-system + path: ./apps/latex-worker + images: + - name: ghcr.io/polinetworkorg/latex-worker + newTag: << inputs.latex_worker_image.tag | quote >> + digest: << inputs.latex_worker_image.digest | quote >> diff --git a/clusters/k3s/apps/regulations-secrets.yaml b/clusters/k3s/apps/regulations-secrets.yaml new file mode 100644 index 0000000..352c4d5 --- /dev/null +++ b/clusters/k3s/apps/regulations-secrets.yaml @@ -0,0 +1,34 @@ +# The regulations editor's gate. It applies every regulations ExternalSecret but +# is Ready only once the core ones sync (the Key Vault entries +# regulations-internal-secret and regulations-worker-secret exist). +# regulations-collab and latex-worker depend on it, so before those entries +# exist no Pod of theirs is created: nothing fails, restarts or alerts. Once +# they exist, both start by themselves within a few minutes. +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: regulations-secrets + namespace: flux-system +spec: + dependsOn: + - name: infrastructure-secret-stores + interval: 30m + retryInterval: 2m + timeout: 2m + prune: true + # Only the core secrets decide readiness; the GitHub App and LLM entries + # are optional and may arrive much later. + wait: false + healthChecks: + - apiVersion: external-secrets.io/v1 + kind: ExternalSecret + name: regulations-core + namespace: backend + - apiVersion: external-secrets.io/v1 + kind: ExternalSecret + name: regulations + namespace: latex-worker + sourceRef: + kind: GitRepository + name: flux-system + path: ./apps/regulations-secrets diff --git a/infrastructure/image-automation/receiver.yaml b/infrastructure/image-automation/receiver.yaml index e1319ec..ce73f27 100644 --- a/infrastructure/image-automation/receiver.yaml +++ b/infrastructure/image-automation/receiver.yaml @@ -28,6 +28,9 @@ spec: - apiVersion: fluxcd.controlplane.io/v1 kind: ResourceSetInputProvider name: bot-ts-image + - apiVersion: fluxcd.controlplane.io/v1 + kind: ResourceSetInputProvider + name: latex-worker-image - apiVersion: fluxcd.controlplane.io/v1 kind: ResourceSetInputProvider name: live-audience-image diff --git a/infrastructure/secret-stores/cluster-secret-stores.yaml b/infrastructure/secret-stores/cluster-secret-stores.yaml index c872981..2561403 100644 --- a/infrastructure/secret-stores/cluster-secret-stores.yaml +++ b/infrastructure/secret-stores/cluster-secret-stores.yaml @@ -15,6 +15,7 @@ spec: - monitoring - influxdb - kuvasz + - latex-worker - live-audience - polinetcc - postgres diff --git a/tests/workload-manifests.test.rb b/tests/workload-manifests.test.rb index d02e95d..41e020f 100644 --- a/tests/workload-manifests.test.rb +++ b/tests/workload-manifests.test.rb @@ -86,6 +86,152 @@ def init_container(resource, name) raise "Live Audience must never enable the fake dev login" if live_audience_env.key?("DEV_LOGIN") raise "Live Audience must read its OIDC client secret from a secret" unless live_audience_env.dig("OIDC_CLIENT_SECRET", "valueFrom", "secretKeyRef") +def network_policy(path, name) + documents(path).find { |doc| doc["kind"] == "NetworkPolicy" && doc.dig("metadata", "name") == name } || + raise("missing NetworkPolicy #{name}") +end + +def hardened!(name, resource, app) + pod = resource.dig("spec", "template", "spec") + raise "#{name} must run as a numeric non-root user" unless pod.dig("securityContext", "runAsNonRoot") && pod.dig("securityContext", "runAsUser").to_i.positive? + raise "#{name} must use the RuntimeDefault seccomp profile" unless pod.dig("securityContext", "seccompProfile", "type") == "RuntimeDefault" + raise "#{name} must not mount a service account token" unless pod["automountServiceAccountToken"] == false + raise "#{name} must have a read-only root filesystem" unless app.dig("securityContext", "readOnlyRootFilesystem") + raise "#{name} must not allow privilege escalation" unless app.dig("securityContext", "allowPrivilegeEscalation") == false + raise "#{name} must drop all capabilities" unless app.dig("securityContext", "capabilities", "drop") == ["ALL"] +end + +def from_pod?(peer, namespace, app) + peer.dig("namespaceSelector", "matchLabels") == { "kubernetes.io/metadata.name" => namespace } && + peer.dig("podSelector", "matchLabels") == { "app" => app } +end + +TRAEFIK = { + "namespaceSelector" => { "matchLabels" => { "kubernetes.io/metadata.name" => "kube-system" } }, + "podSelector" => { "matchLabels" => { "app.kubernetes.io/name" => "traefik" } }, +}.freeze + +def peers(policy, port) + policy.dig("spec", "ingress").select { |rule| rule["ports"].any? { |item| item["port"] == port } }.flat_map { |rule| rule["from"] } +end + +collab_deployment = deployment("apps/regulations-collab/deployment.yaml", "regulations-collab") +collab = container(collab_deployment, "regulations-collab") +collab_policy = network_policy("apps/regulations-collab/network-policy.yaml", "regulations-collab") +raise "Collab must run the backend image, so it ships with the backend's digest" unless collab["image"] == container(deployment("apps/backend/deployment.yaml", "backend"), "backend")["image"] +raise "Collab must run exactly one replica" unless collab_deployment.dig("spec", "replicas") == 1 +raise "Collab must use Recreate so one process owns each open document" unless collab_deployment.dig("spec", "strategy", "type") == "Recreate" +%w[startupProbe readinessProbe livenessProbe].each do |probe| + raise "Collab must have a TCP #{probe}" unless collab.dig(probe, "tcpSocket", "port") == 4100 +end +raise "Collab must declare resource requests and limits" unless collab.dig("resources", "requests") && collab.dig("resources", "limits") +hardened!("Collab", collab_deployment, collab) +raise "Collab must restrict both ingress and egress" unless collab_policy.dig("spec", "policyTypes").sort == %w[Egress Ingress] +raise "Only Traefik may reach the collab websocket" unless peers(collab_policy, 4100) == [TRAEFIK] +raise "Only the backend may reach the collab internal API" unless peers(collab_policy, 4101) == [{ "podSelector" => { "matchLabels" => { "app" => "backend" } } }] +raise "Collab exposes only ports 4100 and 4101" unless collab_policy.dig("spec", "ingress").flat_map { |rule| rule["ports"].map { |item| item["port"] } }.sort == [4100, 4101] + +worker_deployment = deployment("apps/latex-worker/deployment.yaml", "latex-worker") +worker = container(worker_deployment, "latex-worker") +worker_env = worker["env"].to_h { |item| [item["name"], item["value"]] } +worker_policy = network_policy("apps/latex-worker/network-policy.yaml", "latex-worker") +worker_limits = documents("apps/latex-worker/limits.yaml") +hardened!("LaTeX worker", worker_deployment, worker) +raise "LaTeX worker must declare CPU and memory requests and limits" unless %w[requests limits].all? { |kind| worker.dig("resources", kind, "cpu") && worker.dig("resources", kind, "memory") } +work_mount = worker.fetch("volumeMounts", []).find { |mount| mount["mountPath"] == "/work" } || raise("LaTeX worker must mount a /work volume") +work_volume = worker_deployment.dig("spec", "template", "spec", "volumes").find { |volume| volume["name"] == work_mount["name"] } +raise "LaTeX worker /work must be a size-limited tmpfs" unless work_volume.dig("emptyDir", "medium") == "Memory" && work_volume.dig("emptyDir", "sizeLimit") +%w[WORKER_WORK_DIR WORKER_TMP_DIR].each do |name| + mount = worker.fetch("volumeMounts", []).find { |item| item["mountPath"] == worker_env[name] } || raise("LaTeX worker #{name} must be a mounted volume, not the read-only root") + volume = worker_deployment.dig("spec", "template", "spec", "volumes").find { |item| item["name"] == mount["name"] } + raise "LaTeX worker #{name} must be a size-limited tmpfs" unless volume.dig("emptyDir", "medium") == "Memory" && volume.dig("emptyDir", "sizeLimit") +end +collab_env = collab["env"].to_h { |item| [item["name"], item["value"]] } +raise "Collab must not send more parallel compiles than the worker runs" unless collab_env["REGULATIONS_COMPILE_CONCURRENCY"].to_i.between?(1, worker_env["WORKER_CONCURRENCY"].to_i) +# RLIMIT_NPROC (the worker's prlimit --nproc per xelatex) counts every task of +# the uid on the node, so the worker's uid must be its own. +worker_uid = worker_deployment.dig("spec", "template", "spec", "securityContext", "runAsUser") +other_uids = Dir.glob(File.join(ROOT, "{apps,infrastructure}/**/*.yaml")).reject { |path| path.include?("apps/latex-worker/") }.flat_map do |path| + File.read(path).scan(/runAsUser:\s*(\d+)/).flatten.map(&:to_i) +end +raise "LaTeX worker must run as a uid no other workload or common image user (1000) uses" if worker_uid.to_i == 1000 || other_uids.include?(worker_uid.to_i) +raise "LaTeX worker policy must cover every Pod in its namespace" unless worker_policy.dig("spec", "podSelector") == {} +raise "LaTeX worker must restrict both ingress and egress" unless worker_policy.dig("spec", "policyTypes").sort == %w[Egress Ingress] +raise "LaTeX worker must have no egress at all" unless worker_policy.dig("spec", "egress") == [] +raise "LaTeX worker may only be reached by the backend and collab" unless worker_policy.dig("spec", "ingress").flat_map { |rule| rule["from"] }.all? { |peer| from_pod?(peer, "backend", "backend") || from_pod?(peer, "backend", "regulations-collab") } +raise "LaTeX worker namespace needs a LimitRange" unless worker_limits.any? { |doc| doc["kind"] == "LimitRange" } +quota = worker_limits.find { |doc| doc["kind"] == "ResourceQuota" } || raise("LaTeX worker namespace needs a ResourceQuota") +raise "LaTeX worker quota must cap CPU and memory limits" unless quota.dig("spec", "hard", "limits.cpu") && quota.dig("spec", "hard", "limits.memory") + +# The backend stays in-cluster only (#28) except its declared /public prefix +# (regulations spec, "Public backend routes"). A NetworkPolicy cannot filter +# paths, so the guard is routing: in the backend namespace (Ingresses and +# IngressRoutes only reach Services in their own namespace) exactly one router +# targets backend-service, with exactly this rule and these middlewares. +PUBLIC_RULE = "Host(`api.polinetwork.org`) && PathPrefix(`/public/`) && PathRegexp(`^/public(/[A-Za-z0-9_-]+)+$`)".freeze +backend_dirs = Dir.glob(File.join(ROOT, "apps/*/kustomization.yaml")).select { |path| YAML.safe_load(File.read(path))["namespace"] == "backend" }.map { |path| File.dirname(path) } +raise "expected apps/backend and apps/regulations-collab in the backend namespace" unless backend_dirs.map { |dir| File.basename(dir) }.sort == %w[backend regulations-collab] +backend_docs = backend_dirs.flat_map { |dir| Dir.glob(File.join(dir, "*.yaml")) }.reject { |path| path.end_with?("kustomization.yaml") }.flat_map { |path| YAML.load_stream(File.read(path)).compact } +backend_docs += documents("apps/regulations-secrets/external-secrets.yaml").select { |doc| doc.dig("metadata", "namespace") == "backend" } +backend_docs.select { |doc| doc["kind"] == "Ingress" }.flat_map { |ingress| ingress.dig("spec", "rules") }.flat_map { |rule| rule.dig("http", "paths") }.each do |path| + raise "Ingresses in the backend namespace may only publish the collab websocket" unless path.dig("backend", "service") == { "name" => "regulations-collab", "port" => { "number" => 4100 } } +end +raise "Only Traefik CRDs this test understands may route in the backend namespace" if backend_docs.any? { |doc| %w[IngressRouteTCP IngressRouteUDP TraefikService].include?(doc["kind"]) } +backend_routes = backend_docs.select { |doc| doc["kind"] == "IngressRoute" }.flat_map { |route| route.dig("spec", "routes") } +raise "Exactly one route may reach the backend, its /public prefix" unless backend_routes.size == 1 +public_route = backend_routes.first +raise "The backend public route must match only #{PUBLIC_RULE}" unless public_route["match"] == PUBLIC_RULE +raise "The backend public route must target backend-service:3000" unless public_route["services"] == [{ "name" => "backend-service", "port" => 3000 }] +middlewares = backend_docs.select { |doc| doc["kind"] == "Middleware" }.to_h { |doc| [doc.dig("metadata", "name"), doc["spec"]] } +route_middlewares = public_route.fetch("middlewares", []).map { |item| middlewares.fetch(item["name"]) { raise "missing Middleware #{item["name"]}" } } +raise "The backend public route needs a body size limit" unless route_middlewares.any? { |spec| spec.dig("buffering", "maxRequestBodyBytes").to_i.between?(1, 10 * 1024 * 1024) } +raise "The backend public route needs a per-visitor rate limit" unless route_middlewares.any? { |spec| spec.dig("rateLimit", "sourceCriterion", "requestHeaderName") == "Cf-Connecting-Ip" } +raise "The backend public route needs a total rate limit" unless route_middlewares.any? { |spec| spec.dig("rateLimit", "sourceCriterion", "requestHost") } +raise "The backend public route must stamp X-PoliNetwork-Ingress: public" unless route_middlewares.any? { |spec| spec.dig("headers", "customRequestHeaders") == { "X-PoliNetwork-Ingress" => "public" } } + +# The same whitelist Traefik evaluates (RE2 and Ruby agree on this pattern). +public_path = Regexp.new(PUBLIC_RULE[/PathRegexp\(`(.+)`\)/, 1]) +%w[/public/webhooks/github /public/a /public/a-b_c/D9].each do |path| + raise "#{path} must stay reachable" unless path.start_with?("/public/") && public_path.match?(path) +end +%w[ + / /api/trpc /api/trpc/regulations.list /internal/events /trpc /public /public/ /publicx /publicx/a /PUBLIC/a + /public/../api/trpc /public/./a /public/a/.. /public/.. /public//a /public/a/ //public/a /x/public/a + /public/%2e%2e/api/trpc /public/%2E%2E%2Fapi%2Ftrpc /public/..%2fapi/trpc /public%2f..%2fapi /public/a%00 + /public/a;x /public/a\\..\\api /public/webhooks/github.json /public/~a /public/a@b +].each do |path| + raise "#{path} must not match the public route" if path.start_with?("/public/") && public_path.match?(path) +end + +# Merging must be harmless before Lorenzo creates the regulations secrets and +# the worker image is pullable: nothing that needs them may run, fail or alert. +def flux_kustomizations(path) + YAML.load_stream(File.read(File.join(ROOT, path))).compact.flat_map do |doc| + doc["kind"] == "ResourceSet" ? doc.dig("spec", "resources") : [doc] + end.select { |doc| doc["kind"] == "Kustomization" }.to_h { |doc| [doc.dig("metadata", "name"), doc["spec"]] } +end +flux = flux_kustomizations("clusters/k3s/apps/backend.yaml").merge(flux_kustomizations("clusters/k3s/apps/latex-worker.yaml")).merge(flux_kustomizations("clusters/k3s/apps/regulations-secrets.yaml")) +gate = flux.fetch("regulations-secrets") +raise "clusters/k3s/apps/kustomization.yaml must list regulations-secrets.yaml" unless YAML.safe_load(File.read(File.join(ROOT, "clusters/k3s/apps/kustomization.yaml")))["resources"].include?("regulations-secrets.yaml") +raise "regulations-secrets must check only the core secrets, not every ExternalSecret" unless gate["wait"] == false && gate["healthChecks"] +gated_secrets = gate["healthChecks"].select { |check| check["kind"] == "ExternalSecret" }.map { |check| [check["namespace"], check["name"]] } +external_secrets = documents("apps/regulations-secrets/external-secrets.yaml").to_h { |doc| [[doc.dig("metadata", "namespace"), doc.dig("metadata", "name")], doc.dig("spec", "target", "name")] } +%w[regulations-collab latex-worker].each do |name| + raise "#{name} must wait for regulations-secrets" unless flux.fetch(name)["dependsOn"].map { |dep| dep["name"] }.include?("regulations-secrets") +end +raise "The backend Kustomization must not wait for regulations-secrets" if flux.fetch("backend")["dependsOn"].map { |dep| dep["name"] }.include?("regulations-secrets") +{ collab_deployment => "backend", worker_deployment => "latex-worker" }.each do |resource, namespace| + resource.dig("spec", "template", "spec", "containers").flat_map { |item| item.fetch("env", []) }.map { |item| item.dig("valueFrom", "secretKeyRef") }.compact.reject { |ref| ref["optional"] }.each do |ref| + next if ref["name"] == "azure-kv" + raise "#{resource.dig("metadata", "name")} requires Secret #{ref["name"]}, which regulations-secrets does not gate" unless gated_secrets.any? { |ns, es| ns == namespace && external_secrets[[ns, es]] == ref["name"] } + end +end +backend_app = container(deployment("apps/backend/deployment.yaml", "backend"), "backend") +backend_app.fetch("env", []).map { |item| item.dig("valueFrom", "secretKeyRef") }.compact.reject { |ref| ref["name"] == "azure-kv" }.each do |ref| + raise "The backend must read regulations Secret #{ref["name"]} with optional: true" unless ref["optional"] == true && external_secrets.value?(ref["name"]) +end +raise "apps/backend must not hold ExternalSecrets other than azure-kv" unless documents("apps/backend/external-secret.yaml").map { |doc| doc.dig("metadata", "name") } == ["azure-kv"] + %w[core edge gateway].each do |component| name = "defguard-#{component}" resource = deployment("infrastructure/defguard/deployment-#{component}.yaml", name)