diff --git a/.envrc.example b/.envrc.example index e464c745..9c6c1747 100644 --- a/.envrc.example +++ b/.envrc.example @@ -1,23 +1,22 @@ -export KUBECONFIG=$PWD/.mgt-cluster-kubeconfig.yaml +# Copy this file to .envrc and adjust as needed. +# direnv will load it automatically: https://direnv.net/ +# just also reads environment variables (set them via .env or export). -export PATH=$PWD/hack/tools/bin:$PATH +# --- Stack selection --- +# Which provider and stack to build/publish/update. +# Defaults in the justfile are openstack/scs. +export PROVIDER=openstack +export CLUSTER_STACK=scs -export PROVIDER=docker +# --- OCI registry --- +# Leave unset to auto-configure ttl.sh as a temporary dev registry (24h TTL). +# Set both for a persistent registry. +# export OCI_REGISTRY=registry.example.com +# export OCI_REPOSITORY=kaas/cluster-stacks +# export OCI_USERNAME=user +# export OCI_PASSWORD=password +# export OCI_ACCESS_TOKEN=token -# Versions -export K8S_VERSION=1-27 -export CAPI_VERSION=v1.5.1 -export CAPD_VERSION=$CAPI_VERSION - -# Names -export NAMESPACE=scs-cs -export CLUSTER_CLASS_NAME=ferrol -export CLUSTER_NAME=cs-cluster -export CLUSTER_TOPOLOGY=true - -# Debugging -export DISABLE_VERSIONCHECK="true" - -# Release -export RELEASE_CLUSTER_CLASS=$CLUSTER_CLASS_NAME -export RELEASE_KUBERNETES_VERSION=1-27 +# --- Optional --- +# Kubeconfig for the management cluster (useful with direnv). +# export KUBECONFIG=$PWD/.mgt-cluster-kubeconfig.yaml diff --git a/.github/labeler.yaml b/.github/labeler.yaml index 8d3ced06..d81f6212 100644 --- a/.github/labeler.yaml +++ b/.github/labeler.yaml @@ -1,16 +1,17 @@ --- -area/code: - - "controllers/**/*" - - "pkg/**/*" -area/api: - - "api/**/*" - - "config/crd/**/*" +area/providers: + - "providers/**/*" +area/docs: + - "docs/**/*" + - "README.md" area/github: - ".github/**/*" area/hack: - "hack/**/*" - "Makefile" -area/test: - - "test/**/*" -area/templates: - - "templates/**/*" +area/openstack-scs: + - "providers/openstack/scs/**/*" +area/openstack-hcp: + - "providers/openstack/hcp/**/*" +area/docker: + - "providers/docker/**/*" diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 7fd13551..da3b09cb 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,4 +1,4 @@ - + diff --git a/Containerfile b/Containerfile new file mode 100644 index 00000000..54c8e94a --- /dev/null +++ b/Containerfile @@ -0,0 +1,48 @@ +FROM alpine:3.21 + +LABEL org.opencontainers.image.source="https://github.com/SovereignCloudStack/cluster-stacks" +LABEL org.opencontainers.image.description="Cluster Stack Build Tools" +LABEL org.opencontainers.image.licenses="Apache-2.0" + +# Install system dependencies +RUN apk add --no-cache \ + bash \ + git \ + curl \ + tar \ + gzip \ + gawk \ + python3 \ + py3-yaml \ + jq \ + just \ + ca-certificates + +# Install helm +RUN HELM_VERSION=v3.17.3 && \ + curl -fsSL "https://get.helm.sh/helm-${HELM_VERSION}-linux-amd64.tar.gz" | \ + tar -xz -C /usr/local/bin --strip-components=1 linux-amd64/helm + +# Install yq (mikefarah) +RUN YQ_VERSION=v4.45.4 && \ + curl -fsSL "https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_amd64" \ + -o /usr/local/bin/yq && chmod +x /usr/local/bin/yq + +# Install oras +RUN ORAS_VERSION=1.2.2 && \ + curl -fsSL "https://github.com/oras-project/oras/releases/download/v${ORAS_VERSION}/oras_${ORAS_VERSION}_linux_amd64.tar.gz" | \ + tar -xz -C /usr/local/bin oras +WORKDIR /workspace + +# Verify installations +RUN bash --version | head -1 && \ + helm version --short && \ + yq --version && \ + oras version && \ + just --version && \ + git --version + +# Allow git operations inside mounted volumes +RUN git config --global --add safe.directory /workspace + +CMD ["/bin/bash"] diff --git a/Makefile b/Makefile deleted file mode 100644 index e5d63845..00000000 --- a/Makefile +++ /dev/null @@ -1,303 +0,0 @@ -# Copyright 2023 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -SHELL = /usr/bin/env bash -o pipefail -.SHELLFLAGS = -ec -.DEFAULT_GOAL:=help - -##@ General - -# The help target prints out all targets with their descriptions organized -# beneath their categories. The categories are represented by '##@' and the -# target descriptions by '##'. The awk commands is responsible for reading the -# entire set of makefiles included in this invocation, looking for lines of the -# file as xyz: ## something, and then pretty-format the target and help. Then, -# if there's a line with ##@ something, that gets pretty-printed as a category. -# More info on the usage of ANSI control characters for terminal formatting: -# https://en.wikipedia.org/wiki/ANSI_escape_code#SGR_parameters -# More info on the awk command: -# http://linuxcommand.org/lc3_adv_awk.php - -.PHONY: ensure-connected-to-mgt-cluster -ensure-connected-to-mgt-cluster: - ./hack/ensure-connected-to-mgt-cluster.sh - -.PHONY: help -help: ## Display this help. - @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf " \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST) - -############# -# Variables # -############# - -TIMEOUT := $(shell command -v timeout || command -v gtimeout) - -# Directories -BIN_DIR := bin -TOOLS_DIR := hack/tools -TOOLS_BIN_DIR := $(TOOLS_DIR)/$(BIN_DIR) -export PATH := $(abspath $(TOOLS_BIN_DIR)):$(PATH) -export GOBIN := $(abspath $(TOOLS_BIN_DIR)) - -ARTIFACTS_PATH := $(ROOT_DIR)/_artifacts -# Docker -RM := --rm -TTY := -t - -##@ Binaries -############ -# Binaries # -############ - -KUSTOMIZE := $(abspath $(TOOLS_BIN_DIR)/kustomize) -kustomize: $(KUSTOMIZE) ## Build a local copy of kustomize -$(KUSTOMIZE): # Build kustomize from tools folder. - go install sigs.k8s.io/kustomize/kustomize/v5@v5.1.0 - -ENVSUBST := $(abspath $(TOOLS_BIN_DIR)/envsubst) -envsubst: $(ENVSUBST) ## Build a local copy of envsubst -$(ENVSUBST): # Build envsubst from tools folder. - go install github.com/drone/envsubst/v2/cmd/envsubst@latest - -CTLPTL := $(abspath $(TOOLS_BIN_DIR)/ctlptl) -ctlptl: $(CTLPTL) ## Build a local copy of ctlptl -$(CTLPTL): - go install github.com/tilt-dev/ctlptl/cmd/ctlptl@v0.8.20 - -CLUSTERCTL := $(abspath $(TOOLS_BIN_DIR)/clusterctl) -clusterctl: $(CLUSTERCTL) ## Build a local copy of clusterctl -$(CLUSTERCTL): - curl -sSLf https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.5.1/clusterctl-linux-amd64 -o $(CLUSTERCTL) - chmod a+rx $(CLUSTERCTL) - -HELM := $(abspath $(TOOLS_BIN_DIR)/helm) -helm: $(HELM) ## Build a local copy of helm -$(HELM): - go install helm.sh/helm/v3/cmd/helm@v3.12.3 - -KIND := $(abspath $(TOOLS_BIN_DIR)/kind) -kind: $(KIND) ## Build a local copy of kind -$(KIND): - go install sigs.k8s.io/kind@v0.20.0 - -all-tools: $(KIND) $(CTLPTL) $(KIND) $(ENVSUBST) $(KUSTOMIZE) $(CLUSTERCTL) $(HELM) - echo 'done' - -.PHONY: basics -basics: $(KIND) $(CTLPTL) $(KIND) $(ENVSUBST) $(KUSTOMIZE) $(CLUSTERCTL) - @./hack/ensure-env-variables.sh CAPI_VERSION CAPD_VERSION NAMESPACE \ - CLUSTER_CLASS_NAME K8S_VERSION CLUSTER_NAME PROVIDER - @mkdir -p build - -##@ Development -############### -# Development # -############### - -.PHONY: cluster -kind-cluster: basics ## Creates kind-dev Cluster - ./hack/kind-dev.sh - kubectl config set-context --current --namespace $(NAMESPACE) - -.PHONY: watch -watch: ## Show the current state of the CRDs and events. - watch -c "kubectl -n $(NAMESPACE) get cluster; echo; kubectl -n $(NAMESPACE) get machine; echo; kubectl -n $(NAMESPACE) get dockermachine; echo; echo Events; kubectl -A get events --sort-by=metadata.creationTimestamp | tail -5" - -##@ Clean -######### -# Clean # -######### -.PHONY: clean -clean: basics ## Remove all generated files - $(MAKE) clean-bin - -.PHONY: clean-bin -clean-bin: ## Remove all generated helper binaries - rm -rf $(TOOLS_BIN_DIR) - -.PHONY: clean-release -clean-release: ## Remove the release folder - rm -rf $(RELEASE_DIR) - -.PHONY: clean-release-git -clean-release-git: ## Restores the git files usually modified during a release - git restore ./*manager_config_patch.yaml ./*manager_pull_policy.yaml - -##@ Releasing -############# -# Releasing # -############# -## latest git tag for the commit, e.g., v0.3.10 -RELEASE_TAG ?= $(shell git describe --abbrev=0 2>/dev/null) -# the previous release tag, e.g., v0.3.9, excluding pre-release tags -PREVIOUS_TAG ?= $(shell git tag -l | grep -E "^v[0-9]+\.[0-9]+\.[0-9]." | sort -V | grep -B1 $(RELEASE_TAG) | head -n 1 2>/dev/null) -RELEASE_DIR ?= out -RELEASE_NOTES_DIR := _releasenotes - -$(RELEASE_DIR): - mkdir -p $(RELEASE_DIR)/ - -$(RELEASE_NOTES_DIR): - mkdir -p $(RELEASE_NOTES_DIR)/ - -.PHONY: test-release -test-release: basics - $(MAKE) set-manifest-image MANIFEST_IMG=$(IMAGE_PREFIX)/capd-staging MANIFEST_TAG=$(TAG) - $(MAKE) set-manifest-pull-policy PULL_POLICY=IfNotPresent - $(MAKE) release-manifests - -.PHONY: release-manifests -release-manifests: basics generate-manifests generate-go-deepcopy $(RELEASE_DIR) cluster-templates ## Builds the manifests to publish with a release - $(KUSTOMIZE) build config/default > $(RELEASE_DIR)/infrastructure-components.yaml - ## Build capd-components (aggregate of all of the above). - cp metadata.yaml $(RELEASE_DIR)/metadata.yaml - cp templates/cluster-templates/cluster-template* $(RELEASE_DIR)/ - cp templates/cluster-templates/cluster-class* $(RELEASE_DIR)/ - -.PHONY: release-notes -release-notes: basics $(RELEASE_NOTES_DIR) $(RELEASE_NOTES) - go run ./hack/tools/release/notes.go --from=$(PREVIOUS_TAG) > $(RELEASE_NOTES_DIR)/$(RELEASE_TAG).md - - -##@ Testing -########### -# Testing # -########### -ARTIFACTS ?= _artifacts -$(ARTIFACTS): - mkdir -p $(ARTIFACTS)/ - -##@ cluster-class -################# -# cluster-class # -################# - -.PHONY: re-apply-cluster-class-docker -re-apply-cluster-class-docker: ensure-connected-to-mgt-cluster basics ## Re-apply only a cluster-class. - $(HELM) -n $(NAMESPACE) template docker-$(CLUSTER_CLASS_NAME)-$(K8S_VERSION) providers/docker/$(CLUSTER_CLASS_NAME)/$(K8S_VERSION)/cluster-class | kubectl -n $(NAMESPACE) apply -f - - -.PHONY: delete-cluster-class-docker -delete-cluster-class-docker: ensure-connected-to-mgt-cluster basics ## Delete a cluster-class. - $(HELM) -n $(NAMESPACE) template docker-$(CLUSTER_CLASS_NAME)-$(K8S_VERSION) providers/docker/$(CLUSTER_CLASS_NAME)/$(K8S_VERSION)/cluster-class | kubectl -n $(NAMESPACE) delete -f - - -##@ cluster-addon -################# -# cluster-addon # -################# - -## working on cluster-addon -.PHONY: generate-deps-cluster-addon-docker -generate-deps-cluster-addon-docker: ensure-connected-to-mgt-cluster basics ## Build a cluster-class. - cd providers/docker/$(CLUSTER_CLASS_NAME)/$(K8S_VERSION)/cluster-addon && rm -rf ./charts/* && helm dependency update - -.PHONY: package-cluster-addon-docker -package-cluster-addon-docker: ensure-connected-to-mgt-cluster basics ## Build a cluster-class. - @mkdir -p .helm - $(HELM) package providers/docker/$(CLUSTER_CLASS_NAME)/$(K8S_VERSION)/cluster-addon -d .helm - -##@ Main Targets -################ -# Main Targets # -################ -.PHONY: delete-bootstrap-cluster -delete-bootstrap-cluster: ensure-connected-to-mgt-cluster basics ## Deletes Kind-dev Cluster - $(CTLPTL) delete cluster kind-scs-cluster-stacks - $(CTLPTL) delete registry cluster-stacks-registry - -.PHONY: create-bootstrap-cluster -create-bootstrap-cluster: basics kind-cluster ## Create mgt-cluster and install capi-stack. - EXP_RUNTIME_SDK=true CLUSTER_TOPOLOGY=true DISABLE_VERSIONCHECK="true" $(CLUSTERCTL) init --core cluster-api:$(CAPI_VERSION) --bootstrap kubeadm:$(CAPI_VERSION) --control-plane kubeadm:$(CAPI_VERSION) - # kubectl apply -f https://github.com/kubernetes-sigs/cluster-api-addon-provider-helm/releases/download/$(CAAPH_VERSION)/add-on-components.yaml - kubectl wait -n cert-manager deployment cert-manager --for=condition=Available --timeout=300s - kubectl wait -n capi-kubeadm-bootstrap-system deployment capi-kubeadm-bootstrap-controller-manager --for=condition=Available --timeout=300s - kubectl wait -n capi-kubeadm-control-plane-system deployment capi-kubeadm-control-plane-controller-manager --for=condition=Available --timeout=300s - kubectl wait -n capi-system deployment capi-controller-manager --for=condition=Available --timeout=300s - -.PHONY: install-provider-docker -install-provider-docker: create-bootstrap-cluster ## Install Docker Infrastructure provider. - # hangs for ever waiting for cert-manger to get available if called twice. - if kubectl get deployments.apps -n capd-system capd-controller-manager > /dev/null 2>&1; then \ - echo "capd is already installed" ; \ - else \ - echo "installing capd" ; \ - DISABLE_VERSIONCHECK="true" $(CLUSTERCTL) init --infrastructure docker:$(CAPD_VERSION); \ - fi - kubectl wait -n capd-system deployment capd-controller-manager --for=condition=Available --timeout=300s - -.PHONY: prepare-provider-docker -prepare-provider-docker: install-provider-docker ## Prepares the Docker Environment. - kubectl create namespace $(NAMESPACE) --dry-run=client -o yaml | kubectl apply -f - - -.PHONY: apply-cluster-class-docker -apply-cluster-class-docker: $(HELM) prepare-provider-docker package-cluster-addon-docker ## Applies all resources and node-images. - $(HELM) -n $(NAMESPACE) template docker-$(CLUSTER_CLASS_NAME)-$(K8S_VERSION) \ - providers/docker/$(CLUSTER_CLASS_NAME)/$(K8S_VERSION)/cluster-class > build/cluster-class-created.yaml - kubectl -n $(NAMESPACE) apply -f build/cluster-class-created.yaml - # create the docker image. Start first build in background to - docker build -t docker-ferrol-1-27-controlplaneamd64-v1:dev \ - --file providers/docker/$(CLUSTER_CLASS_NAME)/$(K8S_VERSION)/node-images/Dockerfile.controlplane \ - providers/docker/$(CLUSTER_CLASS_NAME)/$(K8S_VERSION)/node-images/ & \ - docker build -t docker-ferrol-1-27-workeramd64-v1:dev \ - --file providers/docker/$(CLUSTER_CLASS_NAME)/$(K8S_VERSION)/node-images/Dockerfile.worker \ - providers/docker/$(CLUSTER_CLASS_NAME)/$(K8S_VERSION)/node-images/ - @echo "" - @echo "Done" - -.PHONY: create-workload-cluster -create-workload-cluster: apply-cluster-class-docker basics ## Creates a workload cluster. - cat providers/docker/$(CLUSTER_CLASS_NAME)/$(K8S_VERSION)/topology-$(PROVIDER).yaml | $(ENVSUBST) - > build/topology.yaml - kubectl apply -f build/topology.yaml - # Wait for the kubeconfig to become available. - ${TIMEOUT} --foreground 5m bash -c "while ! kubectl -n $(NAMESPACE) get secrets | grep $(CLUSTER_NAME)-kubeconfig; do date; echo waiting for secret $(CLUSTER_NAME)-kubeconfig; sleep 1; done" - # Get kubeconfig and store it locally. - @mkdir -p .kubeconfigs - kubectl -n $(NAMESPACE) get secrets $(CLUSTER_NAME)-kubeconfig -o json | jq -r .data.value | base64 --decode > .kubeconfigs/.$(CLUSTER_NAME)-kubeconfig - if [ ! -s ".kubeconfigs/.$(CLUSTER_NAME)-kubeconfig" ]; then echo "failed to create .kubeconfigs/.$(CLUSTER_NAME)-kubeconfig"; exit 1; fi - ${TIMEOUT} --foreground 15m bash -c "while ! kubectl --kubeconfig=.kubeconfigs/.$(CLUSTER_NAME)-kubeconfig -n $(NAMESPACE) get nodes | \ - grep control-plane; do echo 'Waiting for control-plane in workload-cluster'; sleep 1; done" - chmod a=,u=rw .kubeconfigs/.$(CLUSTER_NAME)-kubeconfig - @echo "" - @echo 'Access to workload API server successful.' - @echo 'use KUBECONFIG=.kubeconfigs/.$(CLUSTER_NAME)-kubeconfig to access the workload cluster' - -.PHONY: install-cluster-addons -install-addons-in-workload-cluster: $(HELM) - # Cluster Addons are software tools which provide required functionality to the cluster. - # Example: CNI (Container Network Interface) - # Hint: Applications like PostgreSQL or Prometheus are not cluster addons. - - @# The "ugly" replacement of policy/v1beta1 needs to be done, since `helm template` - @# has no access to the cluster, and the new --dry-run=server is not released yet. - @# Replace the `sed` with `--dry-run=server` when helm supports --dry-run=server. - @# It should be included in helm 3.13.0 - KUBECONFIG=.kubeconfigs/.$(CLUSTER_NAME)-kubeconfig helm template \ - ./providers/docker/$(CLUSTER_CLASS_NAME)/$(K8S_VERSION)/cluster-addon \ - | sed 's#apiVersion: policy/v1beta1#apiVersion: policy/v1#' \ - > build/cluster-addons-$(CLUSTER_NAME).yaml - KUBECONFIG=.kubeconfigs/.$(CLUSTER_NAME)-kubeconfig kubectl apply -f build/cluster-addons-$(CLUSTER_NAME).yaml - - -.PHONY: release-docker -release-docker: clean-release $(HELM) ## Builds and push container images using the latest git tag for the commit. - # @if [ -z "${RELEASE_TAG}" ]; then echo "RELEASE_TAG is not set"; exit 1; fi - # @if ! [ -z "$$(git status --porcelain)" ]; then echo "Your local git repository contains uncommitted changes, use git clean before proceeding."; exit 1; fi - # git checkout "${RELEASE_TAG}" - @./hack/ensure-env-variables.sh RELEASE_CLUSTER_CLASS RELEASE_KUBERNETES_VERSION - @mkdir -p .release - cp providers/docker/$(RELEASE_CLUSTER_CLASS)/$(RELEASE_KUBERNETES_VERSION)/cluster-addon-values.yaml .release/ - cp providers/docker/$(RELEASE_CLUSTER_CLASS)/$(RELEASE_KUBERNETES_VERSION)/node-images.yaml .release/ - cp providers/docker/$(RELEASE_CLUSTER_CLASS)/$(RELEASE_KUBERNETES_VERSION)/metadata.yaml .release/ - cp providers/docker/$(RELEASE_CLUSTER_CLASS)/$(RELEASE_KUBERNETES_VERSION)/topology-* .release/ - $(HELM) package providers/docker/$(RELEASE_CLUSTER_CLASS)/$(RELEASE_KUBERNETES_VERSION)/cluster-addon -d .release/ - $(HELM) package providers/docker/$(RELEASE_CLUSTER_CLASS)/$(RELEASE_KUBERNETES_VERSION)/cluster-class -d .release/ diff --git a/README.md b/README.md index 78f40263..cab05b77 100644 --- a/README.md +++ b/README.md @@ -1,28 +1,86 @@ # Cluster Stacks -:wave: Welcome to the SCS Cluster Stacks. The reference implementation for SCS KaaS. +Reference implementation of SCS Kubernetes-as-a-Service cluster stacks, built on +[Cluster API](https://cluster-api.sigs.k8s.io/) and managed by the +[Cluster Stack Operator](https://github.com/SovereignCloudStack/cluster-stack-operator) (CSO). -#### Useful links +## Quick start -- [Quick Start](providers/docker/scs/README.md) -- [Docs](https://docs.scs.community/docs/category/cluster-stacks) +```bash +# Prerequisites: kind, kubectl, helm, clusterctl, just, yq, oras +# See docs/quickstart.md for full details -## What is Cluster Stacks? +# Create a management cluster and install CAPI + provider +kind create cluster +clusterctl init --infrastructure docker # or: --infrastructure openstack -Cluster Stacks is a comprehensive framework and set of reference -implementations for defining and managing Kubernetes clusters via the Cluster -API. Designed to support multiple providers and a broad range of Kubernetes -versions, it offers a standardized approach to configuring and operating -Kubernetes environments. For detailed information see [docs.scs.community](https://docs.scs.community/docs/container/components/cluster-stacks/components/cluster-stacks/overview). +# Build, publish, install CSO, and apply the ClusterStack in one step +export PROVIDER=docker CLUSTER_STACK=scs +just dev 1.35 --install-cso | kubectl apply -f - +``` + +See [docs/quickstart.md](docs/quickstart.md) for a complete walkthrough. + +## Available cluster stacks + +| Provider | Stack | Description | Versions | +|------------|-------|------------------------------------------|----------------| +| OpenStack | scs | Standard SCS stack (dedicated VMs) | 1-32 .. 1-36 | +| OpenStack | hcp | Hosted Control Plane (CP as pods) | 1-33 .. 1-35 | +| Docker | scs | Local development stack | 1-32 .. 1-35 | + +## Repository structure + +```text +providers/ + / + / + 1-XX/ # per-Kubernetes-minor-version directory + csctl.yaml # stack metadata and addon version pins + cluster-class/ # Helm chart: ClusterClass + infrastructure templates + cluster-addon/ # Helm chart: CNI, CCM, CSI, metrics-server + image-manager.yaml # OpenStack only: aggregated image references +``` + +Each `1-XX/` directory is self-contained: it carries its own `csctl.yaml`, +ClusterClass definition, and addon charts. There is no shared state between +minor versions. + +## Build system + +All workflows are driven by [`just`](https://github.com/casey/just): + +```bash +just build 1.35 # build locally to .release/ +just publish 1.35 # build + push to OCI registry +just dev 1.35 # publish + print ClusterStack YAML +just dev 1.35 --install-cso # also install/upgrade CSO +just matrix # show version/addon matrix +just update versions # update Kubernetes patch versions +just update addons # update addon chart versions +just generate-resources 1.35 # generate ClusterStack + Cluster YAML +just generate-docs # regenerate configuration docs +``` + +Set `PROVIDER` and `CLUSTER_STACK` environment variables (or use direnv with a `.env` file) +to target a different stack (default: `openstack`/`scs`). + +## Documentation + +- [Quickstart](docs/quickstart.md) -- local development with Docker/CAPD +- [Overview](docs/overview.md) -- architecture, versioning, and structure +- [OpenStack / scs](docs/providers/openstack/scs-configuration.md) -- standard SCS stack +- [OpenStack / hcp](docs/providers/openstack/hcp.md) -- Hosted Control Plane stack + +Configuration references are generated from ClusterClass definitions via +`just generate-docs`. ## Releases -Releases for the providers openstack/scs and openstack/scs2 are published in -the [SCS registry](oci://registry.scs.community/kaas/cluster-stacks). +Releases are published as OCI artifacts to the +[SCS registry](https://registry.scs.community/kaas/cluster-stacks). ## Community - [Matrix](https://matrix.to/#/!NZpJdPGjAHISXnHUil:matrix.org) -- [notes](https://input.scs.community/2025-scs-team-container) - - +- [Meeting notes](https://input.scs.community/2025-scs-team-container) diff --git a/docs/build-system.md b/docs/build-system.md new file mode 100644 index 00000000..2dc82486 --- /dev/null +++ b/docs/build-system.md @@ -0,0 +1,147 @@ +# Build System + +The Cluster Stacks are built, published and maintained with bash scripts in `hack/`, wrapped by a +[`justfile`](https://github.com/casey/just). Run `just` to list all recipes. +The `csctl` binary is **not** required anymore. The `csctl.yaml` files are still used as the per-version configuration +(provider, cluster stack name, Kubernetes version, addon constraints), but they are read by the scripts directly. + +## Layout + +``` +providers/// + image-manager.yaml # node images (OpenStack only), maintained by hack/update.sh + 1-35/ # one self-contained directory per Kubernetes minor version + csctl.yaml # provider, stack name, Kubernetes version + clusteraddon.yaml # addon stages and hooks + cluster-class/ # Helm chart rendering the ClusterClass + cluster-addon/ # one sub directory (Helm chart) per addon +``` + +A stack directory containing a file called `DEPRECATED` (currently `openstack/scs2`) is only maintained +(patch releases, addon updates) and does not get new Kubernetes minor versions. + +## Prerequisites + +- `bash`, `helm`, `yq` ([mikefarah](https://github.com/mikefarah/yq)), `git`, `curl`, `tar`, `jq` +- `oras` for publishing to an OCI registry +- `just` as task runner (the scripts in `hack/` also work without it) +- `python3` with `PyYAML` for `just generate-docs` +- `nix develop` (see `flake.nix`) or the `Containerfile` provide all of them + +## Configuration + +Copy `.envrc.example` to `.envrc` (or export the variables, or put them into a `.env` file which `just` reads automatically): + +| Variable | Description | +|---|---| +| `PROVIDER`, `CLUSTER_STACK` | Stack to work on, default `openstack` / `scs` | +| `OCI_REGISTRY`, `OCI_REPOSITORY` | Target registry. If unset, the temporary registry `ttl.sh` is used (24h TTL) | +| `OCI_USERNAME`, `OCI_PASSWORD` or `OCI_ACCESS_TOKEN` | Credentials for the registry | +| `GITHUB_TOKEN` | Optional, higher GitHub API rate limit when resolving Kubernetes versions | +| `OUTPUT_DIR` | Output directory for built releases, default `.release` | +| `CSO_CHART` | Chart for `just install-cso`, default the published chart. A local path works as well, e.g. `../cluster-stack-operator/charts` | +| `CSO_IMAGE_REPOSITORY`, `CSO_IMAGE_TAG` | Override the operator image for `just install-cso`, e.g. to test a locally built operator | + +## Commands + +| Command | Description | +|---|---| +| `just build 1.35` | Build the release of one Kubernetes minor version | +| `just build --all` | Build all minor versions of the stack | +| `just publish 1.35` | Build and push to the OCI registry | +| `just dev 1.35` | Build, push and print the matching `ClusterStack` resource (add `--install-cso` to install the operator as well) | +| `just install-cso` | Install or upgrade the Cluster Stack Operator with the matching OCI configuration (does not build anything) | +| `just clean` | Remove `.release` | +| `just update versions` | Update Kubernetes patch versions and `image-manager.yaml` | +| `just update addons` | Update the Helm chart dependencies of the addons | +| `just update --all --dry-run` | Preview updates for all stacks | +| `just add-version 1.37` | Scaffold a new Kubernetes minor version for all stacks that are not deprecated | +| `just matrix` | Show the version matrix | +| `just generate-resources 1.35` | Generate `ClusterStack` and `Cluster` manifests for testing | +| `just generate-image-manifests 1.35` | Generate ORC `Image` manifests (or `--format image-manager`) | +| `just generate-image-manifests 1.35 --format image-manager` | Same in the format of the OpenStack image manager | +| `just generate-docs` | Update the generated sections of the configuration docs, see below | +| `just migrate-cluster cluster.yaml` | Convert the variables of a `scs2` Cluster manifest to `scs` | +| `just container-build`, `just container-run ...` | Build the tools container and run a recipe inside it | + +A bare Kubernetes minor version as first argument is shorthand for `--version`: `just build 1.35` is the same as +`just build --version 1.35`. All other arguments are passed to the script unchanged. + +Select another stack with `PROVIDER=... CLUSTER_STACK=...`, either as environment variables or as `just` variables: +`CLUSTER_STACK=scs2 just matrix` or `just CLUSTER_STACK=scs2 matrix`. + +## Generated documentation + +`just generate-docs` renders the ClusterClass of the newest minor version of every stack and updates +`docs/providers//-configuration.md`. Only the parts between the markers + +``` + ... + ... +``` + +are replaced, everything else is hand written. Documents without markers are skipped. The descriptions in the +variables table come from the ClusterClass, so improve them there. + +## Releases + +A release consists of the packaged `cluster-class` chart, the `cluster-addon` bundle, `clusteraddon.yaml`, +`csctl.yaml`, `metadata.yaml` and `hashes.json`. It is published as one OCI artifact with the tag + +``` +---- e.g. openstack-scs-1-35-v3 +``` + +- Stable versions are `v1`, `v2`, ...: `hack/build.sh` queries the registry and uses the next free number. +- Development builds (no registry configured, or `ttl.sh`) get the version `v0-ttl.` inside the charts and + `metadata.yaml`, but the **directory and OCI tag use the dash notation** `v0-ttl-`. The operator derives + the tag it downloads from the name of the `ClusterStackRelease`, which does not contain dots. + +## Updating + +```bash +just update --all --dry-run # what would change? +just update versions --all # Kubernetes patch versions + image-manager.yaml +just update addons --all # addon Helm chart versions +``` + +Minor versions with a `FROZEN` file (`openstack/scs` `1-32`) are skipped by `update`. + +`image-manager.yaml` is merged: new versions are added and existing ones updated, but images of minor versions that +have been removed from the repository are kept, because running clusters may still use them. Check that the node image +(and its checksum) has been published before releasing a new patch version. + +## Gateway API CRDs + +The `scs` stacks enable the Gateway API of Cilium. Cilium does not install the Gateway API CRDs, therefore every stack +with `gatewayAPI.enabled` has a `gateway-api` cluster addon that carries them. It is applied before `cni`, so that the +Cilium operator finds the CRDs when it starts, and `gatewayAPI.gatewayClass.create` is set to `"true"` (the default +`auto` only creates the `GatewayClass` if the CRD is visible while rendering the chart, which it never is for the +operator). + +The CRDs are vendored, the version has to match the Cilium version of the `cni` addon (Cilium 1.19: Gateway API v1.4.1, +Cilium 1.20: Gateway API v1.6.3). `just update addons` keeps them in sync with Cilium, or by hand: + +```bash +./hack/update-gateway-api-crds.sh v1.4.1 providers/docker/scs/1-35 +./hack/update-gateway-api-crds.sh v1.6.3 providers/openstack/scs/1-35 providers/openstack/scs/1-36 +``` + +The standard channel is installed plus `TLSRoute`, which the hosted control plane provider needs (experimental channel +before Gateway API v1.5, standard channel since then), since Gateway API v1.5 also `BackendTLSPolicy`, without which +Cilium 1.20 does not enable the Gateway API. Clusters that are upgraded to a release that contains the addon +need a restart of the `cilium-operator` once (`kubectl -n kube-system rollout restart deploy/cilium-operator`), the +operator only looks for the CRDs when it starts. + +## Migrating clusters from scs2 to scs + +`scs` uses unified variables (`flavor`, `rootDisk`, `serverGroupID`, `additionalBlockDevices`, ...) instead of the separate +`controlPlane*` / `worker*` variables of `scs2`. `hack/migrate-cluster.sh` converts the `topology.variables` of a `Cluster` +manifest, see the header of the script for the exact rules and the variables that can not be migrated. + +```bash +./hack/migrate-cluster.sh cluster.yaml > cluster-scs.yaml +``` + +It does not change `topology.class`. Note that the Cluster Stack Operator's webhook does not allow changing the +ClusterStack of an existing cluster. diff --git a/docs/overview.md b/docs/overview.md index 60d45ec4..6f240193 100644 --- a/docs/overview.md +++ b/docs/overview.md @@ -1,148 +1,154 @@ # Overview -## Cluster Stacks +## What is a Cluster Stack? -Cluster Stacks is a comprehensive framework and reference implementations for defining and managing Kubernetes clusters via the Cluster API. It is designed to cater to multiple providers and supports a broad range of Kubernetes versions, offering a standardized approach to managing and configuring Kubernetes clusters. +A cluster stack is a versioned bundle of everything needed to create and operate +a Kubernetes cluster via the Cluster API (CAPI). It packages three layers: -It encapsulates multiple layers, including node configuration, Cluster API setup, and application-level configurations, such as the Container Network Interface (CNI). By packaging these interdependent configurations, the cluster stack allows for efficient management and deployment of Kubernetes clusters, offering standardized, resilient, and self-managed Kubernetes environments. +1. **Cluster Class** -- a CAPI `ClusterClass` resource that defines the + infrastructure templates, machine configurations, and topology variables. +2. **Cluster Addons** -- Helm charts for core cluster components: CNI (Cilium), + Cloud Controller Manager, CSI driver, and metrics-server. +3. **Node Images** -- build instructions or references for the OS images that + run on cluster nodes. -## πŸ”§ Usage +These layers are published together as an OCI artifact. The +[Cluster Stack Operator](https://github.com/SovereignCloudStack/cluster-stack-operator) +(CSO) pulls these artifacts and installs them into a management cluster, making +the `ClusterClass` available for creating workload clusters. -Follow our [quickstart guide](providers/openstack/quickstart.md) for an introduction on how to deploy cluster stacks on openstack. +## Repository structure -## Layers of a Cluster Stack - -In essence, a cluster stack is an amalgamation of various components each of which serves a crucial role in setting up, maintaining, and operating a Kubernetes cluster. In the context of our framework, we categorize these components into three core layers: `cluster-class`, `cluster-addons`, and `node-images`. Let's delve deeper into understanding each of these layers: - -### πŸ“š Cluster Class - -The Cluster Class serves as a blueprint for creating and configuring Kubernetes clusters consistently. It encapsulates various aspects of a cluster, including: - -* The infrastructure provider details -* Networking configurations -* Cluster-class templating -* Other cluster-specific settings - -Essentially, it defines the desired configuration and properties of a Kubernetes cluster. It leverages the [ClusterClass](https://cluster-api.sigs.k8s.io/tasks/experimental-features/cluster-class/) feature of Cluster API, which provides a declarative, Kubernetes-style API for cluster creation, configuration, and management. Any change in this layer or in the node-image or cluster-addon layers triggers a version bump in the cluster class, hence the cluster stack. - -### 🎁 Cluster Addons - -Cluster Addons are core components or services required for the Kubernetes cluster to function correctly and efficiently. These are not user-facing applications but rather foundational services critical to the operation and management of a Kubernetes cluster. They're usually installed and configured after the cluster infrastructure has been provisioned and before the cluster is ready to serve workloads. - -Cluster addons encompass a variety of functionalities, including but not limited to: - -* Container Network Interfaces (CNI): These are plugins that facilitate container networking. A CNI is integral to setting up network connectivity and ensuring communication between pods in a Kubernetes cluster. -* Cloud Controller Manager (CCM): The CCM is a Kubernetes control plane component that embeds the cloud-specific control logic. Its role is to manage the communication with the underlying cloud services. -* Konnectivity service: This is a network proxy that enables connectivity from the control plane to nodes and vice versa. It is a critical component that supports Kubernetes API server connectivity. -* Metrics Server: A cluster-wide aggregator of resource usage data, Metrics Server collects CPU, memory, and other metrics from nodes and pods, enabling features like Horizontal Pod Autoscaling. - -It's important to note that cluster addons are not user-provided applications or services that can be installed multiple times, such as ingress controllers, application-level monitoring tools, or user-facing APIs. Those are left to the discretion and responsibility of the users, who install and manage them according to their specific needs and preferences. - -Each addon version is independent and can be updated separately. However, a change in this layer also necessitates a version bump in the cluster class and the cluster stack, which is reflected in the metadata.yaml. - -### 🎞️ Node Images - -Node images provide the foundation for the operating system environment on each node of a Kubernetes cluster. They are typically a minimal operating system distribution, like a lightweight Linux distro, which may also include container runtime components such as Docker or containerd. - -Node images are responsible for providing the necessary environment and dependencies to support Kubernetes components and workloads. This includes components like kubelet, kube-proxy, and other necessary system utilities and libraries. - -The version of a node image can be different from that of the cluster stack or the cluster class. However, an update to a node image will trigger a version bump in the cluster class and hence the cluster stack. - -In the cluster-stacks repository's directory structure, the build instructions for Node Images are always placed within the respective directory. The instructions outline the steps and configurations required to create the Node Image automatically. The specific method for releasing the Node Image may vary based on the provider's capabilities and requirements. - -During the development phase, the build instructions serve as a reference within the repository itself. These instructions may utilize tools like Packer or other image-building techniques. This allows for flexibility and customization, enabling users to define their Node Images according to specific needs and requirements. - -However, when it comes to the release of the cluster stack, the Node Image can be provided in different ways depending on the capabilities of the provider or the desired deployment method. Here are a few examples: - -1. **URL on a remote endpoint**: In some cases, `providers` may support deploying a Node Image directly from a URL. In this scenario, the Node Image referenced in the `cluster stack`, specifically in the `cluster class`, would be provided as a URL pointing to a pre-built image accessible remotely. -1. **Artifact**: If the provider supports artifacts, the Node Image can be released as an artifact, such as a qcow2 file. The artifact would be uploaded to the provider, and the `cluster stack` references the artifact for node provisioning. -1. **Build Instructions**: In cases where the provider doesn't support direct URL deployment or artifact-based provisioning, the build instructions defined within the repository become critical. The build instructions serve as a comprehensive guide to build the Node Image, specifying all the necessary steps and configurations. - -Regardless of the release method, the cluster stack, specifically the cluster class, references the appropriate Node Image to be used for node provisioning. - -By allowing flexibility in the release and deployment methods of Node Images, the cluster stack framework caters to various provider capabilities and user requirements. This adaptability ensures the cluster stack can be deployed in diverse environments while maintaining a consistent and manageable approach to managing Kubernetes clusters. - -## 🌐 IaaS Provider, Kubernetes Service Provider, and Cluster API -In the context of the `cluster-stacks`, we distinguish between two types of providers: +```text +providers/ + / + / + 1-XX/ # one directory per Kubernetes minor version + csctl.yaml # metadata: provider, name, k8s version, addon pins + cluster-class/ # Helm chart producing the ClusterClass + cluster-addon/ # Helm chart with CNI, CCM, CSI, metrics-server + image-manager.yaml # OpenStack only: aggregated image references +``` -An **IaaS Provider**, in general, offers Infrastructure as a Service - providing the fundamental compute, storage, and network resources on which workloads can be run. In the context of cluster-stacks, an IaaS Provider specifically refers to an entity that owns an API for their infrastructure. If an organization uses a common infrastructure API, such as OpenStack, they are not considered an IaaS Provider in this context. However, if the organization owns the API for its infrastructure, it becomes an IaaS Provider for the purposes of cluster-stacks. +### Per-minor-version directories -A **Kubernetes Service Provider**, on the other hand, is an entity that implements a cluster stack. They do so on top of the IaaS Providers, potentially spanning across multiple IaaS Providers. They use the IaaS Provider's infrastructure services and integrate them into their cluster stack implementations. +Each `1-XX/` directory is completely self-contained. It carries its own +`csctl.yaml`, ClusterClass templates, and addon charts. There is no inheritance +or sharing between minor versions -- changes to one version never affect another. -The **Cluster API (CAPI)** is a Kubernetes project aimed at simplifying the process of managing Kubernetes clusters. It offers a declarative API that automates the creation, configuration, and management of clusters, providing a standardized way to interact with Kubernetes. The cluster stack approach leverages CAPI to deliver self-managed Kubernetes clusters. +This design makes it straightforward to: -## πŸ“Œ Defining and Adding Providers -The structure of this repository is specifically designed to handle multiple providers, multiple cluster stacks per provider, and multiple Kubernetes versions per cluster stack. This organized structure allows us to effectively manage, develop, and maintain multiple cluster stacks across various Kubernetes versions and providers, all in a single repository. +- Support provider-specific API and implementation differences across minors. +- Pin addons to version ranges that match the Kubernetes minor + (e.g. CCM `2.34.x` for K8s 1.34). +- Drop old versions by simply removing their directory. -### πŸ“ Repository Structure -The repository maintains a specific structure: +### csctl.yaml -* Each IaaS Provider has a directory under providers. -* Each IaaS Provider can have multiple cluster stack implementations. -* Each cluster stack supports multiple Kubernetes major and minor versions. +Each version directory contains a `csctl.yaml` that serves as the single source +of truth for that version: -``` -providers/ -└── / - └── / - └── / +```yaml +apiVersion: csctl.clusterstack.x-k8s.io/v1alpha1 +config: + clusterStackName: scs + kubernetesVersion: v1.35.3 + provider: + type: openstack +addons: # version pins used by `just update addons` + ccm: 2.35.x + csi: 2.35.x ``` -The directory structure for adding a new provider would look something like this: +The `addons` section declares SemVer ranges. When you run `just update addons`, +the build system resolves these ranges against upstream Helm repositories and +updates the `Chart.yaml` dependencies in the addon chart. -``` -providers/// -# example -providers/openstack/scs/1-28 -``` -This granular, hierarchical structure allows us to manage different versions of Kubernetes and their associated cluster stacks across different providers. - -We decided to support multiple Kubernetes major and minor versions to provide the flexibility to accommodate different implementation requirements of the provider. However, we deliberately chose not to support Kubernetes patch versions directly. The reason is the high frequency of patch versions release (often weekly), which would complicate maintenance efforts significantly. - -Instead, we represent Kubernetes patch version updates through changes in our cluster stack version. For instance, if a patch version of Kubernetes necessitates a change in the node-image or the cluster-class configuration, this would trigger a version bump in the corresponding cluster stack, hence the cluster class, as reflected in the metadata.yaml. +## Available stacks -In this way, our versioning system, our directory structure, and our approach to Kubernetes versioning are all interlinked, providing us a comprehensive, manageable, and resilient framework for maintaining various Kubernetes distributions or cluster stacks across multiple providers and versions. +### OpenStack / scs -## πŸ“‘ Versioning +The standard SCS cluster stack. Creates dedicated VMs for both control plane and +worker nodes on OpenStack. Supports Kubernetes 1.32 through 1.36. -Note: This section is subject to change, as our new tool [csctl](https://github.com/SovereignCloudStack/csctl) will incorporate future versioning capabilities. +- Versions 1-32 through 1-36 use a `ClusterClass`-based configuration model. +- Versions 1-35 and 1-36 use unified variables (`flavor`) and + per-role overrides via `topology.controlPlane.variables.overrides`. -A fundamental aspect of the cluster stack approach is the encapsulation of versioning within a cluster stack distribution. Each of the components can be updated independently, leading to a flexible and maintainable system. +### OpenStack / hcp -However, the critical point to understand here is the relationship between these component versions and the cluster stack version. Whenever there's a change or an update to either the cluster addon or the node image, the version of the cluster stack must be bumped. And due to the connection between the cluster class and the cluster stack, the cluster class version must be updated to match the new cluster stack version. +Hosted Control Plane stack. The Kubernetes control plane runs as pods in the +management cluster (using the +[teutonet Hosted Control Plane provider](https://github.com/teutonet/cluster-api-provider-hosted-control-plane)); +only worker nodes are OpenStack VMs. -The cluster stack version doesn't simply mirror the versions of its components, but rather, it reflects the "version of change". In essence, the cluster stack version is a reflection of the state of the entire stack as a whole at a particular point in time. Any change in the components warrants a new state, and therefore a new version of the cluster stack. +See [providers/openstack/hcp.md](providers/openstack/hcp.md) for details. -So, an update to the cluster addon component will bump the version of the cluster stack, irrespective of the existing version of the node image. The same applies vice versa. When such an update occurs, the version of the cluster class is also incremented to align with the new cluster stack version, maintaining the unity of the cluster stack framework. +### Docker / scs -This versioning approach ensures a clear and precise track of changes, promoting efficient management, and isolated testing. It offers enhanced resilience for the Kubernetes distribution or the cluster stack, ensuring safe and secure upgrades even in rapid update cycles. It's an efficient method of maintaining stability in the rapidly changing environment of a Kubernetes stack. +A lightweight stack for local development and CI. Uses the CAPI Docker +infrastructure provider. No cloud credentials required. -The versioning of the cluster stack is primarily managed through a file named metadata.yaml, located at the root directory of each cluster stack. This file serves as the source of truth for the versioning information of the cluster stack, cluster class, node images, and cluster addons. +## Versioning -Here is an example of how metadata.yaml could look like: -``` -apiVersion: metadata.clusterstack.x-k8s.io/v1alpha1 -versions: - clusterStack: v3 - kubernetes: v1.27.3 - components: - clusterAddon: v2 - nodeImage: v1 -``` -In this example, the cluster stack (and thus the cluster class) is on version 3, while the cluster addon is on version 2 and node image is on version 1. +A cluster stack version is a single integer (`v1`, `v2`, ...) that represents +the state of the entire bundle at a point in time. Any change to the +ClusterClass, addons, or node images produces a new version. -When there's a change or update in the node images or cluster addons, we would bump the version of the cluster stack and cluster class, while leaving the unaffected component's version intact. So if the node image was updated, the metadata.yaml might then look like this: +The full identifier of a cluster stack release is: +```text +----v ``` -apiVersion: metadata.clusterstack.x-k8s.io/v1alpha1 -versions: - clusterStack: v4 - kubernetes: v1.27.3 - components: - clusterAddon: v2 - nodeImage: v2 -``` - -Here, the cluster stack and cluster class versions were updated to v4, the node image version was bumped to v2 due to the changes, while the cluster addon remained on v2 as it was not affected by the update. -This versioning approach allows us to keep track of changes across different components, manage these components effectively, and conduct isolated testing. This ensures that our Kubernetes distribution or cluster stack remains resilient, and we can perform safe and secure upgrades even in the face of rapid update cycles. The metadata.yaml plays a critical role in maintaining this structure and providing an accurate representation of the state of the whole stack at any given time. +For example: `openstack-scs-1-35-v3` means the third release of the `scs` stack +for Kubernetes 1.35 on OpenStack. + +Kubernetes patch versions are not part of the directory structure. A patch +version update (e.g. 1.35.1 to 1.35.2) is delivered by bumping the cluster +stack version, which updates the `kubernetesVersion` field in `csctl.yaml` and +triggers a rolling update of nodes. + +`just publish` numbers stable releases as the highest published version + 1 +and skips publishing if nothing changed compared to the latest published +release. Published releases are never overwritten. + +## Build system + +The build system uses [`just`](https://github.com/casey/just) as a task runner and a set of bash scripts in `hack/`: + +| Command | Description | +| ------------------------------------------------ | ---------------------------------------------- | +| `just build 1.35` | Build locally to `.release/` | +| `just publish 1.35` | Build and push to OCI registry | +| `just dev 1.35` | Publish and print `ClusterStack` resource YAML | +| `just dev 1.35 --install-cso` | Also install/upgrade CSO via Helm | +| `just install-cso` | Install CSO standalone | +| `just matrix` | Show version and addon matrix | +| `just update versions` | Update Kubernetes patch versions | +| `just update addons` | Update addon charts from upstream | +| `just generate-resources 1.35` | Generate `ClusterStack` + `Cluster` YAML | +| `just generate-docs` | Regenerate configuration documentation | +| `just clean` | Remove `.release/` build artifacts | + +### OCI workflow + +Cluster stack releases are OCI artifacts. For development, the build system +auto-configures [ttl.sh](https://ttl.sh) as a temporary registry (artifacts +expire after 24 hours). For production, set `OCI_REGISTRY` and +`OCI_REPOSITORY` environment variables to point to your registry. + +The CSO is installed via its Helm chart at +`oci://registry.scs.community/cluster-stacks/cso` and configured to pull from +whichever OCI registry you are publishing to. + +## Providers and the Cluster API + +The Cluster API (CAPI) provides a declarative, Kubernetes-native API for +creating and managing clusters. Each infrastructure provider (OpenStack, Docker, +etc.) implements the CAPI contracts for provisioning machines and networks. + +The CSO sits on top of CAPI: it manages the lifecycle of cluster stack releases, +installs ClusterClasses, and handles version upgrades. Users create `Cluster` +resources that reference a `ClusterClass` -- the CAPI topology controller then +reconciles the desired state into actual infrastructure. diff --git a/docs/providers/docker/scs-configuration.md b/docs/providers/docker/scs-configuration.md new file mode 100644 index 00000000..4af621bf --- /dev/null +++ b/docs/providers/docker/scs-configuration.md @@ -0,0 +1,62 @@ +# Configuration (Docker / scs) + +The Docker cluster stack is designed for local development and CI. It uses the +CAPI Docker infrastructure provider and requires no cloud credentials. + +## Version matrix + + +| Version | K8s | CS Version | cilium | metrics-server | +|---------|-----|----------|---------|---------| +| 1-32 | 1.32.11 | - | 1.19.4 | 3.13.0 | +| 1-33 | 1.33.12 | - | 1.19.4 | 3.13.0 | +| 1-34 | 1.34.8 | - | 1.19.4 | 3.13.0 | +| 1-35 | 1.35.5 | - | 1.19.4 | 3.13.0 | +| 1-36 | 1.36.4 | - | 1.19.4 | 3.13.0 | + + +## Example + +```yaml +apiVersion: cluster.x-k8s.io/v1beta2 +kind: Cluster +metadata: + name: my-docker-cluster + namespace: default +spec: + clusterNetwork: + pods: + cidrBlocks: + - 192.168.0.0/16 + serviceDomain: cluster.local + services: + cidrBlocks: + - 10.96.0.0/12 + topology: + classRef: + name: docker-scs-1-35-v1 + controlPlane: + replicas: 1 + version: v1.35.0 + workers: + machineDeployments: + - class: default-worker + name: default-worker + replicas: 1 +``` + +## Available variables + + +|Name|Type|Default|Example|Description|Required| +|----|----|-------|-------|-----------|--------| +|`imageRepository`|string|""|"registry.k8s.io"|imageRepository sets the container registry to pull images from. If empty, the kubeadm default will be used.|false| +|`certSANs`|array|[]|["mydomain.example"]|certSANs sets extra Subject Alternative Names for the API Server signing cert.|false| +|`oidcConfig.clientID`|string||"kubectl"|A client id that all tokens must be issued for.|false| +|`oidcConfig.issuerURL`|string||"https://dex.k8s.scs.community"|URL of the provider that allows the API server to discover public signing keys. Only URLs that use the https:// scheme are accepted.|false| +|`oidcConfig.usernameClaim`|string|"preferred_username"|"preferred_username"|JWT claim to use as the user name.|false| +|`oidcConfig.groupsClaim`|string|"groups"|"groups"|JWT claim to use as the user's group. If the claim is present it must be an array of strings.|false| +|`oidcConfig.usernamePrefix`|string|"oidc:"|"oidc:"|Prefix prepended to username claims to prevent clashes with existing names.|false| +|`oidcConfig.groupsPrefix`|string|"oidc:"|"oidc:"|Prefix prepended to group claims to prevent clashes with existing names.|false| +|`registryMirrors`|array|[]||Registry mirrors for upstream container registries. Configures both containerd and CRI-O to pull through a mirror.|false| + diff --git a/docs/providers/openstack/configuration.md b/docs/providers/openstack/configuration.md deleted file mode 100644 index 3a206128..00000000 --- a/docs/providers/openstack/configuration.md +++ /dev/null @@ -1,85 +0,0 @@ -# Configuration - -This page lists the custom configuration options available, including their default values and if they are optional. The following example shows how these variables can be used inside the `cluster.yaml` file under `spec.topology.variables`. - -## Example - -```yaml -apiVersion: cluster.x-k8s.io/v1beta1 -kind: Cluster -metadata: - name: - namespace: - labels: - managed-secret: cloud-config -spec: - clusterNetwork: - pods: - cidrBlocks: - - 192.168.0.0/16 - serviceDomain: cluster.local - services: - cidrBlocks: - - 10.96.0.0/12 - topology: - variables: // <-- variables from the table can be set here - - name: controller_flavor - value: "SCS-4V-8-20" - - name: worker_flavor - value: "SCS-4V-8-20" - - name: external_id - value: "ebfe5546-f09f-4f42-ab54-094e457d42ec" - class: openstack-alpha-1-29-v2 - controlPlane: - replicas: 2 - version: v1.29.3 - workers: - machineDeployments: - - class: openstack-alpha-1-29-v2 - failureDomain: nova - name: openstack-alpha-1-29-v2 - replicas: 4 -``` - -Variables from the table containing a `.` are to be used in an object with the part before the dot being the object name and the part behind the dot being the value names. The following example demonstrates this with `oidc_config`. - -```yaml ---- -topology: - variables: - - name: oidc_config - value: - issuer_url: "https://dex.k8s.scs.community" - client_id: "kubectl" -``` - -## Available variables - - -| Name | Type | Default | Example | Description | Required | -| ---------------------------------- | ------- | -------------------------------- | -------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------- | -| `external_id` | string | "" | "ebfe5546-f09f-4f42-ab54-094e457d42ec" | ExternalNetworkID is the ID of an external OpenStack Network. This is necessary to get public internet to the VMs. | False | -| `controller_flavor` | string | "SCS-2V-4-20s" | "SCS-2V-4-20s" | OpenStack instance flavor for control-plane nodes. | False | -| `worker_flavor` | string | "SCS-2V-4" | "SCS-2V-4" | OpenStack instance flavor for worker nodes. | False | -| `controller_root_disk` | integer | | 25 | Root disk size in GiB for control-plane nodes. OpenStack volume will be created and used instead of an ephemeral disk defined in flavor. Should only be used for the diskless flavors. | False | -| `worker_root_disk` | integer | 25 | 25 | Root disk size in GiB for worker nodes. OpenStack volume will be created and used instead of an ephemeral disk defined in flavor. Should be used for the diskless flavors. | False | -| `openstack_security_groups` | array | [] | ['security-group-1'] | The names of the security groups to assign to the instance | False | -| `cloud_name` | string | "openstack" | "openstack" | The name of the cloud to use from the clouds secret | False | -| `secret_name` | string | "openstack" | "openstack" | The name of the clouds secret | False | -| `controller_server_group_id` | string | "" | "3adf4e92-bb33-4e44-8ad3-afda9dfe8ec3" | The server group to assign the control plane nodes to. | False | -| `worker_server_group_id` | string | "" | "869fe071-1e56-46a9-9166-47c9f228e297" | The server group to assign the worker nodes to. | False | -| `ssh_key` | string | "" | "capi-keypair" | The ssh key to inject in the nodes. | False | -| `apiserver_loadbalancer` | string | "octavia-amphora" | "none, octavia-amphora, octavia-ovn" | "In this cluster-stack we have two kind of loadbalancers. Each of them has its own configuration variable. This setting here is to configure the loadbalancer that is placed in front of the apiserver.
You can choose from 2 options:

none:
No loadbalancer solution will be deployed

octavia-amphora:
(default) Uses openstack's loadbalancer service (provider:amphora)

octavia-ovn:
Uses openstack's loadbalancer service (provider:ovn)
| False | -| `dns_nameservers` | array | ['5.1.66.255', '185.150.99.255'] | ['5.1.66.255', '185.150.99.255'] | "DNSNameservers is the list of nameservers for the OpenStack Subnet being created. Set this value when you need to create a new network/subnet while the access through DNS is required.
| False | -| `node_cidr` | string | "10.8.0.0/20" | "10.8.0.0/20" | "NodeCIDR is the OpenStack Subnet to be created. Cluster actuator will create a network, a subnet with NodeCIDR, and a router connected to this subnet. If you leave this empty, no network will be created.
| False | -| `certSANs` | array | [] | ['mydomain.example'] | CertSANs sets extra Subject Alternative Names for the API Server signing cert. | False | -| `oidc_config.client_id` | string | | kubectl | A client id that all tokens must be issued for. | | -| `oidc_config.issuer_url` | string | | `https://dex.example.com` | URL of the provider that allows the API server to dis cover public signing keys. Only URLs that use the https:// scheme are acc epted. This is typically the provider's discovery URL, changed to have an emp ty path | | -| `oidc_config.username_claim` | string | preferred_username | preferred_username | JWT claim to use as the user name. By default sub, whi ch is expected to be a unique identifier of the end user. Admins can choose oth er claims, such as email or name, depending on their provider. However, cla ims other than email will be prefixed with the issuer URL to prevent naming cla shes with other plugins. | | -| `oidc_config.groups_claim` | string | groups | groups | JWT claim to use as the user's group. If the claim is present it must be an array of strings. | | -| `oidc_config.username_prefix` | string | oidc: | oidc: | Prefix prepended to username claims to prevent cla shes with existing names (such as system: users). For example, the value oid c: will create usernames like oidc:jane.doe. If this flag isn't provided and --o idc-username-claim is a value other than email the prefix defaults to ( Iss uer URL )# where ( Issuer URL ) is the value of --oidc-issuer-url. The value - c an be used to disable all prefixing. | | -| `oidc_config.groups_prefix` | string | oidc: | oidc: | Prefix prepended to group claims to prevent clashes wit h existing names (such as system: groups). For example, the value oidc: will cre ate group names like oidc:engineering and oidc:infra. | | -| `network_mtu` | integer | | 1500 | NetworkMTU sets the maximum transmission unit (MTU) value to address fragmentation for the private network ID. | False | -| `diskPinAZ` | boolean | | False | This tells capo to create volumes in the same AZ as the VM that they are attached to. This affects cinder volumes used as controlPlane and worker machine root disks as well as additionalBlockDevices. Only works for clouds where volume AZs are matched with compute AZs. There it might be a performance optimization or a requirement. -| `controlPlaneAvailabilityZones` | array | | ['nova'] | ControlPlaneAvailabilityZones is the set of availability zones which control plane machines may be deployed to. | False | -| `controlPlaneOmitAvailabilityZone` | boolean | | True | ControlPlaneOmitAvailabilityZone causes availability zone to be omitted when creating control plane nodes, allowing the Nova scheduler to make a decision on which availability zone to use based on other scheduling constraints. | False | diff --git a/docs/providers/openstack/hcp.md b/docs/providers/openstack/hcp.md new file mode 100644 index 00000000..c8aa6f38 --- /dev/null +++ b/docs/providers/openstack/hcp.md @@ -0,0 +1,177 @@ +# OpenStack HCP (Hosted Control Plane) + +The `hcp` cluster stack runs the Kubernetes control plane as pods inside the +management cluster, using the +[teutonet Hosted Control Plane provider](https://github.com/teutonet/cluster-api-provider-hosted-control-plane). +Only worker nodes are created as OpenStack VMs. + +## How it differs from the `scs` stack + +| Aspect | scs | hcp | +| ------ | --- | --- | +| Control plane | Dedicated OpenStack VMs | Pods in the management cluster | +| Control plane template | `KubeadmControlPlaneTemplate` | `HostedControlPlaneTemplate` (v1alpha1) | +| CAPI API version | `v1beta2` (1-35) | `v1beta1` | +| CP machine infrastructure | `OpenStackMachineTemplate` | None (no CP VMs) | +| API server load balancer | Configurable (Octavia / none) | `none` by default (uses Gateway API) | +| API server floating IP | Configurable | Disabled by default | +| Variable model | Unified (`flavor`, `rootDisk`) | Worker-prefixed (`workerFlavor`, etc.) | +| Unique variables | `oidcConfig`, `registryMirrors` | `gatewayName`, `gatewayNamespace`, `controlPlaneReplicas` | + +## Architecture + +```text +Management Cluster + +-- CP pods (etcd, apiserver, controller-manager, scheduler) + +-- HostedControlPlane resource + +-- Gateway API ingress (optional) + +OpenStack + +-- Worker VMs only + +-- Cilium CNI + +-- CCM + CSI (as DaemonSets/Deployments on workers) +``` + +The control plane is exposed to worker nodes via the management cluster's +network. For external access to the API server, configure Gateway API resources +using the `gatewayName` and `gatewayNamespace` variables. + +## Prerequisites + +In addition to the standard [quickstart prerequisites](../../quickstart.md): + +1. A Gateway API implementation with the `TLSRoute` CRD and a `Gateway` with a TLS passthrough listener on a wildcard + hostname (for example `*..sslip.io`). The workers reach the hosted API server and the Konnectivity + server through it. On an `scs` management cluster the Gateway API CRDs and the `cilium` `GatewayClass` come with the + stack, see [Gateway API CRDs](../../build-system.md#gateway-api-crds). + +2. cert-manager >= 1.18, started with `--enable-certificate-owner-ref=true`. + +3. Install the teutonet Hosted Control Plane provider in the management cluster. The provider has to match the Gateway + API version of the management cluster: + + | Provider | Needs | Works with | + |---|---|---| + | v1.5.0 | `TLSRoute` `v1alpha2` (Gateway API >= 1.3) | Cilium 1.19 (Gateway API v1.4.1), tested | + | v1.6.0 and newer | `TLSRoute` `v1` (Gateway API >= 1.5) | Cilium 1.20 (Gateway API v1.6.3), not tested | + + Newer providers do not start with Cilium 1.19 (`no matches for kind "TLSRoute" in version "gateway.networking.k8s.io/v1"`), + although the README of the provider states Gateway API v1.3+. Installation: + + ```bash + kubectl apply -f https://github.com/teutonet/cluster-api-provider-hosted-control-plane/releases/download/v1.5.0/control-plane-components.yaml + ``` + +4. A default `StorageClass` with fast volumes for etcd. The `HostedControlPlane` does not offer a setting for the + `StorageClass` of etcd, it uses the default one. On a cloud where the default volume type has a high latency etcd + becomes slow, the API server answers watch requests late and the Konnectivity agents on the workers do not become + ready (`kubectl logs` and `kubectl exec` fail, the metrics server does not work). + +5. If the tenant networks of the cloud have a larger MTU than the external network, set the variable `networkMTU` (for + example `1442`). Otherwise connections from the workers to the hosted API server through a floating IP take several + seconds per connection, as the path MTU discovery does not work between the networks. + +## Available versions + +| Version | Kubernetes | Notes | +|---------|------------|----------------------------| +| 1-33 | 1.33 | Initial HCP release | +| 1-34 | 1.34 | Updated CCM/CSI | +| 1-35 | 1.35 | Updated CCM/CSI | +| 1-36 | 1.36 | Needs Cluster API >= 1.12.8 or >= 1.13.1, see [Versioning](../../versioning.md) | + +## Usage + +### Build and publish + +```bash +export PROVIDER=openstack +export CLUSTER_STACK=hcp + +just dev 1.35 --install-cso | kubectl apply -f - +kubectl get clusterclass -w +``` + +### Create a cluster + +```yaml +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + name: my-hcp-cluster + namespace: my-tenant + labels: + managed-secret: cloud-config +spec: + clusterNetwork: + pods: + cidrBlocks: + - 192.168.0.0/16 + serviceDomain: cluster.local + services: + cidrBlocks: + - 10.96.0.0/12 + topology: + class: openstack-hcp-1-35-v1 + controlPlane: + replicas: 3 + version: v1.35.0 + workers: + machineDeployments: + - class: default-worker + name: default-worker + replicas: 3 + variables: + - name: workerFlavor + value: "SCS-4V-8-20" + - name: networkExternalID + value: "your-external-network-uuid" +``` + +### With Gateway API ingress + +To expose the hosted API server externally, configure a Gateway resource in the +management cluster and reference it: + +```yaml + variables: + - name: gatewayName + value: "cluster-gateway" + - name: gatewayNamespace + value: "gateway-system" +``` + +## Configuration variables + +### HCP-specific variables + +| Variable | Type | Default | Description | +| -------- | ---- | ------- | ----------- | +| `controlPlaneReplicas` | integer | 3 | Number of hosted control plane replicas | +| `gatewayName` | string | "" | Gateway API resource name for API server ingress | +| `gatewayNamespace` | string | "default" | Namespace of the Gateway resource | +| `disableAPIServerFloatingIP` | boolean | true | Disable floating IP for API server | +| `apiServerLoadBalancer` | string | "none" | Load balancer type (typically `none` for HCP) | + +### Worker variables + +| Variable | Type | Default | Description | +| -------- | ---- | ------- | ----------- | +| `workerFlavor` | string | "SCS-2V-4" | OpenStack flavor for worker nodes | +| `workerRootDisk` | integer | 25 | Root disk size in GiB | +| `workerServerGroupID` | string | "" | Anti-affinity server group for workers | +| `workerAdditionalBlockDevices` | array | [] | Extra Cinder volumes for workers | +| `workerSecurityGroups` | array | [] | Security group names for workers | +| `workerSecurityGroupIDs` | array | [] | Security group UUIDs for workers | + +### Shared variables (same as scs) + +| Variable | Type | Default | Description | +| -------- | ---- | ------- | ----------- | +| `imageName` | string | "ubuntu-capi-image" | Base OS image name | +| `networkExternalID` | string | "" | External network UUID | +| `networkMTU` | integer | (provider default) | MTU for cluster network | +| `dnsNameservers` | array | ["9.9.9.9", "149.112.112.112"] | DNS nameservers | +| `nodeCIDR` | string | "10.8.0.0/20" | Subnet CIDR for cluster nodes | +| `sshKeyName` | string | "" | SSH key to inject into nodes | +| `certSANs` | array | [] | Extra SANs for API server certificate | diff --git a/docs/providers/openstack/quickstart.md b/docs/providers/openstack/quickstart.md deleted file mode 100644 index 00bccef8..00000000 --- a/docs/providers/openstack/quickstart.md +++ /dev/null @@ -1,290 +0,0 @@ -# Quickstart - -This quickstart guide contains steps to install the [Cluster Stack Operator][CSO] (CSO) utilizing the [Cluster Stack Provider OpenStack][CSPO] (CSPO) to provide [ClusterClasses][ClusterClass] which can be used with the [Kubernetes Cluster API][CAPI] to create Kubernetes Clusters. - -This section guides you through all the necessary steps to create a workload Kubernetes cluster on top of the OpenStack infrastructure. The guide describes a path that utilizes the `clusterctl` CLI tool to manage the lifecycle of a CAPI management cluster and employs `kind` to create a local non-production managemnt cluster. - -Note that it is a common practice to create a temporary, local [bootstrap cluster](https://cluster-api.sigs.k8s.io/reference/glossary#bootstrap-cluster) which is then used to provision a target [management cluster](https://cluster-api.sigs.k8s.io/reference/glossary#management-cluster) on the selected infrastructure. - -## Prerequisites - -- Install [Docker](https://docs.docker.com/get-docker/) and [kind](https://helm.sh/docs/intro/install/) -- Install [kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/) -- Install [Helm](https://helm.sh/docs/intro/install/) -- Install [clusterctl](https://cluster-api.sigs.k8s.io/user/quick-start.html#install-clusterctl) -- Install [go](https://go.dev/doc/install) -- Install [jq](https://jqlang.github.io/jq/) - -## Initialize the management cluster - -Create the kind cluster: - -```bash -kind create cluster -``` - -Transform the Kubernetes cluster into a management cluster by using `clusterctl init` and bootstrap it with CAPI and Cluster API Provider OpenStack ([CAPO](https://github.com/kubernetes-sigs/cluster-api-provider-openstack)) components: - -```bash -export CLUSTER_TOPOLOGY=true -export EXP_CLUSTER_RESOURCE_SET=true -export EXP_RUNTIME_SDK=true -clusterctl init --infrastructure openstack -kubectl apply -f https://github.com/k-orc/openstack-resource-controller/releases/latest/download/install.yaml -``` - -Note that the manual deployment of the openstack resource controller (ORC) is required since capo-0.12. If you use `clusterctl upgrade` to upgrade capo from earlier version, you'll also need to manually add ORC to the management host/cluster. - -### CSO and CSPO variables preparation (CSP) - -The CSO and CSPO must be directed to the Cluster Stacks repository housing releases for the OpenStack provider. -Modify and export the following environment variables if you wish to redirect CSO and CSPO to an alternative Git repository - -Be aware that GitHub enforces limitations on the number of API requests per unit of time. To overcome this, -it is recommended to configure a [personal access token](https://github.com/settings/personal-access-tokens/new) for authenticated calls. This will significantly increase the rate limit for GitHub API requests. -Fine grained PAT with `Public Repositories (read-only)` is enough. - -```bash -export GIT_PROVIDER_B64=Z2l0aHVi # github -export GIT_ORG_NAME_B64=U292ZXJlaWduQ2xvdWRTdGFjaw== # SovereignCloudStack -export GIT_REPOSITORY_NAME_B64=Y2x1c3Rlci1zdGFja3M= # cluster-stacks -export GIT_ACCESS_TOKEN_B64=$(echo -n '' | base64 -w0) -``` - -### CSO and CSPO deployment (CSP) - -Install the [envsubst](https://github.com/drone/envsubst) Go package. It is required to enable the expansion of variables specified in CSPO and CSO manifests. - -```bash -GOBIN=/tmp go install github.com/drone/envsubst/v2/cmd/envsubst@latest -``` -Note: On typical Linux distros, you will have a binary `/usr/bin/envsubst` from the gettext package that does *not* work. - -Get the latest CSO release version and apply CSO manifests to the management cluster. - -```bash -# Get the latest CSO release version and apply CSO manifests -curl -sSL https://github.com/SovereignCloudStack/cluster-stack-operator/releases/latest/download/cso-infrastructure-components.yaml | /tmp/envsubst | kubectl apply -f - -``` - -Get the latest CSPO release version and apply CSPO manifests to the management cluster. - -```bash -# Get the latest CSPO release version and apply CSPO manifests -curl -sSL https://github.com/sovereignCloudStack/cluster-stack-provider-openstack/releases/latest/download/cspo-infrastructure-components.yaml | /tmp/envsubst | kubectl apply -f - -``` - -## Define a namespace for a tenant (CSP/per tenant) - -```sh -export CS_NAMESPACE=my-tenant -``` - -### Deploy CSP-helper chart - -The csp-helper chart is meant to create per tenant credentials as well as the tenants namespace where all resources for this tenant will live in. - -Cloud and secret name default to `openstack`. - -Example `clouds.yaml` - -```yaml -clouds: - openstack: - auth: - auth_url: https://api.gx-scs.sovereignit.cloud:5000/v3 - application_credential_id: "" - application_credential_secret: "" - region_name: "RegionOne" - interface: "public" - identity_api_version: 3 - auth_type: "v3applicationcredential" -``` - -```bash -helm upgrade -i csp-helper-"${CS_NAMESPACE}" -n "${CS_NAMESPACE}" --create-namespace https://github.com/SovereignCloudStack/openstack-csp-helper/releases/latest/download/openstack-csp-helper.tgz -f path/to/clouds.yaml -``` - -## Create Cluster Stack definition (CSP/per tenant) - -Configure the Cluster Stack you want to use: - -```sh -# the name of the cluster stack (must match a name of a directory in https://github.com/SovereignCloudStack/cluster-stacks/tree/main/providers/openstack) -export CS_NAME=scs - -# the kubernetes version of the cluster stack (must match a tag for the kubernetes version and the stack version) -export CS_K8S_VERSION=1.29 - -# the version of the cluster stack (must match a tag for the kubernetes version and the stack version) -export CS_VERSION=v1 -export CS_CHANNEL=stable - -# must match a cloud section name in the used clouds.yaml -export CS_CLOUDNAME=openstack -export CS_SECRETNAME="${CS_CLOUDNAME}" -``` - -This will use the cluster-stack as defined in the `providers/openstack/scs` directory. - -```bash -cat >clusterstack.yaml < cluster.yaml < kubeconfig.yaml -# Communicate with the workload cluster -kubectl --kubeconfig kubeconfig.yaml get nodes -``` - -## Check the workload cluster health - -```bash -$ kubectl --kubeconfig kubeconfig.yaml get pods -A -NAMESPACE NAME READY STATUS RESTARTS AGE -kube-system cilium-8mzrx 1/1 Running 0 7m58s -kube-system cilium-jdxqm 1/1 Running 0 6m43s -kube-system cilium-operator-6bb4c7d6b6-c77tn 1/1 Running 0 7m57s -kube-system cilium-operator-6bb4c7d6b6-l2df8 1/1 Running 0 7m58s -kube-system cilium-p9tkv 1/1 Running 0 6m44s -kube-system cilium-thbc8 1/1 Running 0 6m45s -kube-system coredns-5dd5756b68-k68j4 1/1 Running 0 8m3s -kube-system coredns-5dd5756b68-vjg9r 1/1 Running 0 8m3s -kube-system etcd-cs-cluster-pwblg-xkptx 1/1 Running 0 8m3s -kube-system kube-apiserver-cs-cluster-pwblg-xkptx 1/1 Running 0 8m3s -kube-system kube-controller-manager-cs-cluster-pwblg-xkptx 1/1 Running 0 8m3s -kube-system kube-proxy-54f8w 1/1 Running 0 6m44s -kube-system kube-proxy-8z8kb 1/1 Running 0 6m43s -kube-system kube-proxy-jht46 1/1 Running 0 8m3s -kube-system kube-proxy-mt69p 1/1 Running 0 6m45s -kube-system kube-scheduler-cs-cluster-pwblg-xkptx 1/1 Running 0 8m3s -kube-system metrics-server-6578bd6756-vztzf 1/1 Running 0 7m57s -kube-system openstack-cinder-csi-controllerplugin-776696786b-ksf77 6/6 Running 0 7m57s -kube-system openstack-cinder-csi-nodeplugin-96dlg 3/3 Running 0 6m43s -kube-system openstack-cinder-csi-nodeplugin-crhc4 3/3 Running 0 6m44s -kube-system openstack-cinder-csi-nodeplugin-d7rzz 3/3 Running 0 7m58s -kube-system openstack-cinder-csi-nodeplugin-nkgq6 3/3 Running 0 6m44s -kube-system openstack-cloud-controller-manager-hp2n2 1/1 Running 0 7m9s -``` - -[CAPI]: https://cluster-api.sigs.k8s.io/ -[CSO]: https://github.com/sovereignCloudStack/cluster-stack-operator/ -[CSPO]: https://github.com/SovereignCloudStack/cluster-stacks/tree/main/providers/openstack -[ClusterClass]: https://github.com/kubernetes-sigs/cluster-api/blob/main/docs/proposals/20210526-cluster-class-and-managed-topologies.md diff --git a/docs/providers/openstack/scs-configuration.md b/docs/providers/openstack/scs-configuration.md new file mode 100644 index 00000000..91c50abb --- /dev/null +++ b/docs/providers/openstack/scs-configuration.md @@ -0,0 +1,186 @@ +# OpenStack / scs + +Standard SCS cluster stack: dedicated VMs for both control plane and workers. + +## Getting started + +### Prerequisites + +In addition to the [common prerequisites](../../quickstart.md#prerequisites): + +- An OpenStack cloud with application credentials +- A `clouds.yaml` with your credentials + +### Deploy + +```bash +# 1. Management cluster +kind create cluster +clusterctl init --infrastructure openstack + +# 2. Deploy credentials +export CS_NAMESPACE=my-tenant +helm upgrade -i csp-helper-${CS_NAMESPACE} \ + -n ${CS_NAMESPACE} --create-namespace \ + https://github.com/SovereignCloudStack/openstack-csp-helper/releases/latest/download/openstack-csp-helper.tgz \ + -f path/to/clouds.yaml + +# 3. Build, publish, install CSO, and apply the ClusterStack +just dev 1.36 --install-cso | kubectl apply -f - +kubectl get clusterclass -w + +# 4. Create a workload cluster +kubectl create namespace ${CS_NAMESPACE} +just generate-resources 1.36 --cluster-only --namespace ${CS_NAMESPACE} | kubectl apply -f - +``` + +For a production OCI registry instead of ttl.sh: + +```bash +export OCI_REGISTRY=registry.example.com +export OCI_REPOSITORY=kaas/cluster-stacks +just install-cso +just dev 1.36 | kubectl apply -f - +``` + +--- + +## Configuration + +This page lists the custom configuration options available, including their default values and if they are optional. The following example shows how these variables can be used inside the `cluster.yaml` file under `spec.topology.variables`. + +## Version matrix + + +| Version | K8s | CS Version | cilium | metrics-server | os-csi | os-ccm | +|---------|-----|----------|---------|---------|---------|---------| +| 1-32 | 1.32.5 | - | 1.17.4 | 3.12.2 | 2.32.0 | 2.32.0 | +| 1-33 | 1.33.11 | - | 1.19.4 | 3.13.0 | 2.33.x | 2.33.x | +| 1-34 | 1.34.12 | - | 1.19.4 | 3.13.0 | 2.34.x | 2.34.x | +| 1-35 | 1.35.9 | - | 1.20.3 | 3.14.0 | 2.35.x | 2.35.x | +| 1-36 | 1.36.5 | - | 1.20.2 | 3.14.0 | 2.36.x | 2.36.x | + + +## Example + +```yaml +apiVersion: +kind: Cluster +metadata: + name: my-cluster + namespace: my-namespace + labels: + managed-secret: cloud-config +spec: + topology: + variables: # <-- variables from the table below can be set here + - name: + value: + classRef: # cluster.x-k8s.io/v1beta1 clusters (hcp) use "class: " instead + name: + controlPlane: + replicas: 3 + variables: + overrides: + - name: + value: + version: v + workers: + machineDeployments: + - class: default-worker + name: md-0 + replicas: 3 +``` + +`spec.clusterNetwork` is optional. If no pod CIDR is set, the stack assumes +`192.168.0.0/16` as the default. Set pod or service CIDRs only when you need +non-default values or explicit network settings. + +Variables of type `object` are set as nested values. The following example demonstrates the shape: + +```yaml +... +topology: + variables: + - name: oidcConfig + value: + issuerURL: "https://dex.k8s.scs.community" + clientID: "kubectl" +... +``` + +In v1beta2, per-role overrides (e.g. different flavors for control plane and workers) are set via `topology.controlPlane.variables.overrides` and `topology.workers.machineDeployments[].variables.overrides` instead of separate variable names. + +`componentOverrides` is mainly used to set resource requests and limits. The Kubernetes components (`apiServer`, `controllerManager`, `scheduler`, `etcd`, `coreDNS`) accept `resources` only, the addons (`cni`, `ccm`, `csi`, `metricsServer`) accept any values of their Helm chart: + +```yaml +... +topology: + variables: + - name: componentOverrides + value: + apiServer: + resources: + requests: + cpu: "500m" + memory: "1Gi" + metricsServer: + resources: + requests: + cpu: "100m" + memory: "128Mi" +... +``` + +Addon values and `ciliumHostLegacyRouting` are applied when the Cluster Stack Operator installs or upgrades the addons, i.e. on cluster creation and on cluster upgrades. + +## Available variables + +The variables below apply to 1-35 and 1-36. Older versions do not support all of them. + + +|Name|Type|Default|Example|Description|Required| +|----|----|-------|-------|-----------|--------| +|`imageName`|string|"ubuntu-capi-image"||The base name of the OpenStack image used for provisioning servers.
If `imageIsOrc` is enabled, this name refers to an ORC image resource.
If `imageIsOrc` is disabled, the name is used to filter images available in the OpenStack project. In this case, the specified image must already exist within the project.
If `imageAddVersion` is enabled, the Kubernetes version will be appended to form the complete image name (e.g., imageName-v1.32.5)|false| +|`imageIsOrc`|boolean|false||Indicates whether the image name refers to an ORC image resource.
If set to true (default), the `imageName` is interpreted as a reference to an ORC image.
If set to false, the `imageName` is used to filter images in the OpenStack project instead.|false| +|`imageAddVersion`|boolean|true||Add a suffix with the Kubernetes version to the imageName. E.g. imageName-v1.32.5.|false| +|`networkExternalID`|string||"ebfe5546-f09f-4f42-ab54-094e457d42ec"|ID of an external OpenStack network. Required when multiple
external networks exist and VMs need public internet access.|false| +|`networkMTU`|integer||1500|MTU for the private cluster network. Set this to avoid fragmentation issues.|false| +|`dnsNameservers`|array|["9.9.9.9", "149.112.112.112"]|["9.9.9.9", "149.112.112.112"]|DNS nameservers for the cluster subnet. Only used when a new network/subnet is created.|false| +|`nodeCIDR`|string|"10.8.0.0/20"|"10.8.0.0/20"|CIDR for the cluster subnet. CAPO will create a network, subnet,
and router. Leave empty to skip network creation.|false| +|`flavor`|string|"SCS-2V-4"|"SCS-2V-4"|OpenStack instance flavor. Applies to all nodes by default.
Override per control plane or worker via topology variables overrides.|false| +|`rootDisk`|integer|50|50|Root disk size in GiB. OpenStack volume will be created and used
instead of an ephemeral disk defined in the flavor.
Set to 0 to use the flavor's ephemeral disk.|false| +|`diskPinAZ`|boolean|false|true|Create root and additional volumes in the availability zone (failure domain)
of the machine. Needed on clouds where volumes can only be attached in the
same availability zone.|false| +|`serverGroupID`|string|""|"3adf4e92-bb33-4e44-8ad3-afda9dfe8ec3"|Server group for anti-affinity placement. Override per CP/worker via topology.|false| +|`additionalBlockDevices`|array|[]||Additional block devices (Cinder volumes) to attach to nodes.
Override per CP/worker via topology.|false| +|`controlPlaneAvailabilityZones`|array|[]|["nova"]|Availability zones for control plane nodes (OpenStack cluster-level setting).|false| +|`controlPlaneOmitAvailabilityZone`|boolean|false||Omit availability zone when creating control plane nodes, letting the
Nova scheduler decide based on other scheduling constraints.|false| +|`sshKeyName`|string|""|"capi-keypair"|SSH key to inject into all nodes (for debugging).|false| +|`securityGroups`|array|[]|["security-group-1"]|Extra security groups by name for all nodes.
Ignored if securityGroupIDs is set. Override per CP/worker via topology.|false| +|`securityGroupIDs`|array|[]|["9ae2f488-30a3-4629-bd51-07acb8eb4278"]|Extra security groups by UUID for all nodes.
Takes precedence over securityGroups. Override per CP/worker via topology.|false| +|`identityRef.name`|string|"openstack"|"openstack"|The name of the secret that carries the OpenStack clouds.yaml|false| +|`identityRef.cloudName`|string|"openstack"|"openstack"|The name of the cloud to use from the clouds.yaml|false| +|`disableAPIServerFloatingIP`|boolean|false||Disable the floating IP on the API server load balancer.|false| +|`certSANs`|array|[]|["mydomain.example"]|Extra Subject Alternative Names for the API server TLS certificate.|false| +|`serviceLoadBalancer`|string|"octavia-ovn"|"octavia-ovn"|Load balancer provider that the cloud controller manager uses for Services of type
LoadBalancer in the workload cluster (e.g. in front of an ingress controller).
Options: octavia-amphora, octavia-ovn (default).
Changes are applied to the cloud controller manager with the next cluster upgrade.|false| +|`apiServerLoadBalancer`|string|"octavia-ovn"|"octavia-ovn"|Load balancer in front of the API server.
Options: none, octavia-amphora, octavia-ovn (default).|false| +|`apiServerAllowedCIDRs`|array||["192.168.10.0/24"]|Restrict access to the API server to these CIDRs (network-level ACL).
Requires amphora as load balancer provider (CAPO >= v2.12).
Ensure the management cluster's outgoing IP is included,
otherwise CAPO cannot reconcile the workload cluster.|false| +|`auditLogging`|boolean|true||Write API server audit logs to /var/log/kubernetes/audit on the control plane nodes.|false| +|`oidcConfig.clientID`|string||"kubectl"|A client id that all tokens must be issued for.|false| +|`oidcConfig.issuerURL`|string||"https://dex.k8s.scs.community"|URL of the provider that allows the API server to discover public signing keys. Only URLs that use the https:// scheme are accepted. This is typically the provider's discovery URL, changed to have an empty path.|false| +|`oidcConfig.usernameClaim`|string|"preferred_username"|"preferred_username"|JWT claim to use as the user name. By default sub, which is expected to be a unique identifier of the end user. Admins can choose other claims, such as email or name, depending on their provider. However, claims other than email will be prefixed with the issuer URL to prevent naming clashes with other plugins.|false| +|`oidcConfig.groupsClaim`|string|"groups"|"groups"|JWT claim to use as the user's group. If the claim is present it must be an array of strings.|false| +|`oidcConfig.usernamePrefix`|string|"oidc:"|"oidc:"|Prefix prepended to username claims to prevent clashes with existing names (such as system: users). For example, the value oidc: will create usernames like oidc:jane.doe. If this flag isn't provided and --oidc-username-claim is a value other than email the prefix defaults to ( Issuer URL )# where ( Issuer URL ) is the value of --oidc-issuer-url. The value - can be used to disable all prefixing.|false| +|`oidcConfig.groupsPrefix`|string|"oidc:"|"oidc:"|Prefix prepended to group claims to prevent clashes with existing names (such as system: groups). For example, the value oidc: will create group names like oidc:engineering and oidc:infra.|false| +|`registryMirrors`|array|[]||Registry mirrors for upstream container registries. Configures both containerd and CRI-O to pull through a mirror.|false| +|`ciliumHostLegacyRouting`|boolean|false||Route pod traffic through the host network stack (Cilium bpf.hostLegacyRouting).|false| +|`componentOverrides.apiServer`|object|||kube-apiserver|false| +|`componentOverrides.controllerManager`|object|||kube-controller-manager|false| +|`componentOverrides.scheduler`|object|||kube-scheduler|false| +|`componentOverrides.etcd`|object|||etcd|false| +|`componentOverrides.coreDNS`|object|||CoreDNS|false| +|`componentOverrides.cni`|object|||Values for the cilium Helm chart.|false| +|`componentOverrides.ccm`|object|||Values for the openstack-cloud-controller-manager Helm chart.|false| +|`componentOverrides.csi`|object|||Values for the openstack-cinder-csi Helm chart.|false| +|`componentOverrides.metricsServer`|object|||Values for the metrics-server Helm chart.|false| + diff --git a/docs/providers/openstack/scs2-configuration.md b/docs/providers/openstack/scs2-configuration.md new file mode 100644 index 00000000..3a686e41 --- /dev/null +++ b/docs/providers/openstack/scs2-configuration.md @@ -0,0 +1,98 @@ +# Configuration of the `scs2` Cluster Stack (deprecated) + +> **Deprecated:** `scs2` is only maintained and will be phased out. Use the [`scs` cluster stack](scs-configuration.md) for new clusters. + +This page lists the custom configuration options available, including their default values and if they are optional. The following example shows how these variables can be used inside the `cluster.yaml` file under `spec.topology.variables`. + +## Example + +```yaml +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + name: + namespace: + labels: + managed-secret: cloud-config +spec: + clusterNetwork: + pods: + cidrBlocks: + - 192.168.0.0/16 + serviceDomain: cluster.local + services: + cidrBlocks: + - 10.96.0.0/12 + topology: + variables: // <-- variables from the table can be set here + - name: controller_flavor + value: "SCS-4V-8-20" + - name: worker_flavor + value: "SCS-4V-8-20" + - name: external_id + value: "ebfe5546-f09f-4f42-ab54-094e457d42ec" + class: openstack-alpha-1-29-v2 + controlPlane: + replicas: 2 + version: v1.29.3 + workers: + machineDeployments: + - class: openstack-alpha-1-29-v2 + failureDomain: nova + name: openstack-alpha-1-29-v2 + replicas: 4 +``` + +Variables from the table containing a `.` are to be used in an object with the part before the dot being the object name and the part behind the dot being the value names. The following example demonstrates this with `oidc_config`. + +```yaml +--- +topology: + variables: + - name: oidc_config + value: + issuer_url: "https://dex.k8s.scs.community" + client_id: "kubectl" +``` + +## Available variables + + + +|Name|Type|Default|Example|Description|Required| +|----|----|-------|-------|-----------|--------| +|`imageName`|string|"ubuntu-capi-image"||The base name of the OpenStack image used for provisioning servers.
If `imageIsOrc` is enabled, this name refers to an ORC image resource.
If `imageIsOrc` is disabled, the name is used to filter images available in the OpenStack project. In this case, the specified image must already exist within the project.
If `imageAddVersion` is enabled, the Kubernetes version will be appended to form the complete image name (e.g., imageName-v1.32.5)|false| +|`imageIsOrc`|boolean|false||Indicates whether the image name refers to an ORC image resource.
If set to true (default), the `imageName` is interpreted as a reference to an ORC image.
If set to false, the `imageName` is used to filter images in the OpenStack project instead.|false| +|`imageAddVersion`|boolean|true||Add a suffix with the Kubernetes version to the imageName. E.g. imageName-v1.32.5.|false| +|`disableAPIServerFloatingIP`|boolean|false|false|DisableAPIServerFloatingIP controls whether a floating IP should be attached to the API server.|false| +|`networkExternalID`|string||"ebfe5546-f09f-4f42-ab54-094e457d42ec"|networkExternalID is the ID of an external OpenStack Network. This is necessary to get public internet to the VMs in case there are several external networks.|false| +|`networkMTU`|integer||1500|networkMTU sets the maximum transmission unit (MTU) value to address fragmentation for the private network ID.|false| +|`dnsNameservers`|array|["9.9.9.9", "149.112.112.112"]|["9.9.9.9", "149.112.112.112"]|dnsNameservers is the list of nameservers for the OpenStack Subnet being created. Set this value when you need to create a new network/subnet which requires access to DNS.|false| +|`nodeCIDR`|string|"10.8.0.0/20"|"10.8.0.0/20"|nodeCIDR is the OpenStack Subnet to be created.
Cluster actuator will create a network, a subnet with nodeCIDR,
and a router connected to this subnet.
If you leave this empty, no network will be created.|false| +|`diskPinAZ`|boolean|false|true|Pass availabilityZone (failureDomain) of the machine to its (root) disk volumes.
Some clouds have a 1:1 relationship of storage availability zones with compute AZs.
In these, attaching a disk from another AZ often comes with a performance penalty
or is not possible at all. You can set this to true to tell capo to create the root
disk in the stated failureDomain of this machine. This setting applies to controlPlane
root disks (if they use root volumes), worker machine root disks (if they use root
volumes) and workerAdditionalBlockDevices (if any are configured).|false| +|`controlPlaneFlavor`|string|"SCS-2V-4"|"SCS-2V-4-20s"|OpenStack instance flavor for control plane nodes. (The default, SCS-2V-4, is not ideal;
recommendation is to replace this by SCS-2V-4-20s and set controlPlaneRootDisk to 0
or specify a non-zero controlPlaneRootDisk with a diskless flavor.)|false| +|`controlPlaneRootDisk`|integer|50|25|Root disk size in GiB for control-plane nodes.
OpenStack volume will be created and used instead of an ephemeral disk defined in flavor.
Should only be used for the diskless flavors (and then be set >= 20), otherwise set to 0.|false| +|`controlPlaneServerGroupID`|string|""|"3adf4e92-bb33-4e44-8ad3-afda9dfe8ec3"|The server group to assign the control plane nodes to (can be used for anti-affinity).|false| +|`controlPlaneAvailabilityZones`|array||["nova"]|controlPlaneAvailabilityZones is the set of availability zones which control plane machines may be deployed to.|false| +|`controlPlaneOmitAvailabilityZone`|boolean||true|controlPlaneOmitAvailabilityZone causes availability zone to be omitted when creating control plane nodes,
allowing the Nova scheduler to make a decision on which availability zone to use based on other scheduling constraints.|false| +|`workerFlavor`|string|"SCS-4V-8"|"SCS-4V-8"|OpenStack instance flavor for worker nodes (default: SCS-4V-8, which requires workerRootDisk).|false| +|`workerRootDisk`|integer|50|25|Root disk size in GiB for worker nodes.
OpenStack volume will be created and used instead of an ephemeral disk defined in flavor.
Should be used for the diskless flavors (>= 20), otherwise set to 0.|false| +|`workerServerGroupID`|string|""|"869fe071-1e56-46a9-9166-47c9f228e297"|The server group to assign the worker nodes to.|false| +|`workerAdditionalBlockDevices`|array|[]||You may pass an array with additional volumes (disks) here, consisting of
name, sizeGiB (default: 20) and type (aka storage class, default: __DEFAULT__).|false| +|`sshKeyName`|string|""|"capi-keypair"|The ssh key name to inject in the nodes (for debugging).|false| +|`securityGroups`|array|[]|["security-group-1"]|The names of extra security groups to assign to worker and control plane nodes.
Will be ignored if `securityGroupIDs` is used.|false| +|`securityGroupIDs`|array|[]|["9ae2f488-30a3-4629-bd51-07acb8eb4278"]|The UUIDs of extra security groups to assign to worker and control plane nodes|false| +|`workerSecurityGroups`|array|[]|["security-group-1"]|The names of extra security groups to assign to the worker nodes.
Will be ignored if `workerSecurityGroupIDs` is used.|false| +|`workerSecurityGroupIDs`|array|[]|["9ae2f488-30a3-4629-bd51-07acb8eb4278"]|The UUIDs of extra security groups to assign to the worker nodes|false| +|`identityRef.name`|string|"openstack"|"openstack"|The name of the secret that carries the OpenStack clouds.yaml|false| +|`identityRef.cloudName`|string|"openstack"|"openstack"|The name of the cloud to use from the clouds.yaml|false| +|`certSANs`|array|[]|["mydomain.example"]|certSANs sets extra Subject Alternative Names for the API Server signing cert.|false| +|`apiServerLoadBalancer`|string|"octavia-ovn"|"none, octavia-amphora, octavia-ovn"|Cluster-API by default places a LoadBalancer in front of the kubernetes API server.
(There are also LBs that the CCM creates for a service type LoadBalancer which are configured independently.)
This setting here is to configure the LoadBalancer that is placed in front of the apiServer.
You can choose from 3 options:

none:
No LoadBalancer solution will be deployed

octavia-amphora:
Uses OpenStack's LoadBalancer service Octavia (provider:amphora)

octavia-ovn:
(default) Uses OpenStack's LoadBalancer service Octavia (provider:ovn)|false| +|`apiServerLoadBalancerOctaviaAmphoraAllowedCIDRs`|array||["192.168.10.0/24"]|apiServerLoadBalancerOctaviaAmphoraAllowedCIDRs restrict access to the Kubernetes API server on a network level.
Ensure that at least the outgoing IP of your Management Cluster is added to the list of allowed CIDRs.
Otherwise CAPO can’t reconcile the target Cluster correctly.
This requires amphora as load balancer provider in version >= v2.12.|false| +|`oidcConfig.clientID`|string||"kubectl"|A client id that all tokens must be issued for.|false| +|`oidcConfig.issuerURL`|string||"https://dex.k8s.scs.community"|URL of the provider that allows the API server to discover public signing keys. Only URLs that use the https:// scheme are accepted. This is typically the provider's discovery URL, changed to have an empty path.|false| +|`oidcConfig.usernameClaim`|string|"preferred_username"|"preferred_username"|JWT claim to use as the user name. By default sub, which is expected to be a unique identifier of the end user. Admins can choose other claims, such as email or name, depending on their provider. However, claims other than email will be prefixed with the issuer URL to prevent naming clashes with other plugins.|false| +|`oidcConfig.groupsClaim`|string|"groups"|"groups"|JWT claim to use as the user's group. If the claim is present it must be an array of strings.|false| +|`oidcConfig.usernamePrefix`|string|"oidc:"|"oidc:"|Prefix prepended to username claims to prevent clashes with existing names (such as system: users). For example, the value oidc: will create usernames like oidc:jane.doe. If this flag isn't provided and --oidc-username-claim is a value other than email the prefix defaults to ( Issuer URL )# where ( Issuer URL ) is the value of --oidc-issuer-url. The value - can be used to disable all prefixing.|false| +|`oidcConfig.groupsPrefix`|string|"oidc:"|"oidc:"|Prefix prepended to group claims to prevent clashes with existing names (such as system: groups). For example, the value oidc: will create group names like oidc:engineering and oidc:infra.|false| + diff --git a/docs/quickstart.md b/docs/quickstart.md new file mode 100644 index 00000000..9aaf5d6d --- /dev/null +++ b/docs/quickstart.md @@ -0,0 +1,110 @@ +# Quickstart + +Get a workload cluster running locally in a few minutes using the CAPI Docker +provider (CAPD). No cloud account required. + +For OpenStack, see [providers/openstack/scs-configuration.md](providers/openstack/scs-configuration.md). +For Hosted Control Plane, see [providers/openstack/hcp.md](providers/openstack/hcp.md). + +## Prerequisites + +Install these tools before starting: + +| Tool | Purpose | +|------|---------| +| [Docker](https://docs.docker.com/get-docker/) + [kind](https://kind.sigs.k8s.io/) | Local Kubernetes clusters | +| [kubectl](https://kubernetes.io/docs/tasks/tools/) | Cluster access | +| [clusterctl](https://cluster-api.sigs.k8s.io/user/quick-start.html#install-clusterctl) | CAPI bootstrap | +| [Helm](https://helm.sh/docs/intro/install/) | Chart installation | +| [just](https://github.com/casey/just) | Build recipes | +| [yq](https://github.com/mikefarah/yq) (mikefarah variant) | YAML processing | +| [oras](https://oras.land/docs/installation) | OCI artifact push | + +## Steps + +### 1. Create a management cluster + +```bash +kind create cluster +clusterctl init --infrastructure docker +``` + +### 2. Point the build system at the Docker stack + +Set the target provider and stack: + +```bash +export PROVIDER=docker +export CLUSTER_STACK=scs +``` + +Or create a `.env` file if you use [direnv](https://direnv.net/): + +```bash +cat > .env < Ready +``` + +### 4. Create a workload cluster + +```bash +just generate-resources 1.35 --cluster-only | kubectl apply -f - +``` + +Watch it come up: + +```bash +clusterctl -n cluster describe cluster cs-cluster +``` + +### 5. Access the workload cluster + +```bash +clusterctl -n cluster get kubeconfig cs-cluster > kubeconfig.yaml +kubectl --kubeconfig kubeconfig.yaml get nodes +``` + +## What was created + +``` +.release/ + docker-scs-1-35-/ + csctl.yaml # stack metadata + clusteraddon.yaml # addon lifecycle config + docker-scs-1-35-cluster-class-.tgz + docker-scs-1-35-cluster-addon-.tgz + metadata.yaml +``` + +The Cluster Stack Operator (CSO) pulled this artifact from ttl.sh and installed +the `ClusterClass` into the management cluster. Your `Cluster` resource +references that class. + +## Useful commands + +```bash +just matrix # show K8s versions and addon versions +just clean # remove .release/ build artifacts +just update versions # update K8s patch versions in csctl.yaml +just update addons # update addon chart versions +``` diff --git a/docs/versioning.md b/docs/versioning.md new file mode 100644 index 00000000..b17ff598 --- /dev/null +++ b/docs/versioning.md @@ -0,0 +1,47 @@ +# Versioning + +## Per Kubernetes minor version + +Every Kubernetes minor version of a stack lives in its own, self-contained directory +(`providers///1-XX/`). There is no inheritance or sharing between minor versions. +This allows different ClusterClass API versions (CAPI `v1beta1` and `v1beta2`) and different addon versions per minor version. + +> **Requirements:** The ClusterClasses of `openstack/scs` and `docker/scs` use `cluster.x-k8s.io/v1beta2`. They require +> Cluster API >= 1.11 in the management cluster and a Cluster Stack Operator that supports it +> ([cluster-stack-operator#346](https://github.com/SovereignCloudStack/cluster-stack-operator/pull/346)). +> `openstack/scs2` and `openstack/hcp` still use `v1beta1`. +> +> **Kubernetes 1.36** additionally needs Cluster API **>= 1.12.8** (1.12 line) or **>= 1.13.1**. Older Cluster API versions +> set the kubeadm feature gate `ControlPlaneKubeletLocalMode` for every control plane, which was removed in Kubernetes +> 1.36. `kubeadm init` then fails (`featureGates: ... ControlPlaneKubeletLocalMode is not a valid feature name`), the +> control plane never becomes ready and the machine health check replaces the server again and again. + +The name of a ClusterClass / release contains the Kubernetes minor version and the version of the cluster stack: + +``` +openstack-scs-1-35-v3 + | | | | + | | | +-- cluster stack version, incremented for every release + | | +------- Kubernetes minor version (1.35) + | +----------- cluster stack + +------------------- provider +``` + +## Breaking changes + +The Cluster Stack Operator does not allow to change the ClusterStack of an existing cluster. Clusters can only be +upgraded to newer versions of **the same** ClusterStack. A new cluster stack (e.g. `scs3`) is therefore only created if two +variants are supposed to be maintained in parallel. + +Changes of the configuration (new or changed variables, different defaults) that would break existing clusters are +introduced with a **new Kubernetes minor version** of the stack. Within a minor version only patch releases of Kubernetes, +addon updates and compatible changes are released. + +## Cluster stacks + +| Cluster stack | Status | Notes | +|---|---|---| +| `openstack/scs` | maintained | Unified variables (`flavor`, `rootDisk`, ...), the stack to use. `1-32` is the stack as it was released before (`v1beta1`, variables like `cloud_name` and `ssh_key`), it is frozen and not changed anymore, the new variables start with `1-33` | +| `openstack/hcp` | maintained | Hosted control plane, only worker nodes are created as OpenStack servers | +| `docker/scs` | maintained | For development and testing with the Docker provider | +| `openstack/scs2` | deprecated | Separate `controlPlane*` / `worker*` variables. Only maintained, will be phased out. Migrate with `hack/migrate-cluster.sh` | diff --git a/flake.lock b/flake.lock new file mode 100644 index 00000000..6714924d --- /dev/null +++ b/flake.lock @@ -0,0 +1,61 @@ +{ + "nodes": { + "flake-utils": { + "inputs": { + "systems": "systems" + }, + "locked": { + "lastModified": 1731533236, + "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1768886240, + "narHash": "sha256-C2TjvwYZ2VDxYWeqvvJ5XPPp6U7H66zeJlRaErJKoEM=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "80e4adbcf8992d3fd27ad4964fbb84907f9478b0", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "flake-utils": "flake-utils", + "nixpkgs": "nixpkgs" + } + }, + "systems": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 00000000..45d1b2bb --- /dev/null +++ b/flake.nix @@ -0,0 +1,89 @@ +{ + description = "Cluster Stacks - Build tools for SCS Kubernetes cluster stacks"; + + inputs = { + nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable"; + flake-utils.url = "github:numtide/flake-utils"; + }; + + outputs = { self, nixpkgs, flake-utils }: + flake-utils.lib.eachDefaultSystem (system: + let + pkgs = nixpkgs.legacyPackages.${system}; + in + { + devShells.default = pkgs.mkShell { + name = "cluster-stacks-dev"; + + buildInputs = with pkgs; [ + # Core tools + bash + git + curl + + # Container tools + docker + podman + + # Kubernetes tools + kubectl + kubernetes-helm + kind + kustomize + + # Build tools + just + python3 + python3Packages.pyyaml + yamllint + jq + yq-go + + # OCI/Registry tools + oras + ]; + + shellHook = '' + echo "Cluster Stacks development environment" + echo "" + echo "Available tools:" + echo " just - Run 'just' to see available recipes" + echo " helm - Kubernetes package manager" + echo " kubectl - Kubernetes CLI" + echo " kind - Local Kubernetes clusters" + echo " yq - YAML processor" + echo " oras - OCI registry client" + echo " python3 - With PyYAML" + echo " yamllint - YAML linter (as in CI)" + echo "" + + # Generate shell completions into a cache directory + comp_dir="''${XDG_CACHE_HOME:-$HOME/.cache}/cluster-stacks/completions" + mkdir -p "$comp_dir" + + user_shell=$(getent passwd "$(whoami)" 2>/dev/null | cut -d: -f7) + shell_name=$(basename "''${user_shell:-bash}") + + # Regenerate completions if the directory is empty or tools were updated + if [ ! -f "$comp_dir/.$shell_name-generated" ]; then + kubectl completion "$shell_name" > "$comp_dir/_kubectl" 2>/dev/null || true + helm completion "$shell_name" > "$comp_dir/_helm" 2>/dev/null || true + kind completion "$shell_name" > "$comp_dir/_kind" 2>/dev/null || true + oras completion "$shell_name" > "$comp_dir/_oras" 2>/dev/null || true + touch "$comp_dir/.$shell_name-generated" + fi + + # Make completions available + export FPATH="$comp_dir:$FPATH" + + # Start user's preferred shell instead of bash. + # nix develop always drops into bash; this detects the user's + # real login shell from /etc/passwd and exec's into it. + if [ -n "$user_shell" ] && [ "$shell_name" != "bash" ] && [ -x "$user_shell" ]; then + exec "$user_shell" + fi + ''; + }; + } + ); +} diff --git a/hack/add-version.sh b/hack/add-version.sh new file mode 100755 index 00000000..feebe6d6 --- /dev/null +++ b/hack/add-version.sh @@ -0,0 +1,132 @@ +#!/usr/bin/env bash +# Scaffold a new Kubernetes minor version for all cluster stacks. +# +# For every stack base directory (providers///) the highest existing +# 1-XX/ directory is copied to 1-/ and the version references are adjusted: +# - csctl.yaml: kubernetesVersion and the ccm/csi addon constraints (2.XX.x) +# - cluster-class/Chart.yaml: chart name and description +# - cluster-class/values.yaml: node image tags (Docker stacks, the final tag is set by build.sh) +# - cluster-addon/{ccm,csi}/Chart.yaml: 2..0 (K8s-tied addons) +# +# Stacks with a DEPRECATED file in their base directory are skipped, as well as stacks +# that already have the new version. +# +# Afterwards `hack/update.sh versions --all` resolves the latest Kubernetes patch +# versions (needs network access). Run `hack/update.sh addons` to pick the latest +# matching ccm/csi chart versions. +# +# Usage: +# ./hack/add-version.sh [--no-update] +# +# Example: +# ./hack/add-version.sh 1.37 + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" + +usage() { + echo "Usage: $0 [--no-update]" >&2 + exit 1 +} + +NEW_SHORT="" +RUN_UPDATE=true + +while [[ $# -gt 0 ]]; do + case "$1" in + --no-update) RUN_UPDATE=false; shift ;; + -h|--help) usage ;; + -*) echo "Unknown option: $1" >&2; usage ;; + *) + if [[ -n "$NEW_SHORT" ]]; then + echo "Unexpected argument: $1" >&2 + usage + fi + NEW_SHORT="$1"; shift + ;; + esac +done + +if [[ ! "$NEW_SHORT" =~ ^1\.[0-9]+$ ]]; then + echo "Invalid or missing Kubernetes minor version: '${NEW_SHORT}' (expected e.g. 1.37)" >&2 + usage +fi + +cd "$REPO_ROOT" + +new_dash="${NEW_SHORT//./-}" +new_minor="${NEW_SHORT#1.}" +found=0 + +for stack_base in providers/*/*/; do + stack_base="${stack_base%/}" + [[ -d "$stack_base" ]] || continue + + if [[ -f "$stack_base/DEPRECATED" ]]; then + echo " SKIP $stack_base (deprecated)" >&2 + continue + fi + + latest=$(find "$stack_base" -mindepth 1 -maxdepth 1 -type d -name '1-*' | sort -V | tail -1) + [[ -n "$latest" ]] || continue + + dst="$stack_base/$new_dash" + if [[ -d "$dst" ]]; then + echo " SKIP $dst (already exists)" >&2 + continue + fi + + old_dash="$(basename "$latest")" + old_minor="${old_dash#1-}" + if (( new_minor <= old_minor )); then + echo " SKIP $stack_base (latest version $old_dash is not older than $new_dash)" >&2 + continue + fi + + found=1 + echo "---" >&2 + echo "Creating $dst from $latest ..." >&2 + cp -r "$latest" "$dst" + + csctl="$dst/csctl.yaml" + if [[ -f "$csctl" ]]; then + yq -i ".config.kubernetesVersion = \"v${NEW_SHORT}\"" "$csctl" + sed -i "s/2\.${old_minor}\.x/2.${new_minor}.x/g" "$csctl" + fi + + chart="$dst/cluster-class/Chart.yaml" + if [[ -f "$chart" ]]; then + sed -i "s/${old_dash}/${new_dash}/g; s/1\.${old_minor}\b/1.${new_minor}/g" "$chart" + fi + + values="$dst/cluster-class/values.yaml" + if [[ -f "$values" ]] && yq -e '.images' "$values" >/dev/null 2>&1; then + # build.sh sets the final tag from csctl.yaml, this only keeps the file consistent + sed -i "s|:v1\.${old_minor}\.[0-9]*|:v1.${new_minor}.0|g" "$values" + fi + + for addon in ccm csi; do + addon_chart="$dst/cluster-addon/$addon/Chart.yaml" + [[ -f "$addon_chart" ]] || continue + sed -i -E "s/version: 2\.${old_minor}\.[0-9]+/version: 2.${new_minor}.0/" "$addon_chart" + done + + echo " Done: $dst" >&2 +done + +if [[ "$found" -eq 0 ]]; then + echo "No stacks found to scaffold." >&2 + exit 1 +fi + +if [[ "$RUN_UPDATE" == "true" ]]; then + echo "" >&2 + echo "=== Resolving latest patches ===" >&2 + ./hack/update.sh versions --all +fi + +echo "" >&2 +echo "New version ${NEW_SHORT} scaffolded for all maintained stacks." >&2 +echo "Next: run 'hack/update.sh addons' (or 'just update addons') to pick the latest ccm/csi chart versions." >&2 diff --git a/hack/build.sh b/hack/build.sh new file mode 100755 index 00000000..92fd0182 --- /dev/null +++ b/hack/build.sh @@ -0,0 +1,821 @@ +#!/usr/bin/env bash +# Build and optionally publish cluster-stack release artifacts. +# +# Usage: +# ./hack/build.sh [stack-dir] [options] +# +# The is the base directory containing per-minor-version subdirs +# (e.g., providers/openstack/scs). If omitted, it is derived from $PROVIDER +# and $CLUSTER_STACK (default: providers/openstack/scs). Relative paths are +# resolved from the repository root. +# +# Options: +# --version Build for a specific K8s minor version (e.g., 1.34) +# --release-version Use a specific cluster-stack release version with a single --version build (e.g., v1) +# --all Build for all K8s versions (all 1-* subdirs); mutually exclusive with --version +# --publish Push to OCI registry after building (skipped if unchanged) +# --install-cso Install/upgrade the Cluster Stack Operator with matching OCI config +# --validate Validate addon bundle structure against clusteraddon.yaml +# -h, --help Show this help +# +# Without --version or --all, builds the highest 1-* subdirectory. +# +# Each 1-XX/ subdir must contain a csctl.yaml with at minimum: +# config: +# provider: +# type: openstack +# clusterStackName: scs +# kubernetesVersion: v1.34.3 # minor-only (v1.34) or with patch (v1.34.3) +# +# Addon versions are read directly from cluster-addon/*/Chart.yaml as +# maintained by `just update addons`. The build does not resolve or +# modify addon versions. +# +# Environment: +# PROVIDER Provider name (default: openstack) +# CLUSTER_STACK Cluster stack name (default: scs) +# OCI_REGISTRY OCI registry (default: ttl.sh) +# OCI_REPOSITORY OCI repository (auto-generated for ttl.sh) +# OCI_USERNAME OCI auth username (optional) +# OCI_PASSWORD OCI auth password (optional) +# OCI_ACCESS_TOKEN OCI auth token (optional, alternative to user/pass) +# OUTPUT_DIR Output directory (default: .release) +# CSO_CHART CSO Helm chart reference (default: oci://registry.scs.community/cluster-stacks/cso), +# can also be a local path, e.g. ../cluster-stack-operator/charts +# CSO_IMAGE_REPOSITORY, CSO_IMAGE_TAG +# Override the CSO image (e.g. to test a locally built operator) + +set -euo pipefail +shopt -s inherit_errexit + +readonly SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +readonly REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" +readonly YQ_VERSION="$(yq --version 2>/dev/null || true)" + +require_command() { + local name="$1" + + if ! command -v "$name" >/dev/null 2>&1; then + echo "$name not found. Please install $name and try again." >&2 + exit 1 + fi +} + +log() { + echo "$@" >&2 +} + +warn() { + echo "Warning: $*" >&2 +} + +extract_k8s_minor_version() { + echo "$1" | sed -E -n 's/^([0-9]+\.[0-9]+)(\.[0-9]+)?$/\1/p' +} + +extract_latest_release_number() { + local prefix="$1" + + awk -v prefix="${prefix}-v" 'index($0, prefix) == 1 { + suffix = substr($0, length(prefix) + 1) + if (suffix ~ /^[0-9]+$/) { + print suffix + } + }' | sort -n | tail -1 +} + +normalize_release_version() { + local version="$1" + + if [[ "$version" =~ ^[0-9] ]]; then + version="v${version}" + fi + + if [[ ! "$version" =~ ^v[0-9]+(-[A-Za-z0-9_]+\.[A-Za-z0-9_]+)?$ ]]; then + echo "Invalid release version: ${version}. Expected v or v-., e.g. v1 or v0-sha.abc1234." >&2 + exit 1 + fi + + echo "$version" +} + +oras_auth_opts() { + if [[ -n "${OCI_USERNAME:-}" && -n "${OCI_PASSWORD:-}" ]]; then + printf '%s\n' --username "$OCI_USERNAME" --password "$OCI_PASSWORD" + elif [[ -n "${OCI_ACCESS_TOKEN:-}" ]]; then + printf '%s\n' --password "$OCI_ACCESS_TOKEN" + fi +} + +yq_edit_in_place() { + local expression="$1" + local file="$2" + + if [[ "$YQ_VERSION" == *"https://github.com/mikefarah/yq/"* ]]; then + yq -i "$expression" "$file" + else + yq -y -i "$expression" "$file" + fi +} + +yq_has_path() { + local expression="$1" + local file="$2" + + yq -e "$expression" "$file" >/dev/null 2>&1 +} + +# ============================================ +# Argument parsing +# ============================================ + +usage() { + sed -n '3,/^$/s/^# \?//p' "$0" + exit "${1:-1}" +} + +parse_args() { + while [[ $# -gt 0 ]]; do + case "$1" in + --version) [[ $# -ge 2 ]] || { echo "Error: $1 requires a value" >&2; usage; }; TARGET_VERSION="$2"; shift 2 ;; + --release-version) [[ $# -ge 2 ]] || { echo "Error: $1 requires a value" >&2; usage; }; RELEASE_VERSION=$(normalize_release_version "$2"); shift 2 ;; + --all) BUILD_ALL=true; shift ;; + --publish) PUBLISH=true; shift ;; + --install-cso) INSTALL_CSO=true; shift ;; + --validate) VALIDATE=true; shift ;; + -h|--help) usage 0 ;; + -*) echo "Unknown option: $1" >&2; usage ;; + *) + if [[ -z "$BASE_DIR" ]]; then + BASE_DIR="$1"; shift + else + echo "Unexpected argument: $1" >&2; usage + fi + ;; + esac + done + + if [[ "$BUILD_ALL" == "true" && -n "$TARGET_VERSION" ]]; then + echo "Error: --all and --version cannot be used together." >&2 + usage + fi + + if [[ -n "$RELEASE_VERSION" && -z "$TARGET_VERSION" ]]; then + echo "Error: --release-version can only be used together with --version ." >&2 + usage + fi + + if [[ -n "$RELEASE_VERSION" && "$BUILD_ALL" == "true" ]]; then + echo "Error: --release-version cannot be used with --all." >&2 + usage + fi +} + +resolve_base_dir() { + local dir="$1" + + if [[ -z "$dir" ]]; then + dir="providers/${PROVIDER:-openstack}/${CLUSTER_STACK:-scs}" + fi + + if [[ ! "$dir" = /* ]]; then + dir="$REPO_ROOT/$dir" + fi + + echo "$dir" +} + +BASE_DIR="" +TARGET_VERSION="" +RELEASE_VERSION="" +BUILD_ALL=false +PUBLISH=false +INSTALL_CSO=false +VALIDATE=false + +parse_args "$@" + +require_command yq +require_command curl +require_command helm + +BASE_DIR="$(resolve_base_dir "$BASE_DIR")" + +if [[ ! -d "$BASE_DIR" ]]; then + echo "Stack base directory not found: $BASE_DIR" >&2 + exit 1 +fi + +OUTPUT_DIR="${OUTPUT_DIR:-$REPO_ROOT/.release}" +mkdir -p "$OUTPUT_DIR" +OUTPUT_DIR="$(cd "$OUTPUT_DIR" && pwd)" + +# ============================================ +# Resolve which minor-version directories to build +# ============================================ + +collect_version_dirs() { + if [[ "$BUILD_ALL" == "true" ]]; then + # All 1-* subdirectories, sorted + for d in "$BASE_DIR"/1-*/; do + [[ -d "$d" ]] && echo "$d" + done | sort -t- -k2 -n + elif [[ -n "$TARGET_VERSION" ]]; then + # --version 1.34 β†’ look for 1-34/ + local minor_dash="${TARGET_VERSION//./-}" + local target_dir="$BASE_DIR/$minor_dash" + if [[ ! -d "$target_dir" ]]; then + echo "Version directory not found: $target_dir" >&2 + echo "Available:" >&2 + ls -d "$BASE_DIR"/1-*/ 2>/dev/null | sed 's|/$||; s|.*/||; s/^/ /' >&2 + return 1 + fi + echo "$target_dir" + else + # Default: highest 1-* directory + local highest + highest=$(ls -d "$BASE_DIR"/1-*/ 2>/dev/null | sort -t- -k2 -n | tail -1) + if [[ -z "$highest" ]]; then + echo "No version directories found in: $BASE_DIR" >&2 + return 1 + fi + echo "$highest" + fi +} + +VERSION_DIRS=$(collect_version_dirs) + +if [[ -z "$VERSION_DIRS" ]]; then + echo "No version directories to build" >&2 + exit 1 +fi + +# ============================================ +# OCI registry setup +# ============================================ + +setup_oci() { + if [[ -z "${OCI_REGISTRY:-}" ]]; then + DATE_YYYYMMDD="${OCI_DATE:-$(date +%Y%m%d)}" + export OCI_REGISTRY="ttl.sh" + export OCI_REPOSITORY="clusterstacks-${DATE_YYYYMMDD}" + echo "Auto-configured ttl.sh: $OCI_REGISTRY/$OCI_REPOSITORY (expires in 24h)" >&2 + fi + + if [[ -z "${OCI_REPOSITORY:-}" ]]; then + echo "Error: OCI_REPOSITORY must be set when OCI_REGISTRY is set." >&2 + exit 1 + fi +} + +OCI_SETUP_DONE=false +ensure_oci() { + if [[ "$OCI_SETUP_DONE" != "true" ]]; then + setup_oci + OCI_SETUP_DONE=true + fi +} + +# ============================================ +# CSO installation +# ============================================ + +CSO_CHART="${CSO_CHART:-oci://registry.scs.community/cluster-stacks/cso}" + +install_cso() { + if ! command -v helm >/dev/null 2>&1; then + echo "helm not found β€” install from https://helm.sh/docs/intro/install/" >&2 + exit 1 + fi + + echo "Installing/upgrading CSO..." >&2 + echo " Chart: $CSO_CHART" >&2 + # The CSO expects the repository including the registry host (e.g. registry.example.com/kaas/cluster-stacks) + echo " OCI config: registry=$OCI_REGISTRY repository=$OCI_REGISTRY/$OCI_REPOSITORY" >&2 + + local image_args=() + if [[ -n "${CSO_IMAGE_REPOSITORY:-}" ]]; then + echo " Image: ${CSO_IMAGE_REPOSITORY}:${CSO_IMAGE_TAG:-}" >&2 + image_args+=(--set "controllerManager.manager.image.repository=${CSO_IMAGE_REPOSITORY}") + fi + if [[ -n "${CSO_IMAGE_TAG:-}" ]]; then + image_args+=(--set "controllerManager.manager.image.tag=${CSO_IMAGE_TAG}") + fi + echo "" >&2 + + helm upgrade -i cso "$CSO_CHART" \ + --namespace cso-system --create-namespace \ + --set controllerManager.manager.source=oci \ + --set "clusterStackVariables.ociRegistry=${OCI_REGISTRY}" \ + --set "clusterStackVariables.ociRepository=${OCI_REGISTRY}/${OCI_REPOSITORY}" \ + "${image_args[@]+"${image_args[@]}"}" >&2 + + echo "" >&2 +} + +is_dev_registry() { + [[ "${OCI_REGISTRY:-}" == "ttl.sh" || -z "${OCI_REPOSITORY:-}" ]] +} + +list_oci_tags() { + require_command oras + + local oras_opts=() + mapfile -t oras_opts < <(oras_auth_opts) + + local tags oras_err + oras_err=$(mktemp) + if ! tags=$(oras repo tags "${OCI_REGISTRY}/${OCI_REPOSITORY}" "${oras_opts[@]+"${oras_opts[@]}"}" 2>"$oras_err"); then + # A repository without any pushed artifact does not exist yet: start with v1 + if grep -qiE 'name unknown|NAME_UNKNOWN|repository name not known|not found' "$oras_err"; then + tags="" + else + echo "Failed to query OCI tags from ${OCI_REGISTRY}/${OCI_REPOSITORY}:" >&2 + cat "$oras_err" >&2 + rm -f "$oras_err" + exit 1 + fi + fi + rm -f "$oras_err" + echo "$tags" +} + +latest_published_release_number() { + local tag_prefix="$1" + local tags latest + + tags=$(list_oci_tags) || exit 1 + latest=$(echo "$tags" | extract_latest_release_number "$tag_prefix" || true) + echo "${latest:-0}" +} + +# Determine release version for a given K8s minor version. +# Stable: highest published vN + 1. Dev: v0-ttl.. +get_release_version() { + local provider="$1" + local stack_name="$2" + local k8s_dash="$3" + local tag_prefix="${provider}-${stack_name}-${k8s_dash}" + + if [[ -n "$RELEASE_VERSION" ]]; then + echo "$RELEASE_VERSION" + return + fi + + if is_dev_registry; then + echo "v0-ttl.$(date +%s)" + return + fi + + local latest + latest=$(latest_published_release_number "$tag_prefix") || exit 1 + echo "v$((latest + 1))" +} + +# ============================================ +# Resolve K8s patch version +# ============================================ + +# Given a version like "1.34" (no patch), resolve to latest patch. +# If already has patch (1.34.3), return as-is. +resolve_k8s_version() { + local version="$1" + local provider="$2" + + if [[ "$version" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo "$version" + return + fi + + local latest + if [[ "$provider" == "docker" ]]; then + require_command jq + if ! latest=$(curl -sfL "https://registry.hub.docker.com/v2/repositories/kindest/node/tags?page_size=100&name=v${version}." 2>/dev/null \ + | jq -r '.results[].name' 2>/dev/null \ + | grep -E "^v${version}\.[0-9]+$" \ + | sed 's/^v//' \ + | sort -V \ + | tail -1); then + echo "Failed to resolve the latest kindest/node patch version for Kubernetes ${version}." >&2 + exit 1 + fi + else + if ! latest=$(curl -sfL "https://dl.k8s.io/release/stable-${version}.txt" 2>/dev/null | sed 's/^v//'); then + echo "Failed to resolve the latest patch version for Kubernetes ${version}." >&2 + exit 1 + fi + fi + + if [[ -n "$latest" && "$latest" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo "$latest" + return + fi + + echo "Could not resolve a stable patch version for Kubernetes ${version}." >&2 + exit 1 +} + +finalize_release_dir() { + local staged_dir="$1" + local output_dir="$2" + + mkdir -p "$(dirname "$output_dir")" + + if [[ -e "$output_dir" ]]; then + if ! rm -rf "$output_dir" 2>/dev/null; then + if [[ "$PUBLISH" == "true" ]]; then + warn "Could not replace existing output directory: $output_dir" + warn "Keeping staged artifacts in: $staged_dir" + echo "$staged_dir" + return + fi + + echo "Output directory is not writable: $output_dir" >&2 + echo "Remove it or fix permissions and retry." >&2 + exit 1 + fi + fi + + if mv "$staged_dir" "$output_dir" 2>/dev/null; then + echo "$output_dir" + return + fi + + if [[ "$PUBLISH" == "true" ]]; then + warn "Could not move release artifacts to: $output_dir" + warn "Keeping staged artifacts in: $staged_dir" + echo "$staged_dir" + return + fi + + echo "Failed to move release artifacts to: $output_dir" >&2 + exit 1 +} + +# ============================================ +# Build one version directory +# ============================================ + +build_version_dir() { + local version_dir="$1" + # Remove trailing slash + version_dir="${version_dir%/}" + + local csctl_yaml="$version_dir/csctl.yaml" + if [[ ! -f "$csctl_yaml" ]]; then + echo "csctl.yaml not found in: $version_dir" >&2 + exit 1 + fi + + # Read stack configuration + local provider stack_name k8s_version_raw + provider=$(yq -r '.config.provider.type' "$csctl_yaml") + stack_name=$(yq -r '.config.clusterStackName' "$csctl_yaml") + k8s_version_raw=$(yq -r '.config.kubernetesVersion | sub("^v","")' "$csctl_yaml") + + # Resolve K8s patch version + local k8s_version + k8s_version=$(resolve_k8s_version "$k8s_version_raw" "$provider") + local k8s_short + k8s_short=$(extract_k8s_minor_version "$k8s_version") + local k8s_dash="${k8s_short//./-}" + + echo "" >&2 + echo "Building ${provider}-${stack_name} for K8s ${k8s_version} (from ${version_dir})" >&2 + echo "---" >&2 + + # Get release version + local release_version + release_version=$(get_release_version "$provider" "$stack_name" "$k8s_dash") + mkdir -p "$OUTPUT_DIR" + local output_release_dir="${OUTPUT_DIR}/${provider}-${stack_name}-${k8s_dash}-${release_version//./-}" + local release_dir + release_dir=$(mktemp -d "${OUTPUT_DIR}/.${provider}-${stack_name}-${k8s_dash}-${release_version}.XXXXXX") + + # ---- Prepare working copy ---- + local work_dir + work_dir=$(mktemp -d) + trap "rm -rf '$work_dir'" RETURN + + cp -r "$version_dir/cluster-class" "$work_dir/" + cp -r "$version_dir/cluster-addon" "$work_dir/" + + # Patch cluster-class Chart.yaml: set name and version + local class_chart="$work_dir/cluster-class/Chart.yaml" + yq_edit_in_place ".name = \"${provider}-${stack_name}-${k8s_dash}-cluster-class\"" "$class_chart" + yq_edit_in_place ".version = \"${release_version}\"" "$class_chart" + + # Update kubernetesVersion in csctl.yaml to the resolved patch version + cp "$csctl_yaml" "$work_dir/csctl.yaml" + yq_edit_in_place ".config.kubernetesVersion = \"v${k8s_version}\"" "$work_dir/csctl.yaml" + + # Patch cluster-class values.yaml image names (if the field exists) + local class_values="$work_dir/cluster-class/values.yaml" + if [[ -f "$class_values" ]]; then + if yq_has_path '.images.controlPlane.name' "$class_values"; then + yq_edit_in_place ".images.controlPlane.name = \"ubuntu-capi-image-v${k8s_version}\"" "$class_values" + yq_edit_in_place ".images.worker.name = \"ubuntu-capi-image-v${k8s_version}\"" "$class_values" + elif yq_has_path '.images.controlPlane[0].name' "$class_values"; then + yq_edit_in_place ".images.controlPlane[0].name = \"registry.scs.community/docker.io/kindest/node:v${k8s_version}\"" "$class_values" + yq_edit_in_place ".images.worker[0].name = \"registry.scs.community/docker.io/kindest/node:v${k8s_version}\"" "$class_values" + fi + fi + + # ---- Package cluster-class ---- + echo " Packaging cluster-class..." >&2 + rm -rf "$work_dir/cluster-class/charts" + helm package "$work_dir/cluster-class" -d "$release_dir/" > /dev/null + echo " cluster-class packaged" >&2 + + # ---- Package cluster-addon bundle ---- + echo " Packaging cluster-addon..." >&2 + local addon_temp + addon_temp=$(mktemp -d) + local addon_count=0 + + for addon_dir in "$work_dir"/cluster-addon/*/; do + [[ -d "$addon_dir" ]] || continue + local addon_name + addon_name=$(basename "$addon_dir") + cp -r "$addon_dir" "$addon_temp/$addon_name" + rm -rf "$addon_temp/$addon_name/charts" + addon_count=$((addon_count + 1)) + done + + if [[ $addon_count -eq 0 ]]; then + echo " No addon subdirectories found" >&2 + rm -rf "$addon_temp" + exit 1 + fi + + local addon_tgz="${provider}-${stack_name}-${k8s_dash}-cluster-addon-${release_version}.tgz" + (cd "$addon_temp" && tar -czf "$release_dir/$addon_tgz" */) + rm -rf "$addon_temp" + echo " cluster-addon packaged ($addon_count addons)" >&2 + + # ---- Validate addon bundle ---- + if [[ "$VALIDATE" == "true" && -f "$version_dir/clusteraddon.yaml" ]]; then + echo " Validating addon bundle..." >&2 + local validate_dir + validate_dir=$(mktemp -d) + tar -xzf "$release_dir/$addon_tgz" -C "$validate_dir" + + local expected_addons + expected_addons=$(yq '.addonStages | to_entries | .[].value[].name' "$version_dir/clusteraddon.yaml" 2>/dev/null | sort -u) + + local failed=false + for addon in $expected_addons; do + if [[ ! -d "$validate_dir/$addon" ]]; then + echo " Missing addon: $addon (referenced in clusteraddon.yaml)" >&2 + failed=true + fi + done + rm -rf "$validate_dir" + + if [[ "$failed" == "true" ]]; then + exit 1 + fi + echo " Validation passed" >&2 + fi + + # ---- Copy clusteraddon.yaml ---- + if [[ -f "$version_dir/clusteraddon.yaml" ]]; then + cp "$version_dir/clusteraddon.yaml" "$release_dir/" + fi + + # ---- Copy csctl.yaml (with resolved patch version) ---- + cp "$work_dir/csctl.yaml" "$release_dir/" + + # ---- Generate metadata.yaml ---- + local git_hash + git_hash=$(git rev-parse HEAD 2>/dev/null || echo "unknown") + + cat > "$release_dir/metadata.yaml" < "$release_dir/hashes.json" <&2 +} + +# ============================================ +# Compare with the latest published release +# ============================================ + +normalize_release_for_compare() { + local src="$1" dest="$2" + + mkdir -p "$dest" + local f + for f in "$src"/*.tgz; do + [[ -f "$f" ]] && tar -xzf "$f" -C "$dest" + done + for f in clusteraddon.yaml csctl.yaml; do + [[ -f "$src/$f" ]] && cp "$src/$f" "$dest/" + done + if [[ -f "$src/metadata.yaml" ]]; then + grep -vE '^[[:space:]]*(clusterStack|clusterAddon|nodeImage):' "$src/metadata.yaml" > "$dest/metadata.yaml" + fi + find "$dest" -name Chart.yaml -exec sed -i '/^version:/d' {} + +} + +find_unchanged_published_release() { + local release_dir="$1" tag_prefix="$2" published_dir="$3" + + local latest + latest=$(latest_published_release_number "$tag_prefix") || exit 1 + if [[ "$latest" -eq 0 ]]; then + log " No published release of ${tag_prefix} yet" + return 0 + fi + + local oras_opts=() + mapfile -t oras_opts < <(oras_auth_opts) + + log " Comparing with published release ${tag_prefix}-v${latest}" + if ! (cd "$published_dir" && oras pull "${OCI_REGISTRY}/${OCI_REPOSITORY}:${tag_prefix}-v${latest}" "${oras_opts[@]+"${oras_opts[@]}"}" >/dev/null 2>&1); then + warn "Could not pull ${tag_prefix}-v${latest}, publishing a new release" + return 0 + fi + + local compare_dir + compare_dir=$(mktemp -d) + normalize_release_for_compare "$release_dir" "$compare_dir/new" + normalize_release_for_compare "$published_dir" "$compare_dir/published" + + local identical=false + if diff -qr "$compare_dir/new" "$compare_dir/published" >/dev/null 2>&1; then + identical=true + fi + rm -rf "$compare_dir" + + if [[ "$identical" == "true" ]]; then + echo "v${latest}" + else + log " Changes detected, publishing a new release" + fi +} + +# ============================================ +# Publish to OCI +# ============================================ + +publish_version() { + local release_dir="$1" + local provider="$2" + local stack_name="$3" + local k8s_dash="$4" + local release_version="$5" + # The CSO requests the release tag in the dash notation (e.g. v0-ttl-123, not v0-ttl.123) + local oci_tag="${provider}-${stack_name}-${k8s_dash}-${release_version//./-}" + + if [[ -z "${OCI_REGISTRY:-}" || -z "${OCI_REPOSITORY:-}" ]]; then + echo " OCI_REGISTRY or OCI_REPOSITORY not set" >&2 + exit 1 + fi + + if ! command -v oras >/dev/null 2>&1; then + echo " oras not found β€” install from https://oras.land/docs/installation" >&2 + exit 1 + fi + + local oras_opts=() + mapfile -t oras_opts < <(oras_auth_opts) + + if ! is_dev_registry && oras manifest fetch --descriptor "$OCI_REGISTRY/$OCI_REPOSITORY:$oci_tag" "${oras_opts[@]+"${oras_opts[@]}"}" >/dev/null 2>&1; then + echo " Refusing to overwrite existing release $OCI_REGISTRY/$OCI_REPOSITORY:$oci_tag" >&2 + exit 1 + fi + + log "" + log " Publishing to $OCI_REGISTRY/$OCI_REPOSITORY:$oci_tag" + + local files=() + for f in "$release_dir"/*; do + [[ -f "$f" ]] && files+=("$(basename "$f")") + done + + (cd "$release_dir" && oras push \ + "$OCI_REGISTRY/$OCI_REPOSITORY:$oci_tag" \ + --artifact-type application/vnd.clusterstack.release \ + "${oras_opts[@]+"${oras_opts[@]}"}" \ + "${files[@]}") >&2 + + echo " Published: $OCI_REGISTRY/$OCI_REPOSITORY:$oci_tag" >&2 + echo " Pull: oras pull $OCI_REGISTRY/$OCI_REPOSITORY:$oci_tag" >&2 +} + +# ============================================ +# Main +# ============================================ + +# Resolve OCI config once (needed for --publish and/or --install-cso) +if [[ "$PUBLISH" == "true" || "$INSTALL_CSO" == "true" ]]; then + ensure_oci +fi + +# Install/upgrade CSO if requested +if [[ "$INSTALL_CSO" == "true" ]]; then + install_cso + + # --install-cso alone only installs the operator, it does not build anything + if [[ -z "$TARGET_VERSION" && "$BUILD_ALL" != "true" && "$PUBLISH" != "true" ]]; then + exit 0 + fi +fi + +echo "Stack base: $BASE_DIR" >&2 +echo "Version dirs: $(echo "$VERSION_DIRS" | tr '\n' ' ')" >&2 +echo "" >&2 + +# Track built versions for "next steps" output +declare -a BUILT_K8S_SHORTS=() +declare -a BUILT_CS_VERSIONS=() +declare -a BUILT_PROVIDERS=() +declare -a BUILT_STACK_NAMES=() + +for version_dir in $VERSION_DIRS; do + build_version_dir "$version_dir" +done + +echo "" >&2 +echo "Done." >&2 + +# ============================================ +# Next steps (after publish) +# ============================================ + +if [[ "$PUBLISH" == "true" && ${#BUILT_K8S_SHORTS[@]} -gt 0 ]]; then + echo "" >&2 + echo "================================================================" >&2 + echo "Next steps" >&2 + echo "================================================================" >&2 + + if [[ "$INSTALL_CSO" != "true" ]]; then + echo "" >&2 + echo "1. Install the Cluster Stack Operator (or re-run with --install-cso):" >&2 + echo "" >&2 + echo " helm upgrade -i cso ${CSO_CHART} \\" >&2 + echo " --namespace cso-system --create-namespace \\" >&2 + echo " --set controllerManager.manager.source=oci \\" >&2 + echo " --set clusterStackVariables.ociRegistry=\"${OCI_REGISTRY}\" \\" >&2 + echo " --set clusterStackVariables.ociRepository=\"${OCI_REGISTRY}/${OCI_REPOSITORY}\"" >&2 + echo "" >&2 + echo "2. Apply the ClusterStack resource(s):" >&2 + else + echo "" >&2 + echo "Apply the ClusterStack resource(s):" >&2 + fi + echo "" >&2 + for ((i=0; i<${#BUILT_K8S_SHORTS[@]}; i++)); do + local_provider="${BUILT_PROVIDERS[$i]}" + local_stack="${BUILT_STACK_NAMES[$i]}" + local_k8s="${BUILT_K8S_SHORTS[$i]}" + local_version="${BUILT_CS_VERSIONS[$i]}" + echo " ./hack/generate-resources.sh providers/${local_provider}/${local_stack} --version ${local_k8s} --cs-version ${local_version} | kubectl apply -f -" >&2 + done + echo "" >&2 +fi diff --git a/hack/config-template.md b/hack/config-template.md index 8f05340b..bfc32a1e 100644 --- a/hack/config-template.md +++ b/hack/config-template.md @@ -1,59 +1,60 @@ # Configuration -This page lists the custom configuration options available, including their default values and if they are optional. The following example shows how these variables can be used inside the `cluster.yaml` file under `spec.topology.variables`. +This page lists the custom configuration options available, including their default values and whether they are optional. The following example shows how these variables can be used inside the `cluster.yaml` file under `spec.topology.variables`. + +## Version matrix + +!!matrix!! ## Example ```yaml -apiVersion: cluster.x-k8s.io/v1beta1 +apiVersion: kind: Cluster metadata: - name: - namespace: + name: my-cluster + namespace: my-namespace labels: managed-secret: cloud-config spec: - clusterNetwork: - pods: - cidrBlocks: - - 192.168.0.0/16 - serviceDomain: cluster.local - services: - cidrBlocks: - - 10.96.0.0/12 topology: - variables: // <-- variables from the table can be set here - - name: controller_flavor - value: "SCS-4V-8-20" - - name: worker_flavor - value: "SCS-4V-8-20" - - name: external_id - value: "ebfe5546-f09f-4f42-ab54-094e457d42ec" - class: openstack-alpha-1-29-v2 + variables: # <-- variables from the table below can be set here + - name: + value: + classRef: # cluster.x-k8s.io/v1beta1 clusters (hcp) use "class: " instead + name: controlPlane: - replicas: 2 - version: v1.29.3 + replicas: 3 + variables: + overrides: + - name: + value: + version: v workers: machineDeployments: - - class: openstack-alpha-1-29-v2 - failureDomain: nova - name: openstack-alpha-1-29-v2 - replicas: 4 + - class: default-worker + name: md-0 + replicas: 3 ``` -Variables from the table containing a `.` are to be used in an object with the part before the dot being the object name and the part behind the dot being the value names. The following example demonstrates this with `oidc_config`. +`spec.clusterNetwork` is optional. Set pod or service CIDRs only when you need +non-default values or explicit network settings. + +Variables of type `object` are set as nested values. The following example demonstrates the shape: ```yaml ... topology: variables: - - name: oidc_config + - name: oidcConfig value: - issuer_url: "https://dex.k8s.scs.community" - client_id: "kubectl" + issuerURL: "https://dex.k8s.scs.community" + clientID: "kubectl" ... ``` +In v1beta2, per-role overrides (e.g. different flavors for control plane and workers) are set via `topology.controlPlane.variables.overrides` and `topology.workers.machineDeployments[].variables.overrides` instead of separate variable names. + ## Available variables -!!table!! \ No newline at end of file +!!table!! diff --git a/hack/docugen.py b/hack/docugen.py index 1f58fdd7..fa8bd90b 100755 --- a/hack/docugen.py +++ b/hack/docugen.py @@ -1,149 +1,284 @@ #!/usr/bin/env python3 """ -Generate markdown table from cluster-class variable definitions. +Generate markdown documentation from ClusterClass variable definitions. -This script temporarily renders the helm template of the cluster-class. -Parses the `Cluster.spec.topology.variables` definitions and -generates a markdown table for documentation purposes. +Renders the cluster-class Helm template, parses the topology variables +(openAPIV3Schema), and outputs a markdown table of all configurable options. + +Usage: + ./hack/docugen.py [stack-dir] + ./hack/docugen.py [stack-dir] --version 1.35 --output docs/configuration.md + ./hack/docugen.py [stack-dir] --template hack/config-template.md + ./hack/docugen.py [stack-dir] --matrix + ./hack/docugen.py [stack-dir] --dry-run + ./hack/docugen.py [stack-dir] --matrix --update docs/providers/openstack/scs-configuration.md + +If is omitted, it is derived from $PROVIDER and $CLUSTER_STACK +(default: providers/openstack/scs). Without --version, the highest 1-* version +directory is used. + +For backwards compatibility, a direct version directory like +providers/openstack/scs/1-35 is also accepted. + +With --update only the generated sections of an existing document are replaced, everything else +(hand written text) is kept. The sections are marked with + + ... + ... """ import argparse +import json +import os +import re import subprocess -from pathlib import Path import sys +from pathlib import Path import yaml -BASE_PATH = Path(__file__).parent.parent -TEMPLATE_PATH = BASE_PATH.joinpath("providers", "openstack", "scs", "cluster-class") -DOCS_TMPL_PATH = BASE_PATH.joinpath("hack", "config-template.md") -DOCS_OUT_PATH = BASE_PATH.joinpath("docs", "providers", "openstack", "configuration.md") +SCRIPT_DIR = Path(__file__).resolve().parent +REPO_ROOT = SCRIPT_DIR.parent -def generate_row(content: list): - """Generate string of markdown table row.""" - row_tmpl = "|{content}|" - row_str = "|".join(content) - return row_tmpl.format(content=row_str) +def generate_row(columns: list) -> str: + """Generate a markdown table row from a list of column values.""" + return "|" + "|".join(str(c) for c in columns) + "|" -def parse_variable(tmpl: dict) -> list: - """ - Parse schema of simple cluster-stack variable type. (String, Boolean, Integer, Array) +def format_value(value) -> str: + """Format a default/example value as it would be written in YAML/JSON (no Python repr).""" + if value is None or value == "": + return "" + if isinstance(value, str): + return f'"{value}"' + return json.dumps(value) - Parameters: - tmpl (dict): Dictionary of variable schema, - parsed from cluster-class.yaml via yaml.safe_load(). - Returns: List of variable schema properties. - Each entry represents a column in the final markdown table row. - """ - var_name = tmpl["name"] - var_required = tmpl["required"] - var_schema = tmpl["schema"]["openAPIV3Schema"] +def parse_variable(var: dict) -> list: + """Parse a simple variable (string, boolean, integer, array) into table columns.""" + name = var["name"] + required = var.get("required", False) + schema = var["schema"]["openAPIV3Schema"] - var_type = var_schema["type"] - var_default = var_schema.get("default", "") - var_example = var_schema.get("example", "") - var_desc = var_schema.get("description", "TODO") + var_type = schema["type"] + description = schema.get("description", "TODO").strip().replace("\n", "
") - row = [] - row.append(f"`{var_name}`") - row.append(var_type) + # strings keep their quotes (also an empty default ""), other types are written as JSON + default = schema.get("default") + if var_type == "string" and default is not None: + default = f'"{default}"' + else: + default = format_value(default) + example = format_value(schema.get("example")) - # Make sure that example and default values are quoted in the table - if var_type == "string": - row.append(f'"{var_default}"') - row.append(f'"{var_example}"') + return [f"`{name}`", var_type, default, example, description, str(required).lower()] - # Cast any non-string type example / default to string - if var_type in ["array", "integer", "boolean"]: - row.append(str(var_default)) - row.append(str(var_example)) - row.append(var_desc.replace("\n", "
")) - row.append(str(var_required)) # Convert boolean variable to string +def parse_object(var: dict) -> list: + """Parse an object variable into multiple table rows (one per property).""" + name = var["name"] + props = var["schema"]["openAPIV3Schema"]["properties"] + required = var["schema"]["openAPIV3Schema"].get("required", []) + rows = [] - return row + for prop_name, prop_schema in props.items(): + rows.append([ + f"`{name}.{prop_name}`", + prop_schema["type"], + format_value(prop_schema.get("default")), + format_value(prop_schema.get("example")), + prop_schema.get("description", "TODO").strip().replace("\n", "
"), + str(prop_name in required).lower(), + ]) + return rows -def parse_object(tmpl: dict) -> list: - """ - Parse cluster-stack variable schema of type object. - Separated due to nesting in YAML format. - Parameters: - tmpl (dict): Dictionary of object schema, - parsed from cluster-class.yaml via yaml.safe_load(). - Returns: - List of object properties. - Each entry represents a row in the final markdown table as a list, - consisting of the rows column values. - """ - var_name = tmpl["name"] - props = tmpl["schema"]["openAPIV3Schema"]["properties"] +def render_cluster_class(stack_dir: Path) -> dict: + """Render the cluster-class Helm template and return parsed YAML.""" + cluster_class_dir = stack_dir / "cluster-class" + if not cluster_class_dir.exists(): + print(f"cluster-class directory not found: {cluster_class_dir}", file=sys.stderr) + sys.exit(1) - object_list = [] - for prop in props: - row = [] - row.append(f"`{var_name}.{prop}`") - row.append(props[prop]["type"]) - row.append(props[prop].get("default", "")) - row.append(props[prop].get("example", "")) - row.append(props[prop].get("description", "TODO")) - # append dummy row for required field - row.append("") + result = subprocess.run( + ["helm", "template", "docugen", str(cluster_class_dir), + "-s", "templates/cluster-class.yaml"], + capture_output=True, check=False, + ) - object_list.append(row) - return object_list + if result.returncode != 0: + print(f"helm template failed:\n{result.stderr.decode()}", file=sys.stderr) + sys.exit(1) + return yaml.safe_load(result.stdout.decode("utf-8")) -if __name__ == "__main__": - parser = argparse.ArgumentParser() - parser.add_argument( - "--dry-run", action="store_true", help="Only print result to stdout." - ) - args = parser.parse_args() +def resolve_base_dir(stack_dir: Path | None) -> Path: + """Resolve the stack base directory from CLI input or environment.""" + if stack_dir is None: + stack_dir = Path( + f"providers/{os.environ.get('PROVIDER', 'openstack')}/{os.environ.get('CLUSTER_STACK', 'scs')}" + ) - cmd = [ - "helm", - "template", - "docugen", - TEMPLATE_PATH, - "-s", - "templates/cluster-class.yaml", - ] + if not stack_dir.is_absolute(): + stack_dir = REPO_ROOT / stack_dir + + return stack_dir.resolve() + + +def resolve_version_dir(stack_dir: Path | None, version: str | None) -> tuple[Path, Path]: + """Resolve stack base dir and concrete version dir.""" + resolved_input = resolve_base_dir(stack_dir) - with open(DOCS_TMPL_PATH, "r") as f: - tmpl = f.read() + if resolved_input.name.startswith("1-") and (resolved_input / "cluster-class").exists(): + return resolved_input.parent, resolved_input - cmdout = subprocess.run(cmd, capture_output=True, check=False) - rendered_template = cmdout.stdout.decode("utf-8") + stack_base = resolved_input + if not stack_base.exists(): + print(f"Stack base directory not found: {stack_base}", file=sys.stderr) + sys.exit(1) - template = yaml.safe_load(rendered_template) + if version: + version_dir = stack_base / version.replace(".", "-") + if version_dir.exists(): + return stack_base, version_dir + print(f"Version directory not found: {version_dir}", file=sys.stderr) + sys.exit(1) - result_table = [] - result_table.append("|Name|Type|Default|Example|Description|Required|") - result_table.append("|----|----|-------|-------|-----------|--------|") + candidates = sorted( + (path for path in stack_base.glob("1-*") if path.is_dir()), + key=lambda path: int(path.name.split("-", 1)[1]), + ) + if not candidates: + print(f"No version directories found in: {stack_base}", file=sys.stderr) + sys.exit(1) + + return stack_base, candidates[-1] + + +def generate_table(template: dict) -> str: + """Generate a markdown table from ClusterClass topology variables.""" + rows = [ + "|Name|Type|Default|Example|Description|Required|", + "|----|----|-------|-------|-----------|--------|", + ] for var in template["spec"]["variables"]: - if var["schema"]["openAPIV3Schema"]["type"] in ["object"]: - parsed = parse_object(var) - for object_property in parsed: - result_table.append(generate_row(object_property)) + var_type = var["schema"]["openAPIV3Schema"]["type"] + if var_type == "object": + for row in parse_object(var): + rows.append(generate_row(row)) else: - parsed = parse_variable(var) - result_table.append(generate_row(parsed)) - - output = tmpl.replace("!!table!!", "\n".join(result_table)) + rows.append(generate_row(parse_variable(var))) + + return "\n".join(rows) + + +def update_sections(path: Path, sections: dict, dry_run: bool) -> None: + """Replace the content between BEGIN/END GENERATED markers in an existing document.""" + if not path.exists(): + sys.exit(f"{path} does not exist") + + text = path.read_text() + replaced = [] + for name, content in sections.items(): + if content is None: + continue + pattern = re.compile( + rf"(\n).*?(\n)", + re.DOTALL, + ) + text, count = pattern.subn(lambda m: m.group(1) + content.strip() + m.group(2), text) + if count: + replaced.append(name) + + if not replaced: + sys.exit(f"{path} contains no '' markers, nothing to update") + + if dry_run: + print(text) + else: + path.write_text(text) + print(f"Updated {', '.join(replaced)} in {path}", file=sys.stderr) + + +def main(): + parser = argparse.ArgumentParser( + description="Generate docs from ClusterClass variables", + ) + parser.add_argument( + "stack_dir", type=Path, nargs="?", default=None, + help="Path to the cluster stack base directory or a direct 1-XX version directory", + ) + parser.add_argument( + "--version", type=str, default=None, + help="Kubernetes minor version to document (default: highest 1-* directory)", + ) + parser.add_argument( + "--template", type=Path, default=None, + help="Markdown template file with !!table!! placeholder", + ) + parser.add_argument( + "--output", "-o", type=Path, default=None, + help="Output file path (default: stdout)", + ) + parser.add_argument( + "--matrix", action="store_true", + help="Include version matrix from show-matrix.sh --markdown", + ) + parser.add_argument( + "--update", type=Path, default=None, + help="Replace only the generated sections (between BEGIN/END GENERATED markers) of this file", + ) + parser.add_argument( + "--dry-run", action="store_true", + help="Print to stdout even if --output or --update is set", + ) + args = parser.parse_args() - if args.dry_run: + stack_base, version_dir = resolve_version_dir(args.stack_dir, args.version) + + # Render and parse + template = render_cluster_class(version_dir) + table = generate_table(template) + + # Generate version matrix if requested + matrix = "" + if args.matrix: + script = Path(__file__).parent / "show-matrix.sh" + result = subprocess.run( + ["bash", str(script), str(stack_base), "--markdown"], + capture_output=True, check=False, + ) + if result.returncode == 0: + matrix = result.stdout.decode("utf-8").strip() + else: + print(f"show-matrix.sh failed:\n{result.stderr.decode()}", file=sys.stderr) + + if args.update is not None: + update_sections(args.update, {"variables": table, "matrix": matrix if args.matrix else None}, args.dry_run) + return + + # Apply template if provided + if args.template and args.template.exists(): + output = args.template.read_text(encoding="utf-8") + output = output.replace("!!table!!", table) + output = output.replace("!!matrix!!", matrix) + else: + output = table + + # Output + if args.dry_run or args.output is None: print(output) - sys.exit() + else: + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(output, encoding="utf-8") + print(f"Written: {args.output}", file=sys.stderr) - print(f"Writing output to file {DOCS_OUT_PATH}") - with open(DOCS_OUT_PATH, "w") as f: - f.write(output) + +if __name__ == "__main__": + main() diff --git a/hack/ensure-connected-to-mgt-cluster.sh b/hack/ensure-connected-to-mgt-cluster.sh deleted file mode 100755 index 5eda91e7..00000000 --- a/hack/ensure-connected-to-mgt-cluster.sh +++ /dev/null @@ -1,49 +0,0 @@ -#!/bin/bash -# Copyright 2023 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -euo pipefail - -context="$(kubectl config current-context 2>/dev/null || true)" - -if [ "$context" = "kind-scs-cluster-stacks" ]; then - exit 0 -fi - -if [ "$context" = "" ]; then - echo "No context set" - exit 1 -fi - - -echo "You are connected to $context. Please set KUBECONFIG to .mgt-cluster-kubeconfig.yaml" -exit 1 - -if [ "$#" -lt 1 ]; then - echo "Usage: $0 VAR1 VAR2 ..." - exit 1 -fi - -missing_vars=() -for varname in "$@"; do - eval varvalue="\$$varname" - if [ -z "$varvalue" ]; then - missing_vars+=("$varname") - fi -done - -if [ ${#missing_vars[@]} -gt 0 ]; then - echo "Missing or empty environment variables: ${missing_vars[*]}" - exit 1 -fi diff --git a/hack/ensure-env-variables.sh b/hack/ensure-env-variables.sh deleted file mode 100755 index 8dc027e4..00000000 --- a/hack/ensure-env-variables.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/bin/bash -# Copyright 2023 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -if [ "$#" -lt 1 ]; then - echo "Usage: $0 VAR1 VAR2 ..." - exit 1 -fi - -missing_vars=() -for varname in "$@"; do - eval varvalue="\$$varname" - if [ -z "$varvalue" ]; then - missing_vars+=("$varname") - fi -done - -if [ ${#missing_vars[@]} -gt 0 ]; then - echo "Missing or empty environment variables: ${missing_vars[*]}" - exit 1 -fi diff --git a/hack/generate-image-manifests.sh b/hack/generate-image-manifests.sh new file mode 100755 index 00000000..f2c6f9d9 --- /dev/null +++ b/hack/generate-image-manifests.sh @@ -0,0 +1,432 @@ +#!/usr/bin/env bash +# Generate OpenStack image manifests for Kubernetes CAPI images. +# +# Supports two output formats: +# orc ORC Image CRD (default) β€” for k-orc.cloud Image resources +# image-manager openstack-image-manager YAML β€” for OSISM image-manager +# +# Usage: +# ./hack/generate-image-manifests.sh [stack-dir] [options] +# +# Options: +# --version Generate for a specific K8s minor version +# (X.Y.Z: this patch version instead of the one in csctl.yaml) +# --format Output format: orc (default) or image-manager +# --namespace Namespace for ORC Image manifests +# --check-image-repo Resolve latest patch version via IMAGE_BASE_URL/last- first +# --visibility Image visibility: private (default), public, shared, community +# --output-dir Write to files instead of stdout +# --skip-checksum Skip fetching SHA256 checksums +# -h, --help Show this help +# +# The is the base directory containing per-minor-version subdirs +# (e.g., providers/openstack/scs). If omitted, it is derived from $PROVIDER +# and $CLUSTER_STACK (default: providers/openstack/scs). Relative paths are +# resolved from the repository root. +# +# Only relevant for OpenStack-based stacks (Docker stacks have no node images). +# +# Examples: +# ./hack/generate-image-manifests.sh --version 1.34 +# ./hack/generate-image-manifests.sh --version 1.34 --namespace orc-system +# ./hack/generate-image-manifests.sh --version 1.34 --check-image-repo +# ./hack/generate-image-manifests.sh --format image-manager --visibility public +# ./hack/generate-image-manifests.sh --version 1.34 --visibility shared --skip-checksum +# +# Environment: +# PROVIDER Provider name (default: openstack) +# CLUSTER_STACK Cluster stack name (default: scs) +# IMAGE_BASE_URL Base URL for images (default: https://nbg1.your-objectstorage.com/osism/openstack-k8s-capi-images) +# IMAGE_NAMESPACE Namespace for ORC Image manifests (optional, same as --namespace) +# CLOUD_NAME CloudCredentialsRef cloud name (default: openstack, orc format only) +# SECRET_NAME CloudCredentialsRef secret name (default: openstack, orc format only) + +set -euo pipefail + +readonly SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +readonly REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" + +require_command() { + local name="$1" + + if ! command -v "$name" >/dev/null 2>&1; then + echo "$name not found. Please install $name and try again." >&2 + exit 1 + fi +} + +extract_k8s_minor_version() { + echo "$1" | sed -E -n 's/^([0-9]+\.[0-9]+)(\.[0-9]+)?$/\1/p' +} + +# Defaults +BASE_URL="${IMAGE_BASE_URL:-https://nbg1.your-objectstorage.com/osism/openstack-k8s-capi-images}" +IMAGE_NAMESPACE="${IMAGE_NAMESPACE:-}" +CLOUD_NAME="${CLOUD_NAME:-openstack}" +SECRET_NAME="${SECRET_NAME:-openstack}" + +# ============================================ +# Argument parsing +# ============================================ + +usage() { + sed -n '3,/^$/s/^# \?//p' "$0" + exit "${1:-1}" +} + +parse_args() { + while [[ $# -gt 0 ]]; do + case "$1" in + --version) [[ $# -ge 2 ]] || { echo "Error: $1 requires a value" >&2; usage; }; TARGET_VERSION="$2"; shift 2 ;; + --format) [[ $# -ge 2 ]] || { echo "Error: $1 requires a value" >&2; usage; }; OUTPUT_FORMAT="$2"; shift 2 ;; + --namespace) [[ $# -ge 2 ]] || { echo "Error: $1 requires a value" >&2; usage; }; IMAGE_NAMESPACE="$2"; shift 2 ;; + --check-image-repo) USE_IMAGE_REPO_LATEST=true; shift ;; + --visibility) [[ $# -ge 2 ]] || { echo "Error: $1 requires a value" >&2; usage; }; VISIBILITY="$2"; shift 2 ;; + --output-dir) [[ $# -ge 2 ]] || { echo "Error: $1 requires a value" >&2; usage; }; OUTPUT_DIR="$2"; shift 2 ;; + --skip-checksum) SKIP_CHECKSUM=true; shift ;; + -h|--help) usage 0 ;; + -*) echo "Unknown option: $1" >&2; usage ;; + *) + if [[ -z "$BASE_DIR" ]]; then + BASE_DIR="$1"; shift + else + echo "Unexpected argument: $1" >&2; usage + fi + ;; + esac + done +} + +resolve_base_dir() { + local dir="$1" + + if [[ -z "$dir" ]]; then + dir="providers/${PROVIDER:-openstack}/${CLUSTER_STACK:-scs}" + fi + + if [[ ! "$dir" = /* ]]; then + dir="$REPO_ROOT/$dir" + fi + + echo "$dir" +} + +BASE_DIR="" +TARGET_VERSION="" +OUTPUT_DIR="" +OUTPUT_FORMAT="orc" +VISIBILITY="private" +USE_IMAGE_REPO_LATEST=false +SKIP_CHECKSUM=false + +parse_args "$@" + +require_command yq +require_command curl + +BASE_DIR="$(resolve_base_dir "$BASE_DIR")" + +if [[ ! -d "$BASE_DIR" ]]; then + echo "Stack base directory not found: $BASE_DIR" >&2 + exit 1 +fi + +case "$OUTPUT_FORMAT" in + orc|image-manager) ;; + *) echo "Unknown format: $OUTPUT_FORMAT (use 'orc' or 'image-manager')" >&2; exit 1 ;; +esac + +case "$VISIBILITY" in + private|public|shared|community) ;; + *) echo "Unknown visibility: $VISIBILITY (use private, public, shared, or community)" >&2; exit 1 ;; +esac + +# ============================================ +# Ubuntu version mapping +# ============================================ + +# K8s 1.32 and earlier β†’ Ubuntu 22.04 (2204) +# K8s 1.33 and later β†’ Ubuntu 24.04 (2404) +ubuntu_for_minor() { + local minor="$1" + if [[ "$minor" -le 32 ]]; then + echo "2204" + else + echo "2404" + fi +} + +# ============================================ +# Check provider +# ============================================ + +FIRST_STACK="" +for stack_file in "$BASE_DIR"/1-*/csctl.yaml; do + if [[ -f "$stack_file" ]]; then + FIRST_STACK="$stack_file" + break + fi +done + +if [[ -z "$FIRST_STACK" ]]; then + echo "No csctl.yaml found in $BASE_DIR/1-*/" >&2 + exit 1 +fi + +STACK_PROVIDER=$(yq -r '.config.provider.type' "$FIRST_STACK") +if [[ "$STACK_PROVIDER" != "openstack" ]]; then + echo "Image manifests are only relevant for OpenStack-based stacks (provider: $STACK_PROVIDER)." >&2 + exit 0 +fi + +[[ -n "$OUTPUT_DIR" ]] && mkdir -p "$OUTPUT_DIR" + +# ============================================ +# Resolve K8s patch version +# ============================================ + +resolve_k8s_version() { + local version="$1" + + if [[ "$version" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo "$version" + return + fi + + if [[ "$USE_IMAGE_REPO_LATEST" == "true" ]]; then + local minor_version + minor_version=$(extract_k8s_minor_version "$version") + if [[ -n "$minor_version" ]]; then + local latest_available + if latest_available=$(curl -sfL "${BASE_URL}/last-${minor_version}" 2>/dev/null | awk '{print $NF}' | sed -E -n 's#.*/ubuntu-[0-9]+-kube-v([0-9]+\.[0-9]+\.[0-9]+)\.qcow2#\1#p'); then + if [[ -n "$latest_available" && "$latest_available" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo "$latest_available" + return + fi + fi + fi + fi + + # Resolve latest stable patch via dl.k8s.io + local latest + if ! latest=$(curl -sfL "https://dl.k8s.io/release/stable-${version}.txt" 2>/dev/null | sed 's/^v//'); then + echo "Failed to resolve the latest patch version for Kubernetes ${version}." >&2 + exit 1 + fi + + if [[ -n "$latest" && "$latest" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo "$latest" + return + fi + + echo "Could not resolve a stable patch version for Kubernetes ${version}." >&2 + exit 1 +} + +# ============================================ +# Collect image data +# ============================================ + +# Arrays to collect data for image-manager format (needs all versions grouped) +declare -a ALL_K8S_VERSIONS=() +declare -a ALL_IMAGE_URLS=() +declare -a ALL_CHECKSUMS=() +declare -a ALL_UBUNTU=() + +GENERATED=0 +FAIL_COUNT=0 + +for version_dir in "$BASE_DIR"/1-*/; do + [[ -d "$version_dir" ]] || continue + csctl_yaml="$version_dir/csctl.yaml" + [[ -f "$csctl_yaml" ]] || continue + + k8s_version_raw=$(yq -r '.config.kubernetesVersion | sub("^v","")' "$csctl_yaml") + k8s_short=$(extract_k8s_minor_version "$k8s_version_raw") + k8s_minor=$(echo "$k8s_short" | cut -d. -f2) + + # Filter by --version if specified + if [[ -n "$TARGET_VERSION" ]]; then + target_short=$(extract_k8s_minor_version "$TARGET_VERSION") + if [[ "$k8s_short" != "$target_short" ]]; then + continue + fi + fi + + # Resolve full K8s patch version + [[ "$TARGET_VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] && k8s_version_raw="$TARGET_VERSION" + k8s_version=$(resolve_k8s_version "$k8s_version_raw") + + UBUNTU=$(ubuntu_for_minor "$k8s_minor") + IMAGE_NAME="ubuntu-${UBUNTU}-kube-v${k8s_version}" + IMAGE_DIR="ubuntu-${UBUNTU}-kube-v${k8s_short}" + IMAGE_URL="${BASE_URL}/${IMAGE_DIR}/${IMAGE_NAME}.qcow2" + + # Fetch checksum + CHECKSUM="" + if [[ "$SKIP_CHECKSUM" != "true" ]]; then + CHECKSUM=$(curl -sf "${IMAGE_URL}.CHECKSUM" | awk '{print $1}' || echo "") + if [[ -z "$CHECKSUM" ]]; then + echo "Failed to fetch checksum for ${k8s_version}: ${IMAGE_URL}.CHECKSUM" >&2 + echo "Use --skip-checksum to generate without hash validation" >&2 + FAIL_COUNT=$((FAIL_COUNT + 1)) + continue + fi + fi + + ALL_K8S_VERSIONS+=("$k8s_version") + ALL_IMAGE_URLS+=("$IMAGE_URL") + ALL_CHECKSUMS+=("$CHECKSUM") + ALL_UBUNTU+=("$UBUNTU") + GENERATED=$((GENERATED + 1)) +done + +# ============================================ +# Output: ORC Image CRD format +# ============================================ + +generate_orc() { + for ((i=0; i<${#ALL_K8S_VERSIONS[@]}; i++)); do + local k8s_version="${ALL_K8S_VERSIONS[$i]}" + local image_url="${ALL_IMAGE_URLS[$i]}" + local checksum="${ALL_CHECKSUMS[$i]}" + local ubuntu="${ALL_UBUNTU[$i]}" + + local hash_block="" + if [[ -n "$checksum" ]]; then + hash_block=" + hash: + algorithm: sha256 + value: ${checksum}" + fi + + local namespace_block="" + if [[ -n "$IMAGE_NAMESPACE" ]]; then + namespace_block=" + namespace: ${IMAGE_NAMESPACE}" + fi + + local manifest="--- +apiVersion: openstack.k-orc.cloud/v1alpha1 +kind: Image +metadata: + name: ubuntu-capi-image-v${k8s_version}${namespace_block} +spec: + cloudCredentialsRef: + cloudName: ${CLOUD_NAME} + secretName: ${SECRET_NAME} + managementPolicy: managed + resource: + visibility: ${VISIBILITY} + properties: + hardware: + diskBus: scsi + scsiModel: virtio-scsi + vifModel: virtio + qemuGuestAgent: true + rngModel: virtio + architecture: x86_64 + minDiskGB: 20 + minMemoryMB: 2048 + operatingSystem: + distro: ubuntu + version: \"${ubuntu:0:2}.${ubuntu:2:2}\" + content: + diskFormat: qcow2 + download: + url: ${image_url}${hash_block}" + + if [[ -n "$OUTPUT_DIR" ]]; then + local outfile="${OUTPUT_DIR}/ubuntu-${ubuntu}-kube-v${k8s_version}.yaml" + echo "$manifest" > "$outfile" + echo "Written: $outfile" >&2 + else + echo "$manifest" + fi + done +} + +# ============================================ +# Output: openstack-image-manager format +# ============================================ + +generate_image_manager() { + # Build versions list + local versions_block="" + for ((i=0; i<${#ALL_K8S_VERSIONS[@]}; i++)); do + local k8s_version="${ALL_K8S_VERSIONS[$i]}" + local image_url="${ALL_IMAGE_URLS[$i]}" + local checksum="${ALL_CHECKSUMS[$i]}" + + local checksum_line="" + if [[ -n "$checksum" ]]; then + checksum_line=" + checksum: \"sha256:${checksum}\"" + fi + + versions_block+=" + - version: 'v${k8s_version}' + url: ${image_url}${checksum_line}" + done + + local manifest="--- +images: + - name: ubuntu-capi-image + enable: true + format: raw + login: ubuntu + min_disk: 20 + min_ram: 2048 + status: active + visibility: ${VISIBILITY} + multi: false + separator: \"-\" + meta: + architecture: x86_64 + hw_disk_bus: virtio + hw_rng_model: virtio + hw_scsi_model: virtio-scsi + hw_watchdog_action: reset + hypervisor_type: qemu + os_distro: ubuntu + os_purpose: k8snode + replace_frequency: never + uuid_validity: none + provided_until: none + tags: + - clusterstacks + versions:${versions_block}" + + if [[ -n "$OUTPUT_DIR" ]]; then + local outfile="${OUTPUT_DIR}/kubernetes.yaml" + echo "$manifest" > "$outfile" + echo "Written: $outfile" >&2 + else + echo "$manifest" + fi +} + +# ============================================ +# Generate output +# ============================================ + +if [[ $GENERATED -eq 0 && $FAIL_COUNT -eq 0 ]]; then + echo "No matching versions found" >&2 + exit 1 +fi + +if [[ $FAIL_COUNT -gt 0 && $GENERATED -eq 0 ]]; then + echo "All $FAIL_COUNT version(s) failed checksum fetch" >&2 + exit 1 +fi + +case "$OUTPUT_FORMAT" in + orc) generate_orc ;; + image-manager) generate_image_manager ;; +esac + +if [[ $FAIL_COUNT -gt 0 ]]; then + echo "Generated $GENERATED manifest(s), failed $FAIL_COUNT" >&2 +else + echo "Generated $GENERATED manifest(s)" >&2 +fi diff --git a/hack/generate-resources.sh b/hack/generate-resources.sh new file mode 100755 index 00000000..20d7d9a6 --- /dev/null +++ b/hack/generate-resources.sh @@ -0,0 +1,302 @@ +#!/usr/bin/env bash +# Generate ClusterStack and Cluster YAML resources for testing. +# +# Usage: +# ./hack/generate-resources.sh [stack-dir] --version 1.34 [options] +# +# The is the base directory containing per-minor-version subdirs +# (e.g., providers/openstack/scs). If omitted, it is derived from $PROVIDER +# and $CLUSTER_STACK (default: providers/openstack/scs). Relative paths are +# resolved from the repository root. +# +# Options: +# --version K8s minor version (required) +# --cs-version Cluster stack version (default: auto-detect, see below) +# --namespace Namespace (default: cluster) +# --cluster-name Workload cluster name (default: cs-cluster) +# --cluster-only Only generate the Cluster resource +# --clusterstack-only Only generate the ClusterStack resource +# -h, --help Show this help +# +# Auto-detection of --cs-version (in priority order): +# 1. Local .release/ directory β€” reads metadata.yaml from the latest matching build +# 2. OCI registry β€” queries tags via oras (requires OCI_REGISTRY + OCI_REPOSITORY) +# 3. Falls back to v1 +# +# Output goes to stdout. Pipe to kubectl apply -f - or redirect to a file. +# +# Examples: +# ./hack/generate-resources.sh --version 1.34 +# ./hack/generate-resources.sh --version 1.34 --cs-version v2 +# ./hack/generate-resources.sh --version 1.34 | kubectl apply -f - +# PROVIDER=docker ./hack/generate-resources.sh --version 1.35 + +set -euo pipefail + +readonly SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +readonly REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" + +require_command() { + local name="$1" + + if ! command -v "$name" >/dev/null 2>&1; then + echo "$name not found. Please install $name and try again." >&2 + exit 1 + fi +} + +extract_latest_release_number() { + local prefix="$1" + + awk -v prefix="${prefix}-v" 'index($0, prefix) == 1 { + suffix = substr($0, length(prefix) + 1) + if (suffix ~ /^[0-9]+$/) { + print suffix + } + }' | sort -n | tail -1 +} + +resolve_k8s_version() { + local version="$1" + local provider="$2" + + if [[ "$version" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo "$version" + return + fi + + local latest + if [[ "$provider" == "docker" ]]; then + require_command jq + if ! latest=$(curl -sfL "https://registry.hub.docker.com/v2/repositories/kindest/node/tags?page_size=100&name=v${version}." 2>/dev/null \ + | jq -r '.results[].name' 2>/dev/null \ + | grep -E "^v${version}\.[0-9]+$" \ + | sed 's/^v//' \ + | sort -V \ + | tail -1); then + echo "Failed to resolve the latest kindest/node patch version for Kubernetes ${version}." >&2 + exit 1 + fi + elif ! latest=$(curl -sfL "https://dl.k8s.io/release/stable-${version}.txt" 2>/dev/null | sed 's/^v//'); then + echo "Failed to resolve the latest patch version for Kubernetes ${version}." >&2 + exit 1 + fi + + if [[ -n "$latest" && "$latest" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo "$latest" + return + fi + + echo "Could not resolve a stable patch version for Kubernetes ${version}." >&2 + exit 1 +} + +# ============================================ +# Argument parsing +# ============================================ + +usage() { + sed -n '3,/^$/s/^# \?//p' "$0" + exit "${1:-1}" +} + +parse_args() { + while [[ $# -gt 0 ]]; do + case "$1" in + --version) [[ $# -ge 2 ]] || { echo "Error: $1 requires a value" >&2; usage; }; K8S_VERSION="$2"; shift 2 ;; + --cs-version) [[ $# -ge 2 ]] || { echo "Error: $1 requires a value" >&2; usage; }; CS_VERSION="$2"; shift 2 ;; + --namespace) [[ $# -ge 2 ]] || { echo "Error: $1 requires a value" >&2; usage; }; NAMESPACE="$2"; shift 2 ;; + --cluster-name) [[ $# -ge 2 ]] || { echo "Error: $1 requires a value" >&2; usage; }; CLUSTER_NAME="$2"; shift 2 ;; + --cluster-only) CLUSTER_ONLY=true; shift ;; + --clusterstack-only) CLUSTERSTACK_ONLY=true; shift ;; + -h|--help) usage 0 ;; + -*) echo "Unknown option: $1" >&2; usage ;; + *) + if [[ -z "$BASE_DIR" ]]; then + BASE_DIR="$1"; shift + else + echo "Unexpected argument: $1" >&2; usage + fi + ;; + esac + done +} + +resolve_base_dir() { + local dir="$1" + + if [[ -z "$dir" ]]; then + dir="providers/${PROVIDER:-openstack}/${CLUSTER_STACK:-scs}" + fi + + if [[ ! "$dir" = /* ]]; then + dir="$REPO_ROOT/$dir" + fi + + echo "$dir" +} + +BASE_DIR="" +K8S_VERSION="" +CS_VERSION="" +NAMESPACE="cluster" +CLUSTER_NAME="cs-cluster" +CLUSTER_ONLY=false +CLUSTERSTACK_ONLY=false + +parse_args "$@" + +if [[ "$CLUSTER_ONLY" == "true" && "$CLUSTERSTACK_ONLY" == "true" ]]; then + echo "Error: --cluster-only and --clusterstack-only are mutually exclusive" >&2 + usage +fi + +require_command yq +require_command curl + +BASE_DIR="$(resolve_base_dir "$BASE_DIR")" + +if [[ -z "$K8S_VERSION" ]]; then + echo "Error: --version is required" >&2 + usage +fi + +# Resolve version directory +K8S_DASH="${K8S_VERSION//./-}" +VERSION_DIR="$BASE_DIR/$K8S_DASH" + +if [[ ! -d "$VERSION_DIR" ]]; then + echo "Version directory not found: $VERSION_DIR" >&2 + echo "Available:" >&2 + ls -d "$BASE_DIR"/1-*/ 2>/dev/null | sed 's|.*/||; s|/$||; s/^/ /' >&2 + exit 1 +fi + +CSCTL_YAML="$VERSION_DIR/csctl.yaml" +if [[ ! -f "$CSCTL_YAML" ]]; then + echo "csctl.yaml not found in: $VERSION_DIR" >&2 + exit 1 +fi + +# ============================================ +# Read stack configuration +# ============================================ + +PROVIDER=$(yq -r '.config.provider.type' "$CSCTL_YAML") +CLUSTER_STACK=$(yq -r '.config.clusterStackName' "$CSCTL_YAML") +K8S_VERSION_RAW=$(yq -r '.config.kubernetesVersion | sub("^v","")' "$CSCTL_YAML") + +K8S_FULL=$(resolve_k8s_version "$K8S_VERSION_RAW" "$PROVIDER") + +# Auto-detect CS version (if not specified) +# Priority: 1. local .release/ build output 2. OCI registry 3. fall back to v1 +if [[ -z "$CS_VERSION" ]]; then + RELEASE_DIR="${OUTPUT_DIR:-$REPO_ROOT/.release}" + TAG_PREFIX="${PROVIDER}-${CLUSTER_STACK}-${K8S_DASH}" + + # 1. Check .release/ for a matching build (newest first by mtime) + LATEST_BUILD="" + if [[ -d "$RELEASE_DIR" ]]; then + LATEST_BUILD=$(ls -dt "$RELEASE_DIR/${TAG_PREFIX}"-*/metadata.yaml 2>/dev/null | head -1 || true) + fi + + if [[ -n "$LATEST_BUILD" ]]; then + CS_VERSION=$(yq -r '.versions.clusterStack' "$LATEST_BUILD") + echo "# Auto-detected CS version: ${CS_VERSION} (from $(dirname "$LATEST_BUILD"))" >&2 + # 2. Try OCI registry + elif [[ -n "${OCI_REGISTRY:-}" && -n "${OCI_REPOSITORY:-}" ]] && command -v oras >/dev/null 2>&1; then + LATEST=$(oras repo tags "${OCI_REGISTRY}/${OCI_REPOSITORY}" 2>/dev/null | \ + extract_latest_release_number "$TAG_PREFIX" || echo "") + if [[ -n "$LATEST" ]]; then + CS_VERSION="v${LATEST}" + echo "# Auto-detected CS version: ${CS_VERSION} (from ${OCI_REGISTRY}/${OCI_REPOSITORY})" >&2 + else + CS_VERSION="v1" + echo "# No published versions found, using default: ${CS_VERSION}" >&2 + fi + else + CS_VERSION="v1" + echo "# No .release/ build or OCI registry available, using default: ${CS_VERSION}" >&2 + fi +fi + +# ============================================ +# Generate ClusterStack resource +# ============================================ + +if [[ "$CLUSTER_ONLY" != "true" ]]; then + # The CSO matches spec.channel against the channel of the release version: + # v1, v2, ... are "stable", development builds (v0-ttl.) use "custom". + CHANNEL="custom" + if [[ "$CS_VERSION" =~ ^v[0-9]+$ ]]; then + CHANNEL="stable" + fi + + cat < list: - """ - Read supported versions from file for output or further usage inside this script. - - Parameters: - None - - Returns: - List of supported versions - """ - logger.info("Loading supported versions.") - version_file = SOURCE_PATH.joinpath("versions.yaml") - with open(version_file, encoding="utf-8") as stream: - try: - result = yaml.safe_load(stream) - except yaml.YAMLError as exc: - print(exc) - - return result - - -def get_dash_version(version: str) -> str: - """ - Helper function to convert version from dotted to separated by hyphen. (1.27.14 -> 1-27) - Parameters: - version (string): String containing the full semver version. - - Returns: - String with shortened version separated by a hypgen. - """ - return "-".join(version.split(".")[0:2]) - - -def create_output_dir(version: str) -> PosixPath: - """ - Prepare output directory by creating it and copying the source files over. - This overwrites files existing inside the output directory, as those - should not be edited manually. - - Parameters: - version (string): Semver version string as read from the supported versions list. - - Returns: - PosixPath object of the created directory. - """ - out = get_dash_version(version) - out_dir = DEFAULT_TARGET_PATH.joinpath(out) - - logger.info("Creating output directory at %s", out_dir) - - # TODO: how to handle FileExistsError? - # as the output is being generated, it *should* be safe to overwrite - if out_dir.exists(): - shutil.rmtree(str(out_dir)) - - # Copy whole tree from src dir and remove "versions.yaml" file - shutil.copytree(SOURCE_PATH, out_dir) - out_dir.joinpath("versions.yaml").unlink() - for file in out_dir.joinpath("cluster-addon").rglob("Chart.lock"): - file.unlink() - for folder in out_dir.joinpath("cluster-addon").rglob("charts"): - shutil.rmtree(folder) - - return out_dir - - -def readfile(path: PosixPath): - """ - Helper function to read yaml configuration files. - - Parameters: - path (PosixPath): pathlib object of the file to open. - - Returns: - Content of the yaml configuration file. - """ - # TODO: yaml.safe_load either returns a list or dict, - # depending on the structure of the yaml file. This can be improved / refactored. - with open(path, encoding="utf-8") as stream: - try: - content = yaml.safe_load(stream) - except yaml.YAMLError as exc: - print(exc) - - return content - - -def writefile(path: PosixPath, content): - """ - Helper function to write content to a yaml configuration file. - - Parameters: - path (PosixPath): pathlib object of the target file path. - content: yaml data to be written. This can either be of type list or dict. - - Returns: - None - """ - with open(path, "w", encoding="utf-8") as stream: - yaml.safe_dump(content, stream) - - -def update_cluster_addon( - target: PosixPath, build: bool, build_verbose: bool, **versions -): - """ - Update relevant files inside the cluster-stacks//cluster-addon subdirectory - - Parameters: - target (PosixPath): pathlib object of the relevant file. - build (boolean): Toggle to control if helm dependencies should be build, - build_verbose (boolean): Toggle to control if build output should be printed. - versions (kwargs): Dictionary of version information. - - Returns: - None - """ - logger.info("Updating %s", target) - content = readfile(target) - - for dep in content["dependencies"]: - if dep["name"] == "openstack-cinder-csi": - dep["version"] = versions["cinder_csi"] - - if dep["name"] == "openstack-cloud-controller-manager": - dep["version"] = versions["occm"] - - content["name"] = ( - f"openstack-scs-{get_dash_version(versions['kubernetes'])}-cluster-addon" - ) - - writefile(target, content) - - if build: - logger.info("Building helm dependencies") - cmd = ["helm", "dependency", "build"] - subprocess.run( - cmd, - cwd=str(target).replace("Chart.yaml", ""), - capture_output=build_verbose, - check=False, - ) - - -def update_csctl_conf(target: PosixPath, **versions): - """ - Function to update csctl configuration file. - - Parameters: - target (PosixPath): pathlib object of the relevant file. - versions (kwargs): Dictionary of version information. - - Returns: - None - """ - logger.info("Updating %s", target) - content = readfile(target) - - content["config"]["kubernetesVersion"] = f"v{versions['kubernetes']}" - - writefile(target, content) - - -def update_cluster_class(target: PosixPath, **kwargs): - """ - Update relevant files inside the cluster-stacks//cluster-class subdirectory. - - Parameters: - target (PosixPath): pathlib object of the relevant file. - versions (kwargs): Dictionary of version information. - - Returns: - None - """ - chart_file = target.joinpath("Chart.yaml") - values_file = target.joinpath("values.yaml") - - logger.info("Updating %s", chart_file) - content = readfile(chart_file) - version = get_dash_version(kwargs["kubernetes"]) - content["name"] = f"openstack-scs-{version}-cluster-class" - - writefile(chart_file, content) - - logger.info("Updating %s", values_file) - content = readfile(values_file) - - content["images"]["controlPlane"][ - "name" - ] = f"ubuntu-capi-image-v{kwargs['kubernetes']}" - content["images"]["worker"]["name"] = f"ubuntu-capi-image-v{kwargs['kubernetes']}" - - writefile(values_file, content) - - -def update_node_images(target: PosixPath, **kwargs): - """ - Update relevant files inside the cluster-stacks//node-images subdirectory. - - Parameters: - target (PosixPath): pathlib object of the relevant file. - versions (kwargs): Dictionary of version information. - - Returns: - None - """ - logger.info("Updating %s", target) - content = readfile(target) - - # TODO: can this magic URL be 'removed'? - # pylint: disable=locally-disabled, line-too-long - url = f"https://swift.services.a.regiocloud.tech/swift/v1/AUTH_b182637428444b9aa302bb8d5a5a418c/openstack-k8s-capi-images/ubuntu-2204-kube-v{kwargs['kubernetes'][0:4]}/ubuntu-2204-kube-v{kwargs['kubernetes']}.qcow2" - content["spec"]["resource"]["content"]["download"]["url"] = url - - checksum_url = f"{url}.CHECKSUM" - response = requests.get(checksum_url, timeout=30) - response.raise_for_status() - - checksum_line = response.text.strip() - checksum = checksum_line.split()[0] - - content["spec"]["resource"]["content"]["download"]["hash"]["value"] = checksum - logger.info("Updated checksum: %s", checksum) - - writefile(target, content) - - -if __name__ == "__main__": - LOGFORMAT = "%(asctime)s - %(levelname)s: %(message)s" - logging.basicConfig(level=logging.INFO, encoding="utf-8", format=LOGFORMAT) - # Initialize arg parser - parser = argparse.ArgumentParser() - parser.add_argument( - "-t", - "--target-version", - type=str, - help="Generate files for version specified like 1.XX. See '-l' to list supported versions.", - ) - parser.add_argument( - "-l", "--list", action="store_true", help="List supported versions and exit." - ) - parser.add_argument("--build", action="store_true", help="Build helm dependencies.") - parser.add_argument( - "--build-verbose", action="store_false", help="Show output of helm build" - ) - args = parser.parse_args() - - # Load supported target versions - sup_versions = load_supported_versions() - - if args.list: - print("Supported Kubernetes Versions:") - for v in sup_versions: - print(f"{'.'.join(v['kubernetes'].split('.')[0:2])}") - print("Usage: generate_version.py --target-version VERSION") - sys.exit() - - # filter versions to generate - if args.target_version: - target_versions = [ - v for v in sup_versions if v["kubernetes"].startswith(args.target_version) - ] - else: - target_versions = sup_versions - - for tv in target_versions: - output_dir = create_output_dir(tv["kubernetes"]) - for chart_yaml in output_dir.joinpath("cluster-addon").rglob("Chart.yaml"): - update_cluster_addon( - chart_yaml, - args.build, - args.build_verbose, - **tv, - ) - update_csctl_conf(output_dir.joinpath("csctl.yaml"), **tv) - update_cluster_class(output_dir.joinpath("cluster-class"), **tv) - update_node_images( - output_dir.joinpath("cluster-class", "templates", "image.yaml"), **tv - ) diff --git a/hack/kind-dev.sh b/hack/kind-dev.sh deleted file mode 100755 index 70c2d99e..00000000 --- a/hack/kind-dev.sh +++ /dev/null @@ -1,62 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2023 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -o errexit -set -o pipefail -set -x - -K8S_VERSION=v1.27.2 - -REPO_ROOT=$(git rev-parse --show-toplevel) -cd "${REPO_ROOT}" || exit 1 - -# Creates a kind cluster with the ctlptl tool https://github.com/tilt-dev/ctlptl -function ctlptl_kind-cluster() { - - local CLUSTER_NAME=$1 - local CLUSTER_VERSION=$2 - - cat < flavor (control plane value; workers get an override if different) +# controlPlaneRootDisk/workerRootDisk -> rootDisk (same rule) +# controlPlaneServerGroupID/workerServerGroupID -> serverGroupID (same rule) +# workerAdditionalBlockDevices -> additionalBlockDevices override on every machineDeployment +# (additionalBlockDevices also applies to the control plane in scs) +# apiServerLoadBalancerOctaviaAmphoraAllowedCIDRs -> apiServerAllowedCIDRs +# +# If only the worker variable is set, the control plane keeps the default of the +# scs stack and the worker value is written as override to every machineDeployment. +# +# Not migrated (a warning is printed, the variable is removed): +# workerSecurityGroups, workerSecurityGroupIDs -> use securityGroups / securityGroupIDs +# +# Variables with the same name in scs (e.g. diskPinAZ) are kept as they are. +# +# This script only converts the variables. Changing topology.class and the ClusterStack +# (scs2 -> scs) has to be done separately. +# +# Usage: +# ./hack/migrate-cluster.sh +# ./hack/migrate-cluster.sh --in-place +# +# Without --in-place, the transformed YAML is written to stdout. + +set -euo pipefail + +if [[ $# -lt 1 ]]; then + echo "Usage: $0 [--in-place]" >&2 + exit 1 +fi + +INPUT="$1" +IN_PLACE=false +[[ "${2:-}" == "--in-place" ]] && IN_PLACE=true + +if [[ ! -f "$INPUT" ]]; then + echo "File not found: $INPUT" >&2 + exit 1 +fi + +if ! command -v yq &>/dev/null; then + echo "yq is required but not found" >&2 + exit 1 +fi + +TEMP=$(mktemp) +trap 'rm -f "$TEMP"' EXIT +cp "$INPUT" "$TEMP" + +# Helper: check if a variable exists +has_var() { + local val + val=$(yq -r ".spec.topology.variables[] | select(.name == \"$1\") | .name" "$TEMP" 2>/dev/null) + [[ -n "$val" ]] +} + +# Helper: get the value of a variable as JSON (keeps strings, numbers, lists and objects intact) +get_var_json() { + yq -o=json -I=0 ".spec.topology.variables[] | select(.name == \"$1\") | .value" "$TEMP" +} + +# Helper: delete a variable by name +del_var() { + yq -i "del(.spec.topology.variables[] | select(.name == \"$1\"))" "$TEMP" +} + +# Helper: set a variable (add or update) from a JSON value +set_var() { + local name="$1" value="$2" + if has_var "$name"; then + yq -i "(.spec.topology.variables[] | select(.name == \"$name\")).value = $value" "$TEMP" + else + yq -i ".spec.topology.variables += [{\"name\": \"$name\", \"value\": $value}]" "$TEMP" + fi +} + +# Helper: add a variable override to every machineDeployment +add_worker_override() { + local name="$1" value="$2" + yq -i ".spec.topology.workers.machineDeployments[].variables.overrides |= (. // [])" "$TEMP" + yq -i ".spec.topology.workers.machineDeployments[].variables.overrides += [{\"name\": \"$name\", \"value\": $value}]" "$TEMP" +} + +CHANGES=0 + +# ============================================ +# Rename simple variables +# ============================================ + +if has_var "apiServerLoadBalancerOctaviaAmphoraAllowedCIDRs"; then + val=$(get_var_json "apiServerLoadBalancerOctaviaAmphoraAllowedCIDRs") + del_var "apiServerLoadBalancerOctaviaAmphoraAllowedCIDRs" + set_var "apiServerAllowedCIDRs" "$val" + echo "Renamed: apiServerLoadBalancerOctaviaAmphoraAllowedCIDRs -> apiServerAllowedCIDRs" >&2 + CHANGES=$((CHANGES + 1)) +fi + +# additionalBlockDevices applies to control plane AND workers in scs, workerAdditionalBlockDevices +# only applied to workers. Therefore it is set as override on the machineDeployments. +if has_var "workerAdditionalBlockDevices"; then + val=$(get_var_json "workerAdditionalBlockDevices") + del_var "workerAdditionalBlockDevices" + add_worker_override "additionalBlockDevices" "$val" + echo "Moved: workerAdditionalBlockDevices -> additionalBlockDevices override on the machineDeployments" >&2 + CHANGES=$((CHANGES + 1)) +fi + +# ============================================ +# Unify control plane / worker split variables +# ============================================ + +for pair in \ + "controlPlaneFlavor:workerFlavor:flavor" \ + "controlPlaneRootDisk:workerRootDisk:rootDisk" \ + "controlPlaneServerGroupID:workerServerGroupID:serverGroupID"; do + + cp_var="${pair%%:*}" + rest="${pair#*:}" + worker_var="${rest%%:*}" + new_var="${rest#*:}" + + has_cp=false + has_worker=false + cp_val="" + worker_val="" + + if has_var "$cp_var"; then + cp_val=$(get_var_json "$cp_var") + has_cp=true + fi + if has_var "$worker_var"; then + worker_val=$(get_var_json "$worker_var") + has_worker=true + fi + + if ! $has_cp && ! $has_worker; then + continue + fi + + if $has_cp; then + set_var "$new_var" "$cp_val" + del_var "$cp_var" + fi + + if $has_worker; then + # Workers only need an override if they differ from what the control plane gets (cluster level value or default) + if ! $has_cp || [[ "$cp_val" != "$worker_val" ]]; then + add_worker_override "$new_var" "$worker_val" + fi + del_var "$worker_var" + fi + + echo "Unified: $cp_var + $worker_var -> $new_var" >&2 + CHANGES=$((CHANGES + 1)) +done + +# ============================================ +# Variables without a direct equivalent +# ============================================ + +for old_var in "workerSecurityGroups" "workerSecurityGroupIDs"; do + if has_var "$old_var"; then + echo "WARNING: $old_var is no longer supported. Use securityGroups/securityGroupIDs instead." >&2 + echo " The value has been removed. Please review manually." >&2 + del_var "$old_var" + CHANGES=$((CHANGES + 1)) + fi +done + +# ============================================ +# Output +# ============================================ + +if [[ $CHANGES -eq 0 ]]; then + echo "No migrations needed" >&2 + exit 0 +fi + +echo "Applied $CHANGES migration(s)" >&2 + +if $IN_PLACE; then + cp "$TEMP" "$INPUT" + echo "Updated: $INPUT" >&2 +else + cat "$TEMP" +fi diff --git a/hack/show-matrix.sh b/hack/show-matrix.sh new file mode 100755 index 00000000..544f8190 --- /dev/null +++ b/hack/show-matrix.sh @@ -0,0 +1,256 @@ +#!/usr/bin/env bash +# Display the version matrix for a cluster stack. +# +# Iterates over per-minor-version directories (1-XX/) and shows K8s versions, +# addon versions from Chart.yaml, and addon overrides from csctl.yaml. +# +# Usage: +# ./hack/show-matrix.sh [stack-dir] [options] +# +# Options: +# --markdown Output as a GitHub-flavored Markdown table +# -h, --help Show this help +# +# If is omitted, it is derived from $PROVIDER and $CLUSTER_STACK +# (default: providers/openstack/scs). Relative paths are resolved from the +# repository root. +# +# Environment: +# PROVIDER Provider name (default: openstack) +# CLUSTER_STACK Cluster stack name (default: scs) +# OCI_REGISTRY OCI registry to query for CS versions (optional) +# OCI_REPOSITORY OCI repository to query for CS versions (optional) + +set -euo pipefail + +readonly SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +readonly REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" + +require_command() { + local name="$1" + + if ! command -v "$name" >/dev/null 2>&1; then + echo "$name not found. Please install $name and try again." >&2 + exit 1 + fi +} + +extract_k8s_minor_version() { + echo "$1" | sed -E -n 's/^([0-9]+\.[0-9]+)(\.[0-9]+)?$/\1/p' +} + +extract_latest_release_number() { + local prefix="$1" + + awk -v prefix="${prefix}-v" 'index($0, prefix) == 1 { + suffix = substr($0, length(prefix) + 1) + if (suffix ~ /^[0-9]+$/) { + print suffix + } + }' | sort -n | tail -1 +} + +usage() { + sed -n '3,/^$/s/^# \?//p' "$0" + exit "${1:-1}" +} + +parse_args() { + while [[ $# -gt 0 ]]; do + case "$1" in + --markdown) MARKDOWN=true; shift ;; + -h|--help) usage 0 ;; + -*) echo "Unknown option: $1" >&2; usage ;; + *) + if [[ -z "$BASE_DIR" ]]; then + BASE_DIR="$1"; shift + else + echo "Unexpected argument: $1" >&2; usage + fi + ;; + esac + done +} + +resolve_base_dir() { + local dir="$1" + + if [[ -z "$dir" ]]; then + dir="providers/${PROVIDER:-openstack}/${CLUSTER_STACK:-scs}" + fi + + if [[ ! "$dir" = /* ]]; then + dir="$REPO_ROOT/$dir" + fi + + echo "$dir" +} + +MARKDOWN=false +BASE_DIR="" + +parse_args "$@" + +require_command yq + +BASE_DIR="$(resolve_base_dir "$BASE_DIR")" + +if [[ ! -d "$BASE_DIR" ]]; then + echo "Stack base directory not found: $BASE_DIR" >&2 + exit 1 +fi + +# ============================================ +# Collect data from all version directories +# ============================================ + +# First pass: discover all addon names across all versions +declare -A ALL_ADDON_NAMES +declare -A STACK_ADDONS # key: "1-XX:addon_name" β†’ version + +PROVIDER_NAME="" +STACK_NAME="" + +for version_dir in "$BASE_DIR"/1-*/; do + [[ -d "$version_dir" ]] || continue + local_dir=$(basename "$version_dir") + + csctl_yaml="$version_dir/csctl.yaml" + [[ -f "$csctl_yaml" ]] || continue + + if [[ -z "$PROVIDER_NAME" ]]; then + PROVIDER_NAME=$(yq -r '.config.provider.type' "$csctl_yaml") + STACK_NAME=$(yq -r '.config.clusterStackName' "$csctl_yaml") + fi + + # Collect chart dependency versions + for chart_file in "$version_dir"/cluster-addon/*/Chart.yaml; do + [[ -f "$chart_file" ]] || continue + num_deps=$(yq '.dependencies | length' "$chart_file" 2>/dev/null || echo "0") + for ((i=0; i/dev/null 2>&1 && echo "true" || echo "false") + if [[ "$has_addons" == "true" ]]; then + addon_keys=$(yq -r '.addons | keys | .[]' "$csctl_yaml") + for key in $addon_keys; do + value=$(yq -r ".addons.\"${key}\"" "$csctl_yaml") + # Map short names to chart names for display + case "$key" in + ccm) chart_key="openstack-cloud-controller-manager" ;; + csi) chart_key="openstack-cinder-csi" ;; + *) chart_key="$key" ;; + esac + ALL_ADDON_NAMES["$chart_key"]=1 + # Mark with range (overrides chart default) + STACK_ADDONS["${local_dir}:${chart_key}"]="$value" + done + fi +done + +if [[ -z "$PROVIDER_NAME" ]]; then + echo "No valid version directories found in: $BASE_DIR" >&2 + exit 1 +fi + +# Sort addon names +SORTED_ADDONS=($(echo "${!ALL_ADDON_NAMES[@]}" | tr ' ' '\n' | sort)) + +# ============================================ +# Collect row data +# ============================================ + +declare -a ROW_VERSIONS=() +declare -a ROW_K8S=() +declare -a ROW_CS=() +declare -A ROW_ADDON_VERSIONS # key: "row_idx:addon_name" β†’ version + +ROW_IDX=0 +for version_dir in "$BASE_DIR"/1-*/; do + [[ -d "$version_dir" ]] || continue + local_dir=$(basename "$version_dir") + csctl_yaml="$version_dir/csctl.yaml" + [[ -f "$csctl_yaml" ]] || continue + + k8s_version=$(yq -r '.config.kubernetesVersion | sub("^v","")' "$csctl_yaml") + k8s_short=$(extract_k8s_minor_version "$k8s_version") + k8s_dash="${k8s_short//./-}" + + # Query OCI for CS version + CS_VERSION="-" + if [[ -n "${OCI_REGISTRY:-}" && -n "${OCI_REPOSITORY:-}" ]] && command -v oras >/dev/null 2>&1; then + TAG_PREFIX="${PROVIDER_NAME}-${STACK_NAME}-${k8s_dash}" + LATEST=$(oras repo tags "${OCI_REGISTRY}/${OCI_REPOSITORY}" 2>/dev/null | \ + extract_latest_release_number "$TAG_PREFIX" || echo "") + [[ -n "$LATEST" ]] && CS_VERSION="v${LATEST}" + fi + + ROW_VERSIONS+=("$local_dir") + ROW_K8S+=("$k8s_version") + ROW_CS+=("$CS_VERSION") + + for addon in "${SORTED_ADDONS[@]}"; do + ver="${STACK_ADDONS["${local_dir}:${addon}"]:-"-"}" + ROW_ADDON_VERSIONS["${ROW_IDX}:${addon}"]="$ver" + done + ((ROW_IDX++)) || true +done + +# Short display name for an addon +addon_short() { + echo "$1" | sed 's/openstack-cloud-controller-manager/os-ccm/; s/openstack-cinder-csi/os-csi/' +} + +# ============================================ +# Print the matrix +# ============================================ + +if [[ "$MARKDOWN" == true ]]; then + # --- Markdown table output --- + HEADER="| Version | K8s | CS Version" + SEPARATOR="|---------|-----|----------" + for addon in "${SORTED_ADDONS[@]}"; do + HEADER+=" | $(addon_short "$addon")" + SEPARATOR+="|---------" + done + HEADER+=" |" + SEPARATOR+="|" + echo "$HEADER" + echo "$SEPARATOR" + + for ((i=0; i ... +# hack/update-gateway-api-crds.sh v1.4.1 providers/docker/scs/1-35 +# hack/update-gateway-api-crds.sh v1.6.3 providers/openstack/scs/1-35 providers/openstack/scs/1-36 +# +# The CRDs are those the Cilium Gateway API support needs: the standard channel +# plus TLSRoute, which the hosted control plane provider uses for TLS +# passthrough (experimental channel before Gateway API v1.5, standard channel +# since then). Pick the Gateway API version that the Cilium version of the cni +# addon supports, see +# https://docs.cilium.io/en/stable/network/servicemesh/gateway-api/gateway-api/ +# +# Cilium 1.19: Gateway API v1.4.1, Cilium 1.20: Gateway API v1.6.3 + +set -euo pipefail + +cd "$(dirname "$0")/.." + +if [[ $# -lt 2 || "$1" != v* ]]; then + echo "Usage: $0 ..." >&2 + exit 1 +fi + +VERSION="$1" +shift + +BASE="https://raw.githubusercontent.com/kubernetes-sigs/gateway-api/${VERSION}/config/crd" + +# TLSRoute is part of the standard channel since Gateway API v1.5, before that only the experimental channel has it +tlsroute_channel=experimental +if [[ "$(printf '%s\n' v1.5.0 "$VERSION" | sort -V | head -n1)" == "v1.5.0" ]]; then + tlsroute_channel=standard +fi + +CRDS=( + standard/gateway.networking.k8s.io_gatewayclasses.yaml + standard/gateway.networking.k8s.io_gateways.yaml + standard/gateway.networking.k8s.io_grpcroutes.yaml + standard/gateway.networking.k8s.io_httproutes.yaml + standard/gateway.networking.k8s.io_referencegrants.yaml + "${tlsroute_channel}/gateway.networking.k8s.io_tlsroutes.yaml" +) + +# Cilium 1.20 (Gateway API v1.6.3) does not enable the Gateway API without BackendTLSPolicy, Cilium 1.19 does not use it +if [[ "$tlsroute_channel" == "standard" ]]; then + CRDS+=(standard/gateway.networking.k8s.io_backendtlspolicies.yaml) +fi + +targets=() +for dir in "$@"; do targets+=("${dir%/}/cluster-addon/gateway-api"); done + +tmp=$(mktemp -d) +trap 'rm -rf "$tmp"' EXIT + +out="$tmp/crds.yaml" +{ + echo "# Gateway API CRDs ${VERSION} (standard channel, TLSRoute from the ${tlsroute_channel} channel)." + echo "# Generated by hack/update-gateway-api-crds.sh, do not edit." +} > "$out" + +for crd in "${CRDS[@]}"; do + echo "Fetching ${crd}" >&2 + { + echo "---" + curl -fsSL "${BASE}/${crd}" + } >> "$out" +done + +# The file is rendered by helm template, a literal {{ would be interpreted +if grep -q '{{' "$out"; then + echo "CRDs contain '{{', rendering them as a template would fail" >&2 + exit 1 +fi + +for target in "${targets[@]}"; do + mkdir -p "$target/templates" + cp "$tmp/crds.yaml" "$target/templates/crds.yaml" + cat > "$target/Chart.yaml" <&2 +done diff --git a/hack/update.sh b/hack/update.sh new file mode 100755 index 00000000..4daeb5b1 --- /dev/null +++ b/hack/update.sh @@ -0,0 +1,728 @@ +#!/usr/bin/env bash +set -euo pipefail + +# update.sh β€” Unified version and addon updater for Cluster Stacks +# +# Updates Kubernetes patch versions in csctl.yaml and Helm chart dependencies +# in cluster-addon/*/Chart.yaml. Each per-minor-version directory (1-XX/) is +# self-contained with its own csctl.yaml. +# +# For OpenStack stacks, the `versions` subcommand also maintains +# image-manager.yaml in the stack base directory (e.g., providers/openstack/scs/). +# This file uses merge semantics: new versions are added, existing ones updated, +# but versions for removed minors are never deleted β€” protecting images still +# used by running clusters. +# +# Usage: +# ./hack/update.sh [versions|addons] [stack-dir] [options] +# +# Subcommands: +# versions Update K8s patch versions in csctl.yaml + image-manager.yaml +# addons Update Helm chart dependencies in cluster-addon/*/Chart.yaml, +# the Gateway API CRDs follow the Cilium version +# (none) Run both: versions first, then addons +# +# The is the base directory containing per-minor-version subdirs +# (e.g., providers/openstack/scs). If omitted, it is derived from $PROVIDER +# and $CLUSTER_STACK (default: providers/openstack/scs). Relative paths are +# resolved from the repository root. +# +# Options: +# --dry-run Preview changes without modifying files +# --all Run against all stacks in providers/*/*/ +# --check-image-repo Resolve latest patch via IMAGE_BASE_URL/last- first +# -h, --help Show this help +# +# Environment: +# PROVIDER Provider name (default: openstack) +# CLUSTER_STACK Cluster stack name (default: scs) +# IMAGE_BASE_URL Base URL for CAPI images (default: https://nbg1.your-objectstorage.com/...) +# IMAGE_VISIBILITY Image visibility in image-manager.yaml (default: private) + +readonly SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +readonly REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" +readonly YQ_VERSION="$(yq --version 2>/dev/null || true)" + +require_command() { + local name="$1" + + if ! command -v "$name" >/dev/null 2>&1; then + echo "$name not found. Please install $name and try again." >&2 + exit 1 + fi +} + +extract_k8s_minor_version() { + echo "$1" | sed -E -n 's/^([0-9]+\.[0-9]+)(\.[0-9]+)?$/\1/p' +} + +extract_k8s_minor_number() { + echo "$1" | sed -E -n 's/^v?[0-9]+\.([0-9]+)(\.[0-9]+)?$/\1/p' +} + +yq_edit_in_place() { + local expression="$1" + local file="$2" + + if [[ "$YQ_VERSION" == *"https://github.com/mikefarah/yq/"* ]]; then + yq -i "$expression" "$file" + else + yq -y -i "$expression" "$file" + fi +} + +require_versions_tools() { + require_command curl + require_command jq +} + +require_addons_tools() { + require_command helm + require_command jq +} + +# ============================================ +# Defaults +# ============================================ + +SUBCOMMAND="" # versions, addons, or "" (both) +BASE_DIR="" +DRY_RUN=false +RUN_ALL=false +USE_IMAGE_REPO_LATEST=false + +# Image-manager defaults (for OpenStack image manifest generation) +IMAGE_BASE_URL="${IMAGE_BASE_URL:-https://nbg1.your-objectstorage.com/osism/openstack-k8s-capi-images}" +IMAGE_VISIBILITY="${IMAGE_VISIBILITY:-private}" + +# ============================================ +# Ubuntu version mapping +# ============================================ + +# K8s 1.32 and earlier β†’ Ubuntu 22.04 (2204) +# K8s 1.33 and later β†’ Ubuntu 24.04 (2404) +ubuntu_for_minor() { + local minor="$1" + if [[ "$minor" -le 32 ]]; then + echo "2204" + else + echo "2404" + fi +} + +# ============================================ +# Output helpers +# ============================================ + +info() { echo " $*"; } +ok() { echo " βœ“ $*"; } +warn() { echo " ! $*"; } +change() { echo " β†’ $*"; } + +# ============================================ +# Argument parsing +# ============================================ + +usage() { + sed -n '3,/^$/s/^# \?//p' "$0" + exit "${1:-1}" +} + +parse_args() { + while [[ $# -gt 0 ]]; do + case "$1" in + versions|addons) + if [[ -n "$SUBCOMMAND" ]]; then + echo "Error: multiple subcommands given" >&2; usage + fi + SUBCOMMAND="$1"; shift + ;; + --dry-run) DRY_RUN=true; shift ;; + --all) RUN_ALL=true; shift ;; + --check-image-repo) USE_IMAGE_REPO_LATEST=true; shift ;; + -h|--help) usage 0 ;; + -*) echo "Unknown option: $1" >&2; usage ;; + *) + if [[ -z "$BASE_DIR" ]]; then + BASE_DIR="$1"; shift + else + echo "Unexpected argument: $1" >&2; usage + fi + ;; + esac + done +} + +resolve_base_dir() { + local dir="$1" + + if [[ -z "$dir" ]]; then + dir="providers/${PROVIDER:-openstack}/${CLUSTER_STACK:-scs}" + fi + + # Resolve relative path + if [[ ! "$dir" = /* ]]; then + dir="$REPO_ROOT/$dir" + fi + + echo "$dir" +} + +# Detect provider from stack dir path +detect_provider() { + local dir="$1" + local rel="${dir#"$REPO_ROOT"/}" + echo "$rel" | cut -d/ -f2 +} + +# ============================================ +# K8s version resolution +# ============================================ + +# Resolve latest stable patch for a given minor version (e.g., "34" β†’ "1.34.4") +resolve_latest_patch() { + local minor="$1" + local provider="${2:-openstack}" + + if [[ "$provider" == "docker" ]]; then + curl -sfL "https://registry.hub.docker.com/v2/repositories/kindest/node/tags?page_size=100&name=v1.${minor}." 2>/dev/null \ + | jq -r '.results[].name' 2>/dev/null \ + | grep -E "^v1\.${minor}\.[0-9]+$" \ + | sed 's/^v//' \ + | sort -V \ + | tail -1 + return + fi + + if [[ "$USE_IMAGE_REPO_LATEST" == "true" ]]; then + local latest_available + if latest_available=$(curl -sfL "${IMAGE_BASE_URL}/last-1.${minor}" 2>/dev/null | awk '{print $NF}' | sed -E -n 's#.*/ubuntu-[0-9]+-kube-v([0-9]+\.[0-9]+\.[0-9]+)\.qcow2#\1#p'); then + if [[ -n "$latest_available" && "$latest_available" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo "$latest_available" + return + fi + fi + fi + + local version + if ! version=$(curl -sfL "https://dl.k8s.io/release/stable-1.${minor}.txt" 2>/dev/null | sed 's/^v//'); then + return 1 + fi + if [[ -n "$version" && "$version" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo "$version" + fi +} + +# ============================================ +# Helm repo URL maps for K8s-tied addons +# ============================================ + +declare -A ADDON_HELM_REPOS=( + ["openstack-cloud-controller-manager"]="https://kubernetes.github.io/cloud-provider-openstack" + ["openstack-cinder-csi"]="https://kubernetes.github.io/cloud-provider-openstack" +) + +declare -A ADDON_SHORT_TO_CHART=( + ["ccm"]="openstack-cloud-controller-manager" + ["csi"]="openstack-cinder-csi" +) + +# ============================================ +# Image-manager update (OpenStack only) +# ============================================ + +# Update providers///image-manager.yaml with merge semantics: +# - New minor versions are added +# - Existing minor versions are updated (new patch replaces old) +# - Minor versions no longer in 1-*/ dirs are preserved (never removed) +# +# Args: $1 = base_dir, $2 = name of associative array with resolved patches +update_image_manager() { + local base_dir="${1%/}" # strip trailing slash + local -n patches_ref=$2 # nameref to associative array + + local image_file="$base_dir/image-manager.yaml" + echo "Updating image-manager manifest: $image_file" + + # Build map of existing versions from file (if present), keyed by minor + # Each entry: "version|url|checksum" + declare -A existing_versions=() + if [[ -f "$image_file" ]]; then + local num_versions + num_versions=$(yq '.images[0].versions | length' "$image_file" 2>/dev/null || echo "0") + for ((i=0; i "$image_file" </dev/null || echo "0") + for ((i=0; i/dev/null 2>&1; then + echo "Failed to add Helm repo $dep_name ($dep_repo)." >&2 + exit 1 + fi + repos_added["$dep_name"]="$dep_repo" + fi + done + done + done + + if [[ ${#repos_added[@]} -gt 0 ]]; then + info "Updating Helm repos..." + if ! helm repo update > /dev/null 2>&1; then + echo "Failed to update Helm repositories." >&2 + exit 1 + fi + fi + echo "" + + local total_updates=0 + + for version_dir in "$base_dir"/1-*/; do + [[ -d "$version_dir" ]] || continue + [[ -f "$version_dir/FROZEN" ]] && continue + local dir_name + dir_name=$(basename "$version_dir") + + echo "=== $dir_name ===" + + local cilium_version="" + for addon_dir in "$version_dir"/cluster-addon/*/; do + [[ -f "$addon_dir/Chart.yaml" ]] || continue + local addon_name chart_file + addon_name=$(basename "$addon_dir") + chart_file="$addon_dir/Chart.yaml" + + local num_deps + num_deps=$(yq '.dependencies | length' "$chart_file" 2>/dev/null || echo "0") + [[ "$num_deps" == "0" || "$num_deps" == "null" ]] && continue + + for ((i=0; i/dev/null) + if [[ -n "$range" && "$range" != "null" ]]; then + is_tied=true + fi + fi + done + fi + + # Get latest version + local latest_version="" + if [[ "$is_tied" == "true" ]]; then + # For K8s-tied addons, derive version prefix from csctl.yaml range + local addon_range + addon_range="" + for short_name in "${!ADDON_SHORT_TO_CHART[@]}"; do + if [[ "${ADDON_SHORT_TO_CHART[$short_name]}" == "$dep_name" ]]; then + addon_range=$(yq -r ".addons.\"${short_name}\" // \"\"" "$csctl_yaml" 2>/dev/null) + break + fi + done + # Extract prefix: "2.35.x" β†’ "2.35.", "3.0.x" β†’ "3.0." + local version_prefix + version_prefix=$(echo "$addon_range" | sed -E 's/\.[^.]*$/\./') + latest_version=$(helm search repo "$dep_name/$dep_name" --versions -o json 2>/dev/null | \ + jq -r --arg prefix "$version_prefix" \ + '[.[] | select(.version | startswith($prefix))] | .[0].version // empty' 2>/dev/null) || true + else + latest_version=$(helm search repo "$dep_name/$dep_name" -o json 2>/dev/null | \ + jq -r '.[0].version // empty' 2>/dev/null) || true + fi + + if [[ -z "$latest_version" ]]; then + warn "$dep_name: could not query upstream" + query_failures=$((query_failures + 1)) + continue + fi + + if [[ "$dep_name" == "cilium" ]]; then + cilium_version="$latest_version" + fi + + if [[ "$current_version" == "$latest_version" ]]; then + ok "$dep_name: $current_version (up to date)" + continue + fi + + change "$dep_name: $current_version β†’ $latest_version" + total_updates=$((total_updates + 1)) + + if [[ "$DRY_RUN" != "true" ]]; then + yq_edit_in_place ".dependencies[$i].version = \"$latest_version\"" "$chart_file" + info "Updated $chart_file" + fi + done + done + + # The vendored Gateway API CRDs follow the Cilium version + local gateway_chart="${version_dir}cluster-addon/gateway-api/Chart.yaml" + if [[ -f "$gateway_chart" && -n "$cilium_version" ]]; then + local gateway_version current_gateway + current_gateway=$(yq -r '.appVersion' "$gateway_chart") + gateway_version=$(gateway_api_version_for_cilium "$cilium_version" || true) + if [[ -z "$gateway_version" ]]; then + warn "gateway-api: could not determine the version for Cilium $cilium_version" + query_failures=$((query_failures + 1)) + elif [[ "$current_gateway" != "$gateway_version" ]]; then + change "gateway-api: $current_gateway β†’ $gateway_version" + total_updates=$((total_updates + 1)) + if [[ "$DRY_RUN" != "true" ]]; then + "$SCRIPT_DIR/update-gateway-api-crds.sh" "$gateway_version" "$version_dir" + fi + fi + fi + echo "" + done + + if [[ "$query_failures" -gt 0 ]]; then + echo "$query_failures addon query failure(s) encountered." >&2 + exit 1 + fi + + if [[ "$DRY_RUN" == "true" ]]; then + if [[ "$total_updates" -gt 0 ]]; then + echo "$total_updates addon update(s) available (dry-run: no changes written)." + else + echo "All addons are up to date." + fi + elif [[ "$total_updates" -gt 0 ]]; then + echo "Applied $total_updates addon update(s)." + else + echo "All addons are up to date." + fi +} + +# ============================================ +# --all mode: iterate all stacks +# ============================================ + +run_all() { + local subcommand="$1" + local found=false + + # Find base dirs: providers/// that contain 1-*/ subdirs + for base_dir in "$REPO_ROOT"/providers/*/*/; do + # Must have at least one 1-*/ subdir + ls "$base_dir"/1-*/csctl.yaml >/dev/null 2>&1 || continue + + found=true + echo "==========================================" + echo "$base_dir" + echo "==========================================" + + if [[ -z "$subcommand" ]]; then + cmd_versions "$base_dir" + echo "" + cmd_addons "$base_dir" + elif [[ "$subcommand" == "versions" ]]; then + cmd_versions "$base_dir" + elif [[ "$subcommand" == "addons" ]]; then + cmd_addons "$base_dir" + fi + + echo "" + done + + if [[ "$found" == false ]]; then + echo "No stacks found in providers/*/*/" >&2 + exit 1 + fi +} + +# ============================================ +# Main +# ============================================ + +main() { + parse_args "$@" + + require_command yq + + if [[ -z "$SUBCOMMAND" || "$SUBCOMMAND" != "addons" ]]; then + require_versions_tools + fi + + if [[ -z "$SUBCOMMAND" || "$SUBCOMMAND" != "versions" ]]; then + require_addons_tools + fi + + if [[ "$RUN_ALL" == true ]]; then + run_all "$SUBCOMMAND" + return + fi + + BASE_DIR=$(resolve_base_dir "$BASE_DIR") + + if [[ ! -d "$BASE_DIR" ]]; then + echo "Stack base directory not found: $BASE_DIR" >&2 + exit 1 + fi + + if [[ -z "$SUBCOMMAND" ]]; then + cmd_versions "$BASE_DIR" + echo "" + cmd_addons "$BASE_DIR" + elif [[ "$SUBCOMMAND" == "versions" ]]; then + cmd_versions "$BASE_DIR" + elif [[ "$SUBCOMMAND" == "addons" ]]; then + cmd_addons "$BASE_DIR" + fi +} + +main "$@" diff --git a/justfile b/justfile new file mode 100644 index 00000000..b760810e --- /dev/null +++ b/justfile @@ -0,0 +1,140 @@ +# Cluster Stacks build system, see docs/build-system.md +# +# The stack is selected with PROVIDER and CLUSTER_STACK (default: openstack/scs), either from the +# environment, from a .env file or on the command line: `just CLUSTER_STACK=scs2 matrix`. +# +# Wherever a Kubernetes version is expected, a bare minor version as first argument is +# shorthand for `--version`: `just build 1.35` == `just build --version 1.35`. + +set dotenv-load +set positional-arguments +set shell := ["bash", "-euo", "pipefail", "-c"] + +export PROVIDER := env("PROVIDER", "openstack") +export CLUSTER_STACK := env("CLUSTER_STACK", "scs") + +container_image := "cluster-stack-tools" + +# Show available recipes +default: + @just --list --unsorted + +# ============================================ +# Build & publish +# ============================================ + +# Build release artifacts (e.g. `just build 1.35`, `just build --all`) +build *args: + @if [[ "${1:-}" =~ ^[0-9]+\.[0-9]+$ ]]; then set -- --version "$@"; fi; ./hack/build.sh "$@" + +# Build and publish to the OCI registry (e.g. `just publish 1.35`, `just publish --all`) +publish *args: + @if [[ "${1:-}" =~ ^[0-9]+\.[0-9]+$ ]]; then set -- --version "$@"; fi; ./hack/build.sh --publish "$@" + +# Build, publish and print the ClusterStack resource (e.g. `just dev 1.35 | kubectl apply -f -`) +dev version *args: + #!/usr/bin/env bash + set -euo pipefail + version="$1"; shift + ./hack/build.sh --publish --version "$version" "$@" >&2 + echo "" >&2 + echo "================================================================" >&2 + echo "ClusterStack resource (pipe to kubectl apply -f -)" >&2 + echo "================================================================" >&2 + ./hack/generate-resources.sh --version "$version" --clusterstack-only + +# Install or upgrade the Cluster Stack Operator (CSO_CHART, CSO_IMAGE_REPOSITORY, CSO_IMAGE_TAG to override) +install-cso: + ./hack/build.sh --install-cso + +# Remove build artifacts +clean: + rm -rf .release + @echo "Cleaned .release" + +# ============================================ +# Maintenance +# ============================================ + +# Update Kubernetes patch versions and/or addons (e.g. `just update versions --all --dry-run`) +update *args: + ./hack/update.sh "$@" + +# Scaffold a new Kubernetes minor version for all stacks that are not deprecated (e.g. `just add-version 1.37`) +add-version version *args: + ./hack/add-version.sh "$@" + +# ============================================ +# Resources & docs +# ============================================ + +# Generate ClusterStack and Cluster manifests (e.g. `just generate-resources 1.35`) +generate-resources *args: + @if [[ "${1:-}" =~ ^[0-9]+\.[0-9]+$ ]]; then set -- --version "$@"; fi; ./hack/generate-resources.sh "$@" + +# Generate only the ClusterStack manifest +generate-clusterstack *args: + @if [[ "${1:-}" =~ ^[0-9]+\.[0-9]+$ ]]; then set -- --version "$@"; fi; ./hack/generate-resources.sh --clusterstack-only "$@" + +# Generate only the Cluster manifest +generate-cluster *args: + @if [[ "${1:-}" =~ ^[0-9]+\.[0-9]+$ ]]; then set -- --version "$@"; fi; ./hack/generate-resources.sh --cluster-only "$@" + +# Generate ORC Image (default) or image-manager manifests (e.g. `just generate-image-manifests 1.35`) +generate-image-manifests *args: + @if [[ "${1:-}" =~ ^[0-9]+\.[0-9]+(\.[0-9]+)?$ ]]; then set -- --version "$@"; fi; ./hack/generate-image-manifests.sh "$@" + +# Convert the variables of a Cluster manifest from scs2 to scs (e.g. `just migrate-cluster cluster.yaml`) +migrate-cluster *args: + ./hack/migrate-cluster.sh "$@" + +# Show the version matrix (`--markdown` for a table) +matrix *args: + ./hack/show-matrix.sh "$@" + +# Update the generated sections (version matrix, variables) of docs/providers//-configuration.md +generate-docs: + #!/usr/bin/env bash + set -euo pipefail + for stack_base in providers/*/*/; do + stack_base="${stack_base%/}" + provider=$(basename "$(dirname "$stack_base")") + stack=$(basename "$stack_base") + latest=$(find "$stack_base" -mindepth 1 -maxdepth 1 -type d -name '1-*' | sort -V | tail -1) + [[ -n "$latest" ]] || continue + doc="docs/providers/${provider}/${stack}-configuration.md" + if [[ ! -f "$doc" ]] || ! grep -q '