From 7eb31d62a17b5231750c0bb093ea59bb7c150e05 Mon Sep 17 00:00:00 2001 From: Aleksei Sviridkin Date: Tue, 3 Nov 2026 19:57:51 +0300 Subject: [PATCH] feat(oidc): configure parent k3s authentication after bootstrap Platform OIDC publishes Keycloak without configuring authentication for the parent k3s API. Keep this configuration separate from bootstrap so unavailable issuer discovery cannot prevent the API from starting. Preserve existing API-server arguments and client-certificate access while enabling OIDC after Keycloak is reachable. Assisted-by: LLM Signed-off-by: Aleksei Sviridkin --- .github/workflows/test.yml | 3 + CHANGELOG.rst | 2 + README.md | 35 ++ hack/check-oidc-compat.sh | 30 ++ playbooks/k3s_oidc.yml | 196 +++++++ roles/cozystack/defaults/main.yml | 2 + roles/cozystack/tasks/main.yml | 7 + .../templates/platform-package.yml.j2 | 5 + tests/test-external-ips.yml | 2 + tests/test-master-nodes.yml | 1 + tests/unit/playbooks/test_k3s_oidc.py | 483 ++++++++++++++++++ 11 files changed, 766 insertions(+) create mode 100755 hack/check-oidc-compat.sh create mode 100644 playbooks/k3s_oidc.yml create mode 100644 tests/unit/playbooks/test_k3s_oidc.py diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 03c3dd8..38f05f0 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -52,6 +52,9 @@ jobs: - name: Syntax check RHEL example run: ansible-playbook examples/rhel/prepare-rhel.yml --syntax-check + - name: Check released examples and standalone OIDC compatibility + run: ./hack/check-oidc-compat.sh + - name: Check examples split-invocation contract run: ./hack/check-examples-k3s.sh diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 30f6b44..b04dada 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -5,6 +5,8 @@ cozystack.installer Release Notes Unreleased ========== +- Add opt-in platform OIDC and a separate parent k3s post-install playbook. After Keycloak discovery and signing keys are reachable from every server, the playbook writes a persistent additive drop-in and restarts changed servers one at a time. The generic role does not change host API authentication. Disabling the option leaves an existing API drop-in unmanaged. Until the next release, the OIDC workflow requires the collection from main or the current source checkout; the example pipelines keep their release pins. + - The role docs list the minimal platform variants for ``cozystack_platform_variant``: ``isp-slim``, ``isp-slim-generic`` and ``isp-hosted-slim``. They need a diff --git a/README.md b/README.md index 9f23e9a..9d2c549 100644 --- a/README.md +++ b/README.md @@ -380,6 +380,7 @@ Both stages are handled automatically by the `cozystack` role. | Playbook | Description | | --- | --- | | `cozystack.installer.site` | Install Cozystack on `server[0]` | +| `cozystack.installer.k3s_oidc` | Configure opt-in OIDC on parent k3s servers after Keycloak is reachable | ## Role: cozystack.installer.cozystack @@ -409,6 +410,7 @@ Runs on `server[0]` only. | `cozystack_create_platform_package` | `true` | Create Platform Package CR after install | | `cozystack_platform_variant` | `isp-full-generic` | Platform variant: default, isp-full, isp-hosted, isp-full-generic. Cozystack releases that include cozystack/cozystack#4595 also accept isp-slim-generic (the minimal variant for k3s), isp-slim (its Talos counterpart) and isp-hosted-slim. | | `cozystack_root_host` | `""` | Domain for Cozystack services (empty = skip publishing) | +| `cozystack_oidc_enabled` | `false` | Unreleased: enable platform OIDC. Configure the parent k3s API separately with `cozystack.installer.k3s_oidc`. Requires `cozystack_root_host`. | | `cozystack_external_ips` | `[]` | List of external IPs for ingress-nginx Service. Required on platforms without a native LB (cloud VMs, bare metal). Each entry must be a valid IPv4/IPv6 address. | | `cozystack_tenant_root_ingress` | `false` | Enable ingress on the root tenant. When `true`, patches the root Tenant CR after Platform Package apply to create IngressClass and ingress-nginx controller. | | `cozystack_pod_cidr` | `10.42.0.0/16` | Pod CIDR for Platform Package | @@ -477,6 +479,39 @@ ansible-playbook --inventory inventory.yml k3s.orchestration.site Keeping them in the inventory is required, not cosmetic. `set_fact`/`group_by` (play scope) live only inside one process, so splitting the run dropped them. A playbook-adjacent `group_vars/all.yml` is not loaded for the imported `k3s.orchestration.site` play either, so under an external inventory (as CI uses) it silently produced an upstream-default k3s. Inventory vars avoid both traps. Custom inventories must carry these k3s vars and the `k3s_cluster` group (`children: server, agent`) themselves. +### Opt-in OIDC for the parent k3s API + +This feature is **Unreleased** and is not included in collection tag `v1.7.0`. It requires the collection from `main` or a build of the current source checkout. The example requirements remain pinned to the released version; their normal pipelines do not import the new playbook. After installing the example requirements, install the unreleased collection explicitly: + +```bash +ansible-galaxy collection install 'git+https://github.com/cozystack/ansible-cozystack.git,main' --force +``` + +Platform OIDC and API-server OIDC are separate settings. Set these variables in the `cluster` inventory group's `vars`: + +```yaml +cozystack_oidc_enabled: true +cozystack_root_host: "example.org" +cozystack_tenant_root_ingress: true +``` + +Run the normal bootstrap/site workflow first, then invoke the parent API configuration separately: + +```bash +ansible-playbook --inventory inventory.yml site.yml +ansible-playbook --inventory inventory.yml cozystack.installer.k3s_oidc +``` + +With the unreleased collection installed, the first command bootstraps k3s without OIDC arguments and enables platform OIDC during Cozystack installation. Storage, ingress routing, DNS and a trusted HTTPS certificate for `keycloak.example.org` must be available. The second command waits for matching issuer discovery and non-empty signing keys from every server before modifying any server. It then writes `/etc/rancher/k3s/config.yaml.d/90-cozystack-oidc.yaml`, restarts changed servers one at a time, and waits for each local API's `/readyz` before continuing. A failed preflight leaves every server unchanged; a failed rollout stops before the next server. If Cozystack and Keycloak are already installed and published, only the second command is needed. + +The issuer defaults to `https://keycloak./realms/cozy`. Set `cozystack_oidc_issuer` to override it when the published Keycloak issuer differs. The client is `kubernetes`, the username claim is `preferred_username`, and the group claim is `groups`. The optional `cozystack_kubeconfig_context` defaults to `default`; the readiness check uses `cozystack_kubeconfig`, defaulting to `/etc/rancher/k3s/k3s.yaml` on each server. The API address is explicitly `https://127.0.0.1:` (`api_port` defaults to `6443`), so readiness cannot be satisfied by another server behind a load balancer. + +The drop-in appends with `kube-apiserver-arg+`, preserving API arguments in the base configuration and surviving `k3s.orchestration` site/upgrade reruns. Existing OIDC flags, structured `authentication-config`, command-line or environment API-argument overrides, custom config paths, and later drop-ins that replace the argument list are rejected before rollout. Migrate those settings explicitly rather than mixing authentication mechanisms. This workflow requires systemd-managed k3s with its default configuration path and [k3s YAML drop-in support](https://docs.k3s.io/installation/configuration#value-merge-behavior); it does not configure other Kubernetes distributions. + +Setting `cozystack_oidc_enabled: false` skips the parent API playbook: it does **not** remove the existing drop-in, disable API OIDC or restart k3s. Plan authentication removal separately. A normal rerun with unchanged content does not restart servers. In `--check` mode, enabled preflight still reads discovery, signing keys and local configuration, but no drop-in is written and no service is restarted. + +The generic `cozystack.installer.site` playbook only enables the platform setting; it never changes the host Kubernetes control plane. On kubeadm, RKE2 or managed Kubernetes, configure API authentication using that distribution's supported procedure. Creating users and granting tenant or administrator RBAC are separate from enabling OIDC; keep an administrative client-certificate kubeconfig available during rollout. + ## Important notes ### apiServerHost must be the internal IP diff --git a/hack/check-oidc-compat.sh b/hack/check-oidc-compat.sh new file mode 100755 index 0000000..63c4185 --- /dev/null +++ b/hack/check-oidc-compat.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +set -euo pipefail + +root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)" +scratch="$(mktemp --directory "${TMPDIR:-/tmp}/cozystack-oidc-compat.XXXXXXXX")" +printf 'Compatibility test artifacts: %s\n' "$scratch" + +export ANSIBLE_COLLECTIONS_SCAN_SYS_PATH=false +export ANSIBLE_COLLECTIONS_PATH="$scratch/stable" +ansible-galaxy collection install \ + --requirements-file "$root/examples/ubuntu/requirements.yml" \ + --collections-path "$ANSIBLE_COLLECTIONS_PATH" + +stable="$ANSIBLE_COLLECTIONS_PATH/ansible_collections/cozystack/installer" +if [[ -e "$stable/playbooks/k3s_oidc.yml" ]]; then + printf 'Expected the pinned release without the unreleased OIDC playbook\n' >&2 + exit 1 +fi +for distro in ubuntu rhel suse; do + ansible-playbook "$root/examples/$distro/site.yml" \ + --inventory "$root/examples/$distro/inventory.yml" --syntax-check +done + +mkdir "$scratch/artifact" +ansible-galaxy collection build "$root" --output-path "$scratch/artifact" +export ANSIBLE_COLLECTIONS_PATH="$scratch/current" +ansible-galaxy collection install "$scratch"/artifact/cozystack-installer-*.tar.gz \ + --collections-path "$ANSIBLE_COLLECTIONS_PATH" --no-deps +ansible-playbook cozystack.installer.k3s_oidc --inventory server, --syntax-check +ansible-playbook cozystack.installer.k3s_oidc --inventory server, --check diff --git a/playbooks/k3s_oidc.yml b/playbooks/k3s_oidc.yml new file mode 100644 index 0000000..cbed3ee --- /dev/null +++ b/playbooks/k3s_oidc.yml @@ -0,0 +1,196 @@ +--- +- name: Verify parent k3s OIDC prerequisites + hosts: server + gather_facts: false + become: true + any_errors_fatal: true + pre_tasks: + - name: Leave existing OIDC configuration unmanaged when disabled + ansible.builtin.meta: end_host + when: not (cozystack_oidc_enabled | default(false) | bool) + tasks: + - name: Resolve OIDC configuration + ansible.builtin.set_fact: + _cozystack_oidc_issuer: >- + {{ cozystack_oidc_issuer | default('https://keycloak.' ~ (cozystack_root_host | default('')) ~ '/realms/cozy', true) }} + _cozystack_oidc_config_path: /etc/rancher/k3s/config.yaml.d/90-cozystack-oidc.yaml + + - name: Validate OIDC issuer + ansible.builtin.assert: + that: + - cozystack_root_host | default('') | length > 0 + - _cozystack_oidc_issuer is match('^https://[^/@?#\s]+/[^?#\s]+$') + fail_msg: Set cozystack_root_host and use an HTTPS issuer without credentials, query parameters or fragments. + quiet: true + + - name: Read OIDC discovery from each server + ansible.builtin.uri: + url: "{{ _cozystack_oidc_issuer }}/.well-known/openid-configuration" + use_proxy: false + use_netrc: false + validate_certs: true + register: _cozystack_oidc_discovery + until: _cozystack_oidc_discovery.status == 200 + retries: 60 + delay: 10 + check_mode: false + + - name: Validate OIDC discovery + ansible.builtin.assert: + that: + - _cozystack_oidc_discovery.json.issuer == _cozystack_oidc_issuer + - _cozystack_oidc_discovery.json.jwks_uri == _cozystack_oidc_issuer ~ '/protocol/openid-connect/certs' + quiet: true + + - name: Read OIDC signing keys from each server + ansible.builtin.uri: + url: "{{ _cozystack_oidc_discovery.json.jwks_uri }}" + use_proxy: false + use_netrc: false + validate_certs: true + register: _cozystack_oidc_jwks + until: >- + _cozystack_oidc_jwks.status | default(0) == 200 and + ((_cozystack_oidc_jwks.json | default({})).get('keys', []) | length) > 0 + retries: 60 + delay: 10 + check_mode: false + + - name: Read k3s service configuration + ansible.builtin.systemd_service: + name: k3s + daemon_reload: false + register: _cozystack_oidc_service + no_log: true + + - name: Require a running k3s server + ansible.builtin.assert: + that: _cozystack_oidc_service.status.MainPID | int > 0 + quiet: true + + - name: Read the running k3s environment + ansible.builtin.slurp: + src: "/proc/{{ _cozystack_oidc_service.status.MainPID }}/environ" + register: _cozystack_oidc_environment + no_log: true + + - name: Validate k3s configuration sources + ansible.builtin.assert: + that: + - "'--kube-apiserver-arg' not in _cozystack_oidc_service.status.ExecStart" + - _cozystack_oidc_cli_configs | reject('equalto', '/etc/rancher/k3s/config.yaml') | list | length == 0 + - "'K3S_KUBE_APISERVER_ARG=' not in (_cozystack_oidc_environment.content | b64decode)" + - _cozystack_oidc_env_configs | reject('equalto', '/etc/rancher/k3s/config.yaml') | list | length == 0 + fail_msg: Use the default k3s config path and move CLI or environment kube-apiserver arguments into YAML before enabling OIDC. + quiet: true + vars: + _cozystack_oidc_cli_configs: >- + {{ _cozystack_oidc_service.status.ExecStart | regex_findall('(?:^|\s)(?:--config(?:=|\s+)|-c(?:=|\s)*)([^\s;]*)') }} + _cozystack_oidc_env_configs: >- + {{ (_cozystack_oidc_environment.content | b64decode) | regex_findall('(?:^|\x00)K3S_CONFIG_FILE=([^\x00]+)') }} + no_log: true + + - name: Inspect the k3s configuration directory + ansible.builtin.stat: + path: /etc/rancher/k3s/config.yaml.d + register: _cozystack_oidc_config_dir + + - name: Find existing k3s configuration fragments + ansible.builtin.find: + paths: /etc/rancher/k3s/config.yaml.d + patterns: ['*.[yY][aA][mM][lL]', '*.[yY][mM][lL]'] + hidden: true + file_type: any + register: _cozystack_oidc_fragments + when: _cozystack_oidc_config_dir.stat.exists + + - name: Inspect the optional base k3s configuration + ansible.builtin.stat: + path: /etc/rancher/k3s/config.yaml + register: _cozystack_oidc_base + + - name: Read existing k3s configuration + ansible.builtin.slurp: + src: "{{ item }}" + loop: >- + {{ (['/etc/rancher/k3s/config.yaml'] if _cozystack_oidc_base.stat.exists else []) + + (_cozystack_oidc_fragments.files | default([]) | rejectattr('isdir') | map(attribute='path') | + reject('equalto', _cozystack_oidc_config_path) | list) }} + register: _cozystack_oidc_existing + no_log: true + + - name: Validate existing API-server configuration + ansible.builtin.assert: + that: + - _cozystack_oidc_config is none or _cozystack_oidc_config is mapping + - _cozystack_oidc_api_arg_names | select('match', '^(oidc-|authentication-config$)') | list | length == 0 + - item.item <= _cozystack_oidc_config_path or 'kube-apiserver-arg' not in _cozystack_oidc_config_map + fail_msg: Existing API authentication or a later replacing kube-apiserver-arg conflicts with the managed OIDC drop-in. + quiet: true + vars: + _cozystack_oidc_config: "{{ item.content | b64decode | from_yaml }}" + _cozystack_oidc_config_map: "{{ _cozystack_oidc_config if _cozystack_oidc_config is mapping else {} }}" + _cozystack_oidc_base_args: "{{ _cozystack_oidc_config_map.get('kube-apiserver-arg', []) | default([], true) }}" + _cozystack_oidc_extra_args: "{{ _cozystack_oidc_config_map.get('kube-apiserver-arg+', []) | default([], true) }}" + _cozystack_oidc_api_args: >- + {{ ([_cozystack_oidc_base_args] if _cozystack_oidc_base_args is string else _cozystack_oidc_base_args) + + ([_cozystack_oidc_extra_args] if _cozystack_oidc_extra_args is string else _cozystack_oidc_extra_args) }} + _cozystack_oidc_api_arg_names: >- + {{ _cozystack_oidc_api_args | map('regex_replace', '^--', '') | map('regex_replace', '=.*$', '') | list }} + loop: "{{ _cozystack_oidc_existing.results }}" + no_log: true + +- name: Configure parent k3s OIDC one server at a time + hosts: server + gather_facts: false + become: true + serial: 1 + any_errors_fatal: true + pre_tasks: + - name: Leave existing OIDC configuration unmanaged when disabled + ansible.builtin.meta: end_host + when: not (cozystack_oidc_enabled | default(false) | bool) + tasks: + - name: Create the k3s configuration directory + ansible.builtin.file: + path: /etc/rancher/k3s/config.yaml.d + state: directory + owner: root + group: root + mode: '0700' + + - name: Append Kubernetes OIDC arguments + ansible.builtin.copy: + dest: /etc/rancher/k3s/config.yaml.d/90-cozystack-oidc.yaml + content: | + kube-apiserver-arg+: + - oidc-issuer-url={{ _cozystack_oidc_issuer }} + - oidc-client-id=kubernetes + - oidc-username-claim=preferred_username + - oidc-groups-claim=groups + owner: root + group: root + mode: '0600' + notify: Restart k3s for OIDC + + - name: Apply changed OIDC arguments + ansible.builtin.meta: flush_handlers + + - name: Wait for the local Kubernetes API + ansible.builtin.command: >- + k3s kubectl --context={{ cozystack_kubeconfig_context | default('default') }} + --kubeconfig={{ cozystack_kubeconfig | default('/etc/rancher/k3s/k3s.yaml') }} + --server=https://127.0.0.1:{{ api_port | default(6443) }} + --request-timeout=10s get --raw=/readyz + register: _cozystack_oidc_api_ready + changed_when: false + until: _cozystack_oidc_api_ready.rc == 0 and _cozystack_oidc_api_ready.stdout == 'ok' + retries: 60 + delay: 5 + when: not ansible_check_mode + + handlers: + - name: Restart k3s for OIDC + ansible.builtin.systemd_service: + name: k3s + state: restarted diff --git a/roles/cozystack/defaults/main.yml b/roles/cozystack/defaults/main.yml index 154f50d..bd89fe2 100644 --- a/roles/cozystack/defaults/main.yml +++ b/roles/cozystack/defaults/main.yml @@ -86,3 +86,5 @@ cozystack_join_cidr: "100.64.0.0/16" # Timeout to wait for operator and CRDs (seconds) cozystack_operator_wait_timeout: "300" + +cozystack_oidc_enabled: false diff --git a/roles/cozystack/tasks/main.yml b/roles/cozystack/tasks/main.yml index fbf5852..cca5d3a 100644 --- a/roles/cozystack/tasks/main.yml +++ b/roles/cozystack/tasks/main.yml @@ -24,6 +24,13 @@ templates/cozystack-operator.yaml and provides no override. Remove cozystack_namespace from inventory. +- name: Validate OIDC publishing domain + ansible.builtin.assert: + that: cozystack_root_host | length > 0 + fail_msg: cozystack_root_host is required to publish Keycloak when cozystack_oidc_enabled is true. + quiet: true + when: cozystack_oidc_enabled | bool + - name: Compute and validate master node IPs for kube-ovn ansible.builtin.include_tasks: compute-master-nodes.yml when: cozystack_create_platform_package diff --git a/roles/cozystack/templates/platform-package.yml.j2 b/roles/cozystack/templates/platform-package.yml.j2 index b906187..c656436 100644 --- a/roles/cozystack/templates/platform-package.yml.j2 +++ b/roles/cozystack/templates/platform-package.yml.j2 @@ -32,3 +32,8 @@ spec: externalIPs: {{ cozystack_external_ips | to_json }} {% endif %} {% endif %} +{% if cozystack_oidc_enabled | bool %} + authentication: + oidc: + enabled: true +{% endif %} diff --git a/tests/test-external-ips.yml b/tests/test-external-ips.yml index fed362f..7fcdd47 100644 --- a/tests/test-external-ips.yml +++ b/tests/test-external-ips.yml @@ -11,6 +11,7 @@ cozystack_api_server_host: "{{ cozystack_api_server_host | default('127.0.0.1') }}" cozystack_api_server_port: "{{ cozystack_api_server_port | default('6443') }}" cozystack_platform_variant: "{{ cozystack_platform_variant | default('isp-full-generic') }}" + cozystack_oidc_enabled: "{{ cozystack_oidc_enabled | default(false) }}" cozystack_root_host: "{{ cozystack_root_host | default('') }}" cozystack_external_ips: "{{ cozystack_external_ips | default([]) }}" cozystack_pod_cidr: "{{ cozystack_pod_cidr | default('10.42.0.0/16') }}" @@ -74,6 +75,7 @@ cozystack_api_server_host: "{{ cozystack_api_server_host | default('127.0.0.1') }}" cozystack_api_server_port: "{{ cozystack_api_server_port | default('6443') }}" cozystack_platform_variant: "{{ cozystack_platform_variant | default('isp-full-generic') }}" + cozystack_oidc_enabled: "{{ cozystack_oidc_enabled | default(false) }}" cozystack_root_host: "cozy.example.com" cozystack_external_ips: [] cozystack_pod_cidr: "{{ cozystack_pod_cidr | default('10.42.0.0/16') }}" diff --git a/tests/test-master-nodes.yml b/tests/test-master-nodes.yml index 0ae78a8..03d8599 100644 --- a/tests/test-master-nodes.yml +++ b/tests/test-master-nodes.yml @@ -12,6 +12,7 @@ cozystack_api_server_host: "{{ cozystack_api_server_host | default('127.0.0.1') }}" cozystack_api_server_port: "{{ cozystack_api_server_port | default('6443') }}" cozystack_platform_variant: "{{ cozystack_platform_variant | default('isp-full-generic') }}" + cozystack_oidc_enabled: "{{ cozystack_oidc_enabled | default(false) }}" cozystack_root_host: "{{ cozystack_root_host | default('') }}" cozystack_external_ips: "{{ cozystack_external_ips | default([]) }}" cozystack_pod_cidr: "{{ cozystack_pod_cidr | default('10.42.0.0/16') }}" diff --git a/tests/unit/playbooks/test_k3s_oidc.py b/tests/unit/playbooks/test_k3s_oidc.py new file mode 100644 index 0000000..8db28f5 --- /dev/null +++ b/tests/unit/playbooks/test_k3s_oidc.py @@ -0,0 +1,483 @@ +# -*- coding: utf-8 -*- + +# Copyright: (c) 2026, Cozystack Contributors +# Apache License 2.0 (see LICENSE file in the repository root) + +from __future__ import absolute_import, division, print_function + +__metaclass__ = type + +import base64 +import copy +import fnmatch +import json +import os +import shutil +import subprocess + +import pytest +import yaml +from ansible.plugins.filter.core import to_bool +from jinja2 import Environment, FileSystemLoader + +ROOT = os.path.realpath(os.path.join(os.path.dirname(__file__), "..", "..", "..")) +PLAYBOOK = "playbooks/k3s_oidc.yml" +DROPIN = "/etc/rancher/k3s/config.yaml.d/90-cozystack-oidc.yaml" + + +def _read(path): + with open(os.path.join(ROOT, path), encoding="utf-8") as stream: + return stream.read() + + +def _plays(): + return yaml.safe_load(_read(PLAYBOOK)) + + +def _task(name): + for play in _plays(): + for task in play.get("tasks", []): + if task.get("name") == name: + return copy.deepcopy(task) + raise AssertionError("Task not found: " + name) + + +def _run(tmp_path, tasks, variables=None, check=False): + executable = shutil.which("ansible-playbook") + assert executable, "ansible-playbook is required" + path = tmp_path / "test.yml" + path.write_text( + yaml.safe_dump( + [ + { + "name": "Test OIDC tasks locally", + "hosts": "localhost", + "gather_facts": False, + "vars": variables or {}, + "tasks": tasks, + } + ] + ), + encoding="utf-8", + ) + command = [ + executable, + str(path), + "--inventory", + "localhost,", + "--connection", + "local", + ] + if check: + command.append("--check") + return subprocess.run(command, capture_output=True, text=True, check=False) + + +def test_opt_in_and_existing_kubernetes_isolation(): + defaults = yaml.safe_load(_read("roles/cozystack/defaults/main.yml")) + assert defaults["cozystack_oidc_enabled"] is False + assert "k3s_oidc" not in _read("playbooks/site.yml") + for distro in ("ubuntu", "rhel", "suse"): + plays = yaml.safe_load(_read("examples/%s/site.yml" % distro)) + assert ( + plays[-1]["ansible.builtin.import_playbook"] == "cozystack.installer.site" + ) + assert "k3s_oidc" not in _read("examples/%s/site.yml" % distro) + for play in _plays(): + assert play["hosts"] == "server" + assert play["any_errors_fatal"] is True + assert play["pre_tasks"][0]["ansible.builtin.meta"] == "end_host" + assert "cozystack_oidc_enabled" in play["pre_tasks"][0]["when"] + assert _plays()[1]["serial"] == 1 + assert "state: absent" not in _read(PLAYBOOK) + + +@pytest.mark.parametrize( + "enabled,expected", + [(False, False), (True, True), ("false", False), ("true", True)], +) +def test_platform_oidc_is_opt_in(enabled, expected): + environment = Environment( + loader=FileSystemLoader(os.path.join(ROOT, "roles/cozystack/templates")) + ) + environment.filters["to_json"] = json.dumps + environment.filters["bool"] = to_bool + variables = dict( + cozystack_platform_variant="isp-full-generic", + cozystack_api_server_host="192.0.2.10", + cozystack_api_server_port="6443", + _cozystack_master_nodes="192.0.2.10", + cozystack_pod_cidr="10.42.0.0/16", + cozystack_pod_gateway="10.42.0.1", + cozystack_svc_cidr="10.43.0.0/16", + cozystack_join_cidr="100.64.0.0/16", + cozystack_root_host="example.org", + _cozystack_api_endpoint="https://api.example.org", + cozystack_external_ips=[], + cozystack_oidc_enabled=enabled, + ) + template = environment.get_template("platform-package.yml.j2") + values = yaml.safe_load(template.render(**variables))["spec"]["components"][ + "platform" + ]["values"] + if expected: + assert values["authentication"]["oidc"]["enabled"] is True + else: + assert "authentication" not in values + + +def test_disabled_playbook_makes_no_connections(tmp_path): + result = subprocess.run( + [ + shutil.which("ansible-playbook"), + os.path.join(ROOT, PLAYBOOK), + "--inventory", + "server,", + "--extra-vars", + '{"cozystack_oidc_enabled": false}', + ], + capture_output=True, + text=True, + check=False, + ) + assert result.returncode == 0, result.stdout + result.stderr + assert "UNREACHABLE" not in result.stdout + assert "Read OIDC discovery" not in result.stdout + assert "Append Kubernetes OIDC arguments" not in result.stdout + + +@pytest.mark.parametrize( + "path,config,allowed", + [ + ( + "/etc/rancher/k3s/config.yaml", + {"kube-apiserver-arg": ["audit-log-maxage=10"]}, + True, + ), + ( + "/etc/rancher/k3s/config.yaml.d/99-extra.yml", + {"kube-apiserver-arg+": ["audit-log-maxage=10"]}, + True, + ), + ( + "/etc/rancher/k3s/config.yaml.d/99-extra.yml", + {"kube-apiserver-arg": ["audit-log-maxage=10"]}, + False, + ), + ( + "/etc/rancher/k3s/config.yaml", + {"kube-apiserver-arg": ["authentication-config=/etc/auth.yaml"]}, + False, + ), + ( + "/etc/rancher/k3s/config.yaml", + {"kube-apiserver-arg+": ["oidc-client-id=other"]}, + False, + ), + ( + "/etc/rancher/k3s/config.yaml", + {"kube-apiserver-arg": ["--oidc-issuer-url=https://other.example"]}, + False, + ), + ( + "/etc/rancher/k3s/config.yaml", + {"node-label": ["example.org/oidc-enabled=true"]}, + True, + ), + ( + "/etc/rancher/k3s/config.yaml", + {"kube-apiserver-arg": ["audit-log-path=/var/log/oidc-audit.log"]}, + True, + ), + ( + "/etc/rancher/k3s/config.yaml", + { + "kube-apiserver-arg": "audit-policy-file=/etc/kubernetes/authentication-config-audit.yaml" + }, + True, + ), + ("/etc/rancher/k3s/config.yaml", {"kube-apiserver-arg": None}, True), + ("/etc/rancher/k3s/config.yaml", {"kube-apiserver-arg+": None}, True), + ("/etc/rancher/k3s/config.yaml", [], False), + ], +) +def test_existing_config_guards(tmp_path, path, config, allowed): + task = _task("Validate existing API-server configuration") + task.pop("loop") + task.pop("no_log", None) + variables = { + "_cozystack_oidc_config_path": DROPIN, + "item": { + "item": path, + "content": base64.b64encode(yaml.safe_dump(config).encode()).decode(), + }, + } + result = _run(tmp_path, [task], variables) + assert (result.returncode == 0) is allowed, result.stdout + result.stderr + + +@pytest.mark.parametrize( + "arguments,environment,allowed", + [ + ("/usr/local/bin/k3s server --disable=traefik", "PATH=/bin\0", True), + ( + "/usr/local/bin/k3s server --kube-apiserver-arg=audit-log-maxage=10", + "", + False, + ), + ("/usr/local/bin/k3s server --config=/etc/rancher/k3s/config.yaml", "", True), + ("/usr/local/bin/k3s server --config /etc/rancher/k3s/config.yaml", "", True), + ("/usr/local/bin/k3s server -c /etc/rancher/k3s/config.yaml", "", True), + ("/usr/local/bin/k3s server -c/etc/rancher/k3s/config.yaml", "", True), + ("/usr/local/bin/k3s server -c/etc/other.yaml", "", False), + ( + "/usr/local/bin/k3s server", + "K3S_CONFIG_FILE=/etc/rancher/k3s/config.yaml\0", + True, + ), + ("/usr/local/bin/k3s server", "K3S_CONFIG_FILE=\0PATH=/bin\0", True), + ("/usr/local/bin/k3s server --config=", "", False), + ("/usr/local/bin/k3s server -c=", "", False), + ("/usr/local/bin/k3s server --config=/etc/other.yaml", "", False), + ("/usr/local/bin/k3s server -c /etc/other.yaml", "", False), + ( + "/usr/local/bin/k3s server", + "K3S_KUBE_APISERVER_ARG=audit-log-maxage=10\0", + False, + ), + ("/usr/local/bin/k3s server", "K3S_CONFIG_FILE=/etc/other.yaml\0", False), + ], +) +def test_command_line_and_environment_guards(tmp_path, arguments, environment, allowed): + task = _task("Validate k3s configuration sources") + task.pop("no_log", None) + variables = { + "_cozystack_oidc_service": {"status": {"ExecStart": arguments}}, + "_cozystack_oidc_environment": { + "content": base64.b64encode(environment.encode()).decode() + }, + } + result = _run(tmp_path, [task], variables) + assert (result.returncode == 0) is allowed, result.stdout + result.stderr + + +def test_dropin_preserves_base_and_is_idempotent(tmp_path): + base = tmp_path / "config.yaml" + base.write_text("kube-apiserver-arg:\n - audit-log-maxage=10\n", encoding="utf-8") + before = base.read_bytes() + task = _task("Append Kubernetes OIDC arguments") + task["ansible.builtin.copy"]["dest"] = str(tmp_path / "90-oidc.yaml") + task["ansible.builtin.copy"].pop("owner", None) + task["ansible.builtin.copy"].pop("group", None) + task.pop("notify") + first, second = copy.deepcopy(task), copy.deepcopy(task) + first["register"] = "first" + second["register"] = "second" + result = _run( + tmp_path, + [ + first, + second, + { + "name": "Check idempotency", + "ansible.builtin.assert": { + "that": ["first.changed", "not second.changed"] + }, + }, + ], + {"_cozystack_oidc_issuer": "https://keycloak.example.org/realms/cozy"}, + ) + assert result.returncode == 0, result.stdout + result.stderr + assert base.read_bytes() == before + args = yaml.safe_load((tmp_path / "90-oidc.yaml").read_text())[ + "kube-apiserver-arg+" + ] + assert args == [ + "oidc-issuer-url=https://keycloak.example.org/realms/cozy", + "oidc-client-id=kubernetes", + "oidc-username-claim=preferred_username", + "oidc-groups-claim=groups", + ] + assert _task("Append Kubernetes OIDC arguments")["notify"] == "Restart k3s for OIDC" + + +def test_check_mode_does_not_write_the_dropin(tmp_path): + task = _task("Append Kubernetes OIDC arguments") + destination = tmp_path / "90-oidc.yaml" + task["ansible.builtin.copy"]["dest"] = str(destination) + task["ansible.builtin.copy"].pop("owner", None) + task["ansible.builtin.copy"].pop("group", None) + task.pop("notify") + result = _run( + tmp_path, + [task], + { + "_cozystack_oidc_issuer": "https://keycloak.example.org/realms/cozy", + }, + check=True, + ) + assert result.returncode == 0, result.stdout + result.stderr + assert not destination.exists() + + +def test_role_rejects_oidc_without_publishing_domain(tmp_path): + tasks = yaml.safe_load(_read("roles/cozystack/tasks/main.yml")) + task = next( + task for task in tasks if task.get("name") == "Validate OIDC publishing domain" + ) + for host, enabled, allowed in ( + ("", True, False), + ("example.org", True, True), + ("", False, True), + ): + result = _run( + tmp_path, + [task], + { + "cozystack_oidc_enabled": enabled, + "cozystack_root_host": host, + }, + ) + assert (result.returncode == 0) is allowed, result.stdout + result.stderr + + +@pytest.mark.parametrize( + "issuer,allowed", + [ + ("https://keycloak.example.org/realms/cozy", True), + ("http://keycloak.example.org/realms/cozy", False), + ("https://user@keycloak.example.org/realms/cozy", False), + ("https://keycloak.example.org/realms/cozy?query=value", False), + ("https://keycloak.example.org/realms/cozy#fragment", False), + ], +) +def test_preflight_validates_issuer(tmp_path, issuer, allowed): + result = _run( + tmp_path, + [_task("Validate OIDC issuer")], + {"cozystack_root_host": "example.org", "_cozystack_oidc_issuer": issuer}, + ) + assert (result.returncode == 0) is allowed, result.stdout + result.stderr + + +def test_optional_base_configuration_can_be_absent(tmp_path): + task = _task("Read existing k3s configuration") + task.pop("no_log", None) + result = _run( + tmp_path, + [task], + { + "_cozystack_oidc_base": {"stat": {"exists": False}}, + "_cozystack_oidc_fragments": {"files": []}, + "_cozystack_oidc_config_path": DROPIN, + }, + ) + assert result.returncode == 0, result.stdout + result.stderr + + +def test_signing_key_readiness_is_retried(tmp_path): + import http.server + import threading + + requests = [] + + class Handler(http.server.BaseHTTPRequestHandler): + def do_GET(self): + requests.append(self.path) + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.end_headers() + keys = [] if len(requests) == 1 else [{"kid": "test"}] + self.wfile.write(json.dumps({"keys": keys}).encode()) + + def log_message(self, *args): + pass + + server = http.server.HTTPServer(("127.0.0.1", 0), Handler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + task = _task("Read OIDC signing keys from each server") + assert task.get("retries", 0) >= 2 + assert "until" in task + task["ansible.builtin.uri"]["url"] = ( + "http://127.0.0.1:%s/keys" % server.server_port + ) + task["delay"] = 0 + task["retries"] = 3 + result = _run(tmp_path, [task]) + assert result.returncode == 0, result.stdout + result.stderr + assert len(requests) == 2 + finally: + server.shutdown() + server.server_close() + thread.join() + + +def test_fragment_discovery_matches_k3s_files_and_skips_directories(tmp_path): + directory = tmp_path / "config.yaml.d" + directory.mkdir() + filenames = [ + "10-base.yaml", + "20-extra.yml", + "30-upper.YML", + "40-mixed.YaMl", + ".hidden.yaml", + ] + for filename in filenames: + (directory / filename).write_text("{}\n", encoding="utf-8") + (directory / "ignored.txt").write_text("not YAML", encoding="utf-8") + (directory / "nested.yaml").mkdir() + (directory / "50-link.yml").symlink_to(directory / "10-base.yaml") + expected = [str(directory / name) for name in filenames + ["50-link.yml"]] + find = _task("Find existing k3s configuration fragments") + find["ansible.builtin.find"]["paths"] = str(directory) + read = _task("Read existing k3s configuration") + read.pop("no_log", None) + result = _run( + tmp_path, + [ + find, + read, + { + "name": "Verify all supported fragments were read", + "ansible.builtin.assert": { + "that": "_cozystack_oidc_existing.results | map(attribute='item') | sort == expected | sort", + }, + }, + ], + { + "_cozystack_oidc_config_dir": {"stat": {"exists": True}}, + "_cozystack_oidc_base": {"stat": {"exists": False}}, + "_cozystack_oidc_config_path": DROPIN, + "expected": expected, + }, + ) + assert result.returncode == 0, result.stdout + result.stderr + + +def test_preflight_verifies_tls_and_every_supported_fragment_extension(): + find = _task("Find existing k3s configuration fragments")["ansible.builtin.find"] + for filename in ("10-config.yaml", "99-auth.yml", "95-extra.YaML"): + assert any( + fnmatch.fnmatchcase(filename, pattern) for pattern in find["patterns"] + ) + assert find["hidden"] is True + assert find["file_type"] == "any" + for name in ( + "Read OIDC discovery from each server", + "Read OIDC signing keys from each server", + ): + task = _task(name) + assert task["ansible.builtin.uri"]["validate_certs"] is True + assert task["check_mode"] is False + names = [task["name"] for task in _plays()[1]["tasks"]] + assert names.index("Append Kubernetes OIDC arguments") < names.index( + "Apply changed OIDC arguments" + ) + assert names.index("Apply changed OIDC arguments") < names.index( + "Wait for the local Kubernetes API" + ) + assert ( + _task("Wait for the local Kubernetes API")["when"] == "not ansible_check_mode" + )