From b11259d7be694652f1bd19f3b7fd47837c43ab68 Mon Sep 17 00:00:00 2001 From: Rayhan Hossain Date: Mon, 19 Oct 2026 16:38:55 -0700 Subject: [PATCH 01/27] test(cnpg): add golden snapshots for rendered CNPG clusters Locks the rendered Cluster for a matrix of DocumentDB inputs so the product-neutral refactor can prove byte-identical output. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 33b3cb3c-1b25-4ffb-bff1-ebf0f3d04058 Signed-off-by: Rayhan Hossain --- .../src/internal/cnpg/cnpg_golden_test.go | 225 ++++++++++++++++++ .../cnpg/testdata/golden/defaults.json | 130 ++++++++++ .../cnpg/testdata/golden/env-overrides.json | 159 +++++++++++++ .../golden/explicit-component-resources.json | 153 ++++++++++++ .../internal/cnpg/testdata/golden/full.json | 176 ++++++++++++++ .../cnpg/testdata/golden/recovery-backup.json | 128 ++++++++++ .../cnpg/testdata/golden/recovery-pv.json | 132 ++++++++++ .../replica-region-ignores-recovery.json | 130 ++++++++++ 8 files changed, 1233 insertions(+) create mode 100644 operator/src/internal/cnpg/cnpg_golden_test.go create mode 100644 operator/src/internal/cnpg/testdata/golden/defaults.json create mode 100644 operator/src/internal/cnpg/testdata/golden/env-overrides.json create mode 100644 operator/src/internal/cnpg/testdata/golden/explicit-component-resources.json create mode 100644 operator/src/internal/cnpg/testdata/golden/full.json create mode 100644 operator/src/internal/cnpg/testdata/golden/recovery-backup.json create mode 100644 operator/src/internal/cnpg/testdata/golden/recovery-pv.json create mode 100644 operator/src/internal/cnpg/testdata/golden/replica-region-ignores-recovery.json diff --git a/operator/src/internal/cnpg/cnpg_golden_test.go b/operator/src/internal/cnpg/cnpg_golden_test.go new file mode 100644 index 00000000..e0fce879 --- /dev/null +++ b/operator/src/internal/cnpg/cnpg_golden_test.go @@ -0,0 +1,225 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package cnpg + +import ( + "encoding/json" + "os" + "path/filepath" + "testing" + + cnpgv1 "github.com/cloudnative-pg/cloudnative-pg/api/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/pointer" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + + dbpreview "github.com/documentdb/documentdb-operator/api/preview" + util "github.com/documentdb/documentdb-operator/internal/utils" +) + +// renderEnv pins every operator environment variable that influences the +// rendered Cluster so golden output is independent of the developer machine. +var renderEnv = []string{ + util.DOCUMENTDB_VERSION_ENV, + util.GATEWAY_IMAGE_PULL_POLICY_ENV, + util.DOCUMENTDB_IMAGE_PULL_POLICY_ENV, + util.DOCUMENTDB_EXTENSION_IMAGE_REPO_ENV, + util.GATEWAY_IMAGE_REPO_ENV, + util.OTEL_COLLECTOR_IMAGE_ENV, + util.POSTGRES_IMAGE_ENV, + util.IOURING_SECCOMP_PROFILE_ENV, + util.GATEWAY_MEMORY_FRACTION_ENV, + util.GATEWAY_MEMORY_CAP_ENV, + util.GATEWAY_CPU_LIMIT_ENV, + util.OTEL_MEMORY_REQUEST_ENV, + util.OTEL_MEMORY_LIMIT_ENV, + util.OTEL_CPU_REQUEST_ENV, + util.OTEL_CPU_LIMIT_ENV, +} + +func goldenBaseDocumentDB() *dbpreview.DocumentDB { + return &dbpreview.DocumentDB{ + TypeMeta: metav1.TypeMeta{APIVersion: "documentdb.io/preview", Kind: "DocumentDB"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "golden", + Namespace: "golden-ns", + UID: "00000000-0000-0000-0000-000000000001", + }, + Spec: dbpreview.DocumentDBSpec{ + NodeCount: 1, + InstancesPerNode: 1, + Resource: dbpreview.Resource{ + Storage: dbpreview.StorageConfiguration{PvcSize: "10Gi"}, + }, + }, + } +} + +type goldenCase struct { + name string + env map[string]string + mutate func(*dbpreview.DocumentDB) + extImage string + storageClass string + primary bool +} + +func goldenCases() []goldenCase { + return []goldenCase{ + {name: "defaults", primary: true}, + { + name: "full", + primary: true, + storageClass: "premium", + mutate: func(d *dbpreview.DocumentDB) { + d.Spec.InstancesPerNode = 3 + d.Spec.Resource.Memory = "16Gi" + d.Spec.Resource.CPU = "4" + d.Spec.Image = &dbpreview.ImageSpec{ + DocumentDB: "reg.example/ext:9.9.9", + Gateway: "reg.example/gw:9.9.9", + Postgres: "reg.example/pg:18", + } + d.Spec.ImagePullSecrets = []corev1.LocalObjectReference{{Name: "pull"}} + d.Spec.DocumentDbCredentialSecret = "creds" + d.Spec.Plugins = &dbpreview.PluginsSpec{SidecarInjectorName: "custom.injector.io"} + d.Spec.Postgres = &dbpreview.PostgresSpec{ + UID: pointer.Int64(26), + GID: pointer.Int64(26), + PostInitSQL: []string{"SELECT 1"}, + Parameters: map[string]string{"work_mem": "64MB", "max_wal_senders": "99"}, + } + d.Spec.FeatureGates = map[string]bool{ + dbpreview.FeatureGateChangeStreams: true, + dbpreview.FeatureGateIOUring: true, + } + d.Spec.LogLevel = "debug" + d.Spec.Timeouts.StopDelay = 120 + d.Spec.TLS = &dbpreview.TLSConfiguration{ + Postgres: &cnpgv1.CertificatesConfiguration{ServerTLSSecret: "pg-server-tls", ServerCASecret: "pg-ca"}, + } + d.Status.TLS = &dbpreview.TLSStatus{Ready: true, SecretName: "gw-tls"} + d.Spec.Monitoring = &dbpreview.MonitoringSpec{ + Enabled: true, + Exporter: &dbpreview.ExporterSpec{ + OTLP: &dbpreview.OTLPExporterSpec{Endpoint: "otel:4317"}, + Prometheus: &dbpreview.PrometheusExporterSpec{Port: 9187}, + }, + } + d.Spec.Affinity = cnpgv1.AffinityConfiguration{ + EnablePodAntiAffinity: pointer.Bool(true), + TopologyKey: "kubernetes.io/hostname", + } + }, + }, + { + name: "explicit-component-resources", + primary: true, + mutate: func(d *dbpreview.DocumentDB) { + d.Spec.Resource.Database = &dbpreview.ComponentResources{Memory: "4Gi", CPU: "2"} + d.Spec.Resource.Gateway = &dbpreview.ComponentResources{Memory: "1Gi", CPU: "500m"} + d.Spec.Monitoring = &dbpreview.MonitoringSpec{Enabled: true} + d.Spec.Resource.OTel = &dbpreview.ComponentResources{Memory: "256Mi", CPU: "100m"} + }, + }, + { + name: "recovery-backup", + primary: true, + mutate: func(d *dbpreview.DocumentDB) { + d.Spec.Bootstrap = &dbpreview.BootstrapConfiguration{ + Recovery: &dbpreview.RecoveryConfiguration{Backup: cnpgv1.LocalObjectReference{Name: "bk"}}, + } + }, + }, + { + name: "recovery-pv", + primary: true, + mutate: func(d *dbpreview.DocumentDB) { + d.Spec.Bootstrap = &dbpreview.BootstrapConfiguration{ + Recovery: &dbpreview.RecoveryConfiguration{ + PersistentVolume: &dbpreview.PVRecoveryConfiguration{Name: "pv-1"}, + }, + } + }, + }, + { + name: "replica-region-ignores-recovery", + primary: false, + mutate: func(d *dbpreview.DocumentDB) { + d.Spec.Bootstrap = &dbpreview.BootstrapConfiguration{ + Recovery: &dbpreview.RecoveryConfiguration{Backup: cnpgv1.LocalObjectReference{Name: "bk"}}, + } + }, + }, + { + name: "env-overrides", + primary: true, + extImage: "override/ext:1", + env: map[string]string{ + util.DOCUMENTDB_IMAGE_PULL_POLICY_ENV: "Always", + util.GATEWAY_IMAGE_PULL_POLICY_ENV: "Never", + util.POSTGRES_IMAGE_ENV: "env/pg:17", + util.GATEWAY_CPU_LIMIT_ENV: "1", + util.IOURING_SECCOMP_PROFILE_ENV: "profiles/custom.json", + util.OTEL_COLLECTOR_IMAGE_ENV: "env/otel:1", + }, + mutate: func(d *dbpreview.DocumentDB) { + d.Spec.Resource.Memory = "8Gi" + d.Spec.FeatureGates = map[string]bool{dbpreview.FeatureGateIOUring: true} + d.Spec.Monitoring = &dbpreview.MonitoringSpec{Enabled: true} + }, + }, + } +} + +// TestGetCnpgClusterSpecGolden locks the rendered CNPG Cluster for a matrix of +// DocumentDB inputs. Regenerate with UPDATE_GOLDEN=1 only for intentional +// rendering changes. +func TestGetCnpgClusterSpecGolden(t *testing.T) { + update := os.Getenv("UPDATE_GOLDEN") == "1" + for _, tc := range goldenCases() { + t.Run(tc.name, func(t *testing.T) { + for _, key := range renderEnv { + t.Setenv(key, "") + } + for k, v := range tc.env { + t.Setenv(k, v) + } + d := goldenBaseDocumentDB() + if tc.mutate != nil { + tc.mutate(d) + } + req := ctrl.Request{} + req.Name = d.Name + req.Namespace = d.Namespace + + cluster := GetCnpgClusterSpec(req, d, tc.extImage, d.Name, tc.storageClass, tc.primary, zap.New(zap.WriteTo(os.Stderr))) + got, err := json.MarshalIndent(cluster, "", " ") + if err != nil { + t.Fatal(err) + } + got = append(got, '\n') + + path := filepath.Join("testdata", "golden", tc.name+".json") + if update { + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, got, 0o644); err != nil { + t.Fatal(err) + } + return + } + want, err := os.ReadFile(path) + if err != nil { + t.Fatalf("missing golden file %s (run with UPDATE_GOLDEN=1): %v", path, err) + } + if string(want) != string(got) { + t.Errorf("rendered cluster differs from %s\n--- want\n%s\n--- got\n%s", path, want, got) + } + }) + } +} diff --git a/operator/src/internal/cnpg/testdata/golden/defaults.json b/operator/src/internal/cnpg/testdata/golden/defaults.json new file mode 100644 index 00000000..925a1653 --- /dev/null +++ b/operator/src/internal/cnpg/testdata/golden/defaults.json @@ -0,0 +1,130 @@ +{ + "metadata": { + "name": "golden", + "namespace": "golden-ns", + "ownerReferences": [ + { + "apiVersion": "documentdb.io/preview", + "kind": "DocumentDB", + "name": "golden", + "uid": "00000000-0000-0000-0000-000000000001", + "controller": true, + "blockOwnerDeletion": true + } + ] + }, + "spec": { + "inheritedMetadata": { + "labels": { + "app": "golden", + "replica_type": "primary" + } + }, + "instances": 1, + "postgresql": { + "parameters": { + "autovacuum_analyze_scale_factor": "0.05", + "autovacuum_max_workers": "4", + "autovacuum_vacuum_cost_delay": "2ms", + "autovacuum_vacuum_scale_factor": "0.1", + "checkpoint_completion_target": "0.9", + "cron.database_name": "postgres", + "effective_cache_size": "512MB", + "effective_io_concurrency": "200", + "maintenance_work_mem": "128MB", + "max_connections": "300", + "max_prepared_transactions": "100", + "max_replication_slots": "10", + "max_wal_senders": "10", + "max_wal_size": "2GB", + "min_wal_size": "256MB", + "random_page_cost": "1.1", + "shared_buffers": "256MB", + "wal_buffers": "16MB", + "work_mem": "16MB" + }, + "pg_hba": [ + "host all all localhost trust", + "hostssl replication streaming_replica all cert" + ], + "syncReplicaElectionConstraint": { + "enabled": false + }, + "shared_preload_libraries": [ + "pg_cron", + "pg_documentdb_core", + "pg_documentdb" + ], + "extensions": [ + { + "name": "documentdb", + "image": { + "reference": "ghcr.io/documentdb/documentdb-kubernetes-operator/documentdb:1.0.0" + }, + "extension_control_path": [ + "share" + ], + "dynamic_library_path": [ + "lib" + ], + "ld_library_path": [ + "lib", + "system" + ] + } + ] + }, + "bootstrap": { + "initdb": { + "postInitSQL": [ + "CREATE EXTENSION documentdb CASCADE", + "CREATE ROLE documentdb WITH LOGIN PASSWORD 'Admin100'", + "ALTER ROLE documentdb WITH SUPERUSER CREATEDB CREATEROLE REPLICATION BYPASSRLS" + ] + } + }, + "storage": { + "size": "10Gi" + }, + "stopDelay": 30, + "affinity": {}, + "resources": {}, + "primaryUpdateMethod": "switchover", + "backup": { + "volumeSnapshot": { + "snapshotOwnerReference": "backup", + "onlineConfiguration": {} + }, + "target": "primary" + }, + "logLevel": "info", + "managed": { + "roles": [ + { + "name": "otel_monitor", + "ensure": "absent", + "connectionLimit": -1, + "inherit": true + } + ] + }, + "plugins": [ + { + "name": "cnpg-i-sidecar-injector.documentdb.io", + "enabled": true, + "parameters": { + "documentDbCredentialSecret": "documentdb-credentials", + "gatewayImage": "ghcr.io/documentdb/documentdb-kubernetes-operator/gateway:1.0.0" + } + } + ] + }, + "status": { + "managedRolesStatus": {}, + "topology": {}, + "secretsResourceVersion": {}, + "configMapResourceVersion": {}, + "certificates": {}, + "switchReplicaClusterStatus": {} + } +} diff --git a/operator/src/internal/cnpg/testdata/golden/env-overrides.json b/operator/src/internal/cnpg/testdata/golden/env-overrides.json new file mode 100644 index 00000000..e8e264f9 --- /dev/null +++ b/operator/src/internal/cnpg/testdata/golden/env-overrides.json @@ -0,0 +1,159 @@ +{ + "metadata": { + "name": "golden", + "namespace": "golden-ns", + "ownerReferences": [ + { + "apiVersion": "documentdb.io/preview", + "kind": "DocumentDB", + "name": "golden", + "uid": "00000000-0000-0000-0000-000000000001", + "controller": true, + "blockOwnerDeletion": true + } + ] + }, + "spec": { + "inheritedMetadata": { + "labels": { + "app": "golden", + "replica_type": "primary" + } + }, + "imageName": "env/pg:17", + "instances": 1, + "postgresql": { + "parameters": { + "autovacuum_analyze_scale_factor": "0.05", + "autovacuum_max_workers": "4", + "autovacuum_vacuum_cost_delay": "2ms", + "autovacuum_vacuum_scale_factor": "0.1", + "checkpoint_completion_target": "0.9", + "cron.database_name": "postgres", + "effective_cache_size": "4896MB", + "effective_io_concurrency": "200", + "io_method": "io_uring", + "maintenance_work_mem": "652MB", + "max_connections": "300", + "max_prepared_transactions": "100", + "max_replication_slots": "10", + "max_wal_senders": "10", + "max_wal_size": "2GB", + "min_wal_size": "256MB", + "random_page_cost": "1.1", + "shared_buffers": "1632MB", + "wal_buffers": "16MB", + "work_mem": "5MB" + }, + "pg_hba": [ + "host all all localhost trust", + "hostssl replication streaming_replica all cert" + ], + "syncReplicaElectionConstraint": { + "enabled": false + }, + "shared_preload_libraries": [ + "pg_cron", + "pg_documentdb_core", + "pg_documentdb" + ], + "extensions": [ + { + "name": "documentdb", + "image": { + "reference": "override/ext:1", + "pullPolicy": "Always" + }, + "extension_control_path": [ + "share" + ], + "dynamic_library_path": [ + "lib" + ], + "ld_library_path": [ + "lib", + "system" + ] + } + ] + }, + "bootstrap": { + "initdb": { + "postInitSQL": [ + "CREATE EXTENSION documentdb CASCADE", + "CREATE ROLE documentdb WITH LOGIN PASSWORD 'Admin100'", + "ALTER ROLE documentdb WITH SUPERUSER CREATEDB CREATEROLE REPLICATION BYPASSRLS" + ] + } + }, + "storage": { + "size": "10Gi" + }, + "stopDelay": 30, + "affinity": {}, + "resources": { + "limits": { + "memory": "6528Mi" + }, + "requests": { + "memory": "6528Mi" + } + }, + "primaryUpdateMethod": "switchover", + "backup": { + "volumeSnapshot": { + "snapshotOwnerReference": "backup", + "onlineConfiguration": {} + }, + "target": "primary" + }, + "logLevel": "info", + "managed": { + "roles": [ + { + "name": "otel_monitor", + "comment": "Dedicated role for the OTel Collector monitoring sidecar", + "ensure": "present", + "connectionLimit": -1, + "inherit": true, + "disablePassword": true, + "login": true + } + ] + }, + "seccompProfile": { + "type": "Localhost", + "localhostProfile": "profiles/custom.json" + }, + "plugins": [ + { + "name": "cnpg-i-sidecar-injector.documentdb.io", + "enabled": true, + "parameters": { + "documentDbCredentialSecret": "documentdb-credentials", + "gatewayCpuLimit": "1", + "gatewayCpuRequest": "1", + "gatewayImage": "ghcr.io/documentdb/documentdb-kubernetes-operator/gateway:1.0.0", + "gatewayImagePullPolicy": "Never", + "gatewayMemoryLimit": "1536Mi", + "gatewayMemoryRequest": "1536Mi", + "otelCollectorImage": "env/otel:1", + "otelConfigHash": "fbb647d6c740ff44", + "otelConfigMapName": "golden-otel-config", + "otelCpuLimit": "200m", + "otelCpuRequest": "50m", + "otelMemoryLimit": "128Mi", + "otelMemoryRequest": "48Mi" + } + } + ] + }, + "status": { + "managedRolesStatus": {}, + "topology": {}, + "secretsResourceVersion": {}, + "configMapResourceVersion": {}, + "certificates": {}, + "switchReplicaClusterStatus": {} + } +} diff --git a/operator/src/internal/cnpg/testdata/golden/explicit-component-resources.json b/operator/src/internal/cnpg/testdata/golden/explicit-component-resources.json new file mode 100644 index 00000000..2aec3f10 --- /dev/null +++ b/operator/src/internal/cnpg/testdata/golden/explicit-component-resources.json @@ -0,0 +1,153 @@ +{ + "metadata": { + "name": "golden", + "namespace": "golden-ns", + "ownerReferences": [ + { + "apiVersion": "documentdb.io/preview", + "kind": "DocumentDB", + "name": "golden", + "uid": "00000000-0000-0000-0000-000000000001", + "controller": true, + "blockOwnerDeletion": true + } + ] + }, + "spec": { + "inheritedMetadata": { + "labels": { + "app": "golden", + "replica_type": "primary" + } + }, + "instances": 1, + "postgresql": { + "parameters": { + "autovacuum_analyze_scale_factor": "0.05", + "autovacuum_max_workers": "4", + "autovacuum_vacuum_cost_delay": "2ms", + "autovacuum_vacuum_scale_factor": "0.1", + "checkpoint_completion_target": "0.9", + "cron.database_name": "postgres", + "effective_cache_size": "3GB", + "effective_io_concurrency": "200", + "maintenance_work_mem": "409MB", + "max_connections": "300", + "max_prepared_transactions": "100", + "max_replication_slots": "10", + "max_wal_senders": "10", + "max_wal_size": "2GB", + "min_wal_size": "256MB", + "random_page_cost": "1.1", + "shared_buffers": "1GB", + "wal_buffers": "16MB", + "work_mem": "4MB" + }, + "pg_hba": [ + "host all all localhost trust", + "hostssl replication streaming_replica all cert" + ], + "syncReplicaElectionConstraint": { + "enabled": false + }, + "shared_preload_libraries": [ + "pg_cron", + "pg_documentdb_core", + "pg_documentdb" + ], + "extensions": [ + { + "name": "documentdb", + "image": { + "reference": "ghcr.io/documentdb/documentdb-kubernetes-operator/documentdb:1.0.0" + }, + "extension_control_path": [ + "share" + ], + "dynamic_library_path": [ + "lib" + ], + "ld_library_path": [ + "lib", + "system" + ] + } + ] + }, + "bootstrap": { + "initdb": { + "postInitSQL": [ + "CREATE EXTENSION documentdb CASCADE", + "CREATE ROLE documentdb WITH LOGIN PASSWORD 'Admin100'", + "ALTER ROLE documentdb WITH SUPERUSER CREATEDB CREATEROLE REPLICATION BYPASSRLS" + ] + } + }, + "storage": { + "size": "10Gi" + }, + "stopDelay": 30, + "affinity": {}, + "resources": { + "limits": { + "cpu": "2", + "memory": "4Gi" + }, + "requests": { + "cpu": "2", + "memory": "4Gi" + } + }, + "primaryUpdateMethod": "switchover", + "backup": { + "volumeSnapshot": { + "snapshotOwnerReference": "backup", + "onlineConfiguration": {} + }, + "target": "primary" + }, + "logLevel": "info", + "managed": { + "roles": [ + { + "name": "otel_monitor", + "comment": "Dedicated role for the OTel Collector monitoring sidecar", + "ensure": "present", + "connectionLimit": -1, + "inherit": true, + "disablePassword": true, + "login": true + } + ] + }, + "plugins": [ + { + "name": "cnpg-i-sidecar-injector.documentdb.io", + "enabled": true, + "parameters": { + "documentDbCredentialSecret": "documentdb-credentials", + "gatewayCpuLimit": "500m", + "gatewayCpuRequest": "500m", + "gatewayImage": "ghcr.io/documentdb/documentdb-kubernetes-operator/gateway:1.0.0", + "gatewayMemoryLimit": "1Gi", + "gatewayMemoryRequest": "1Gi", + "otelCollectorImage": "otel/opentelemetry-collector-contrib:0.149.0", + "otelConfigHash": "fbb647d6c740ff44", + "otelConfigMapName": "golden-otel-config", + "otelCpuLimit": "100m", + "otelCpuRequest": "100m", + "otelMemoryLimit": "256Mi", + "otelMemoryRequest": "256Mi" + } + } + ] + }, + "status": { + "managedRolesStatus": {}, + "topology": {}, + "secretsResourceVersion": {}, + "configMapResourceVersion": {}, + "certificates": {}, + "switchReplicaClusterStatus": {} + } +} diff --git a/operator/src/internal/cnpg/testdata/golden/full.json b/operator/src/internal/cnpg/testdata/golden/full.json new file mode 100644 index 00000000..6e306c01 --- /dev/null +++ b/operator/src/internal/cnpg/testdata/golden/full.json @@ -0,0 +1,176 @@ +{ + "metadata": { + "name": "golden", + "namespace": "golden-ns", + "ownerReferences": [ + { + "apiVersion": "documentdb.io/preview", + "kind": "DocumentDB", + "name": "golden", + "uid": "00000000-0000-0000-0000-000000000001", + "controller": true, + "blockOwnerDeletion": true + } + ] + }, + "spec": { + "inheritedMetadata": { + "labels": { + "app": "golden", + "replica_type": "primary" + } + }, + "imageName": "reg.example/pg:18", + "postgresUID": 26, + "postgresGID": 26, + "instances": 3, + "postgresql": { + "parameters": { + "autovacuum_analyze_scale_factor": "0.05", + "autovacuum_max_workers": "4", + "autovacuum_vacuum_cost_delay": "2ms", + "autovacuum_vacuum_scale_factor": "0.1", + "checkpoint_completion_target": "0.9", + "cron.database_name": "postgres", + "effective_cache_size": "9888MB", + "effective_io_concurrency": "200", + "io_method": "io_uring", + "maintenance_work_mem": "1318MB", + "max_connections": "300", + "max_prepared_transactions": "100", + "max_replication_slots": "10", + "max_wal_senders": "10", + "max_wal_size": "2GB", + "min_wal_size": "256MB", + "random_page_cost": "1.1", + "shared_buffers": "3296MB", + "wal_buffers": "16MB", + "wal_level": "logical", + "work_mem": "64MB" + }, + "pg_hba": [ + "host all all localhost trust", + "hostssl replication streaming_replica all cert" + ], + "syncReplicaElectionConstraint": { + "enabled": false + }, + "shared_preload_libraries": [ + "pg_cron", + "pg_documentdb_core", + "pg_documentdb" + ], + "extensions": [ + { + "name": "documentdb", + "image": { + "reference": "reg.example/ext:9.9.9" + }, + "extension_control_path": [ + "share" + ], + "dynamic_library_path": [ + "lib" + ], + "ld_library_path": [ + "lib", + "system" + ] + } + ] + }, + "bootstrap": { + "initdb": { + "postInitSQL": [ + "CREATE EXTENSION documentdb CASCADE", + "CREATE ROLE documentdb WITH LOGIN PASSWORD 'Admin100'", + "ALTER ROLE documentdb WITH SUPERUSER CREATEDB CREATEROLE REPLICATION BYPASSRLS", + "SELECT 1" + ] + } + }, + "certificates": { + "serverCASecret": "pg-ca", + "serverTLSSecret": "pg-server-tls" + }, + "imagePullSecrets": [ + { + "name": "pull" + } + ], + "storage": { + "storageClass": "premium", + "size": "10Gi" + }, + "stopDelay": 120, + "affinity": { + "enablePodAntiAffinity": true, + "topologyKey": "kubernetes.io/hostname" + }, + "resources": { + "limits": { + "cpu": "3950m", + "memory": "13184Mi" + }, + "requests": { + "cpu": "3950m", + "memory": "13184Mi" + } + }, + "primaryUpdateMethod": "switchover", + "backup": { + "volumeSnapshot": { + "snapshotOwnerReference": "backup", + "onlineConfiguration": {} + }, + "target": "primary" + }, + "logLevel": "debug", + "managed": { + "roles": [ + { + "name": "otel_monitor", + "comment": "Dedicated role for the OTel Collector monitoring sidecar", + "ensure": "present", + "connectionLimit": -1, + "inherit": true, + "disablePassword": true, + "login": true + } + ] + }, + "seccompProfile": { + "type": "Localhost", + "localhostProfile": "profiles/documentdb-iouring.json" + }, + "plugins": [ + { + "name": "custom.injector.io", + "enabled": true, + "parameters": { + "documentDbCredentialSecret": "creds", + "gatewayImage": "reg.example/gw:9.9.9", + "gatewayMemoryLimit": "3Gi", + "gatewayMemoryRequest": "3Gi", + "gatewayTLSSecret": "gw-tls", + "otelCollectorImage": "otel/opentelemetry-collector-contrib:0.149.0", + "otelConfigHash": "d60e2ff6f9bb55d4", + "otelConfigMapName": "golden-otel-config", + "otelCpuLimit": "200m", + "otelCpuRequest": "50m", + "otelMemoryLimit": "128Mi", + "otelMemoryRequest": "48Mi", + "prometheusPort": "9187" + } + } + ] + }, + "status": { + "managedRolesStatus": {}, + "topology": {}, + "secretsResourceVersion": {}, + "configMapResourceVersion": {}, + "certificates": {}, + "switchReplicaClusterStatus": {} + } +} diff --git a/operator/src/internal/cnpg/testdata/golden/recovery-backup.json b/operator/src/internal/cnpg/testdata/golden/recovery-backup.json new file mode 100644 index 00000000..4066f677 --- /dev/null +++ b/operator/src/internal/cnpg/testdata/golden/recovery-backup.json @@ -0,0 +1,128 @@ +{ + "metadata": { + "name": "golden", + "namespace": "golden-ns", + "ownerReferences": [ + { + "apiVersion": "documentdb.io/preview", + "kind": "DocumentDB", + "name": "golden", + "uid": "00000000-0000-0000-0000-000000000001", + "controller": true, + "blockOwnerDeletion": true + } + ] + }, + "spec": { + "inheritedMetadata": { + "labels": { + "app": "golden", + "replica_type": "primary" + } + }, + "instances": 1, + "postgresql": { + "parameters": { + "autovacuum_analyze_scale_factor": "0.05", + "autovacuum_max_workers": "4", + "autovacuum_vacuum_cost_delay": "2ms", + "autovacuum_vacuum_scale_factor": "0.1", + "checkpoint_completion_target": "0.9", + "cron.database_name": "postgres", + "effective_cache_size": "512MB", + "effective_io_concurrency": "200", + "maintenance_work_mem": "128MB", + "max_connections": "300", + "max_prepared_transactions": "100", + "max_replication_slots": "10", + "max_wal_senders": "10", + "max_wal_size": "2GB", + "min_wal_size": "256MB", + "random_page_cost": "1.1", + "shared_buffers": "256MB", + "wal_buffers": "16MB", + "work_mem": "16MB" + }, + "pg_hba": [ + "host all all localhost trust", + "hostssl replication streaming_replica all cert" + ], + "syncReplicaElectionConstraint": { + "enabled": false + }, + "shared_preload_libraries": [ + "pg_cron", + "pg_documentdb_core", + "pg_documentdb" + ], + "extensions": [ + { + "name": "documentdb", + "image": { + "reference": "ghcr.io/documentdb/documentdb-kubernetes-operator/documentdb:1.0.0" + }, + "extension_control_path": [ + "share" + ], + "dynamic_library_path": [ + "lib" + ], + "ld_library_path": [ + "lib", + "system" + ] + } + ] + }, + "bootstrap": { + "recovery": { + "backup": { + "name": "bk" + } + } + }, + "storage": { + "size": "10Gi" + }, + "stopDelay": 30, + "affinity": {}, + "resources": {}, + "primaryUpdateMethod": "switchover", + "backup": { + "volumeSnapshot": { + "snapshotOwnerReference": "backup", + "onlineConfiguration": {} + }, + "target": "primary" + }, + "logLevel": "info", + "managed": { + "roles": [ + { + "name": "otel_monitor", + "ensure": "absent", + "connectionLimit": -1, + "inherit": true + } + ] + }, + "plugins": [ + { + "name": "cnpg-i-sidecar-injector.documentdb.io", + "enabled": true, + "parameters": { + "documentDbCredentialSecret": "documentdb-credentials", + "gatewayImage": "ghcr.io/documentdb/documentdb-kubernetes-operator/gateway:1.0.0" + } + } + ] + }, + "status": { + "managedRolesStatus": {}, + "topology": {}, + "secretsResourceVersion": {}, + "configMapResourceVersion": {}, + "certificates": {}, + "switchReplicaClusterStatus": {} + } +} diff --git a/operator/src/internal/cnpg/testdata/golden/recovery-pv.json b/operator/src/internal/cnpg/testdata/golden/recovery-pv.json new file mode 100644 index 00000000..db5cea34 --- /dev/null +++ b/operator/src/internal/cnpg/testdata/golden/recovery-pv.json @@ -0,0 +1,132 @@ +{ + "metadata": { + "name": "golden", + "namespace": "golden-ns", + "ownerReferences": [ + { + "apiVersion": "documentdb.io/preview", + "kind": "DocumentDB", + "name": "golden", + "uid": "00000000-0000-0000-0000-000000000001", + "controller": true, + "blockOwnerDeletion": true + } + ] + }, + "spec": { + "inheritedMetadata": { + "labels": { + "app": "golden", + "replica_type": "primary" + } + }, + "instances": 1, + "postgresql": { + "parameters": { + "autovacuum_analyze_scale_factor": "0.05", + "autovacuum_max_workers": "4", + "autovacuum_vacuum_cost_delay": "2ms", + "autovacuum_vacuum_scale_factor": "0.1", + "checkpoint_completion_target": "0.9", + "cron.database_name": "postgres", + "effective_cache_size": "512MB", + "effective_io_concurrency": "200", + "maintenance_work_mem": "128MB", + "max_connections": "300", + "max_prepared_transactions": "100", + "max_replication_slots": "10", + "max_wal_senders": "10", + "max_wal_size": "2GB", + "min_wal_size": "256MB", + "random_page_cost": "1.1", + "shared_buffers": "256MB", + "wal_buffers": "16MB", + "work_mem": "16MB" + }, + "pg_hba": [ + "host all all localhost trust", + "hostssl replication streaming_replica all cert" + ], + "syncReplicaElectionConstraint": { + "enabled": false + }, + "shared_preload_libraries": [ + "pg_cron", + "pg_documentdb_core", + "pg_documentdb" + ], + "extensions": [ + { + "name": "documentdb", + "image": { + "reference": "ghcr.io/documentdb/documentdb-kubernetes-operator/documentdb:1.0.0" + }, + "extension_control_path": [ + "share" + ], + "dynamic_library_path": [ + "lib" + ], + "ld_library_path": [ + "lib", + "system" + ] + } + ] + }, + "bootstrap": { + "recovery": { + "volumeSnapshots": { + "storage": { + "apiGroup": "", + "kind": "PersistentVolumeClaim", + "name": "golden-pv-recovery-temp" + } + } + } + }, + "storage": { + "size": "10Gi" + }, + "stopDelay": 30, + "affinity": {}, + "resources": {}, + "primaryUpdateMethod": "switchover", + "backup": { + "volumeSnapshot": { + "snapshotOwnerReference": "backup", + "onlineConfiguration": {} + }, + "target": "primary" + }, + "logLevel": "info", + "managed": { + "roles": [ + { + "name": "otel_monitor", + "ensure": "absent", + "connectionLimit": -1, + "inherit": true + } + ] + }, + "plugins": [ + { + "name": "cnpg-i-sidecar-injector.documentdb.io", + "enabled": true, + "parameters": { + "documentDbCredentialSecret": "documentdb-credentials", + "gatewayImage": "ghcr.io/documentdb/documentdb-kubernetes-operator/gateway:1.0.0" + } + } + ] + }, + "status": { + "managedRolesStatus": {}, + "topology": {}, + "secretsResourceVersion": {}, + "configMapResourceVersion": {}, + "certificates": {}, + "switchReplicaClusterStatus": {} + } +} diff --git a/operator/src/internal/cnpg/testdata/golden/replica-region-ignores-recovery.json b/operator/src/internal/cnpg/testdata/golden/replica-region-ignores-recovery.json new file mode 100644 index 00000000..925a1653 --- /dev/null +++ b/operator/src/internal/cnpg/testdata/golden/replica-region-ignores-recovery.json @@ -0,0 +1,130 @@ +{ + "metadata": { + "name": "golden", + "namespace": "golden-ns", + "ownerReferences": [ + { + "apiVersion": "documentdb.io/preview", + "kind": "DocumentDB", + "name": "golden", + "uid": "00000000-0000-0000-0000-000000000001", + "controller": true, + "blockOwnerDeletion": true + } + ] + }, + "spec": { + "inheritedMetadata": { + "labels": { + "app": "golden", + "replica_type": "primary" + } + }, + "instances": 1, + "postgresql": { + "parameters": { + "autovacuum_analyze_scale_factor": "0.05", + "autovacuum_max_workers": "4", + "autovacuum_vacuum_cost_delay": "2ms", + "autovacuum_vacuum_scale_factor": "0.1", + "checkpoint_completion_target": "0.9", + "cron.database_name": "postgres", + "effective_cache_size": "512MB", + "effective_io_concurrency": "200", + "maintenance_work_mem": "128MB", + "max_connections": "300", + "max_prepared_transactions": "100", + "max_replication_slots": "10", + "max_wal_senders": "10", + "max_wal_size": "2GB", + "min_wal_size": "256MB", + "random_page_cost": "1.1", + "shared_buffers": "256MB", + "wal_buffers": "16MB", + "work_mem": "16MB" + }, + "pg_hba": [ + "host all all localhost trust", + "hostssl replication streaming_replica all cert" + ], + "syncReplicaElectionConstraint": { + "enabled": false + }, + "shared_preload_libraries": [ + "pg_cron", + "pg_documentdb_core", + "pg_documentdb" + ], + "extensions": [ + { + "name": "documentdb", + "image": { + "reference": "ghcr.io/documentdb/documentdb-kubernetes-operator/documentdb:1.0.0" + }, + "extension_control_path": [ + "share" + ], + "dynamic_library_path": [ + "lib" + ], + "ld_library_path": [ + "lib", + "system" + ] + } + ] + }, + "bootstrap": { + "initdb": { + "postInitSQL": [ + "CREATE EXTENSION documentdb CASCADE", + "CREATE ROLE documentdb WITH LOGIN PASSWORD 'Admin100'", + "ALTER ROLE documentdb WITH SUPERUSER CREATEDB CREATEROLE REPLICATION BYPASSRLS" + ] + } + }, + "storage": { + "size": "10Gi" + }, + "stopDelay": 30, + "affinity": {}, + "resources": {}, + "primaryUpdateMethod": "switchover", + "backup": { + "volumeSnapshot": { + "snapshotOwnerReference": "backup", + "onlineConfiguration": {} + }, + "target": "primary" + }, + "logLevel": "info", + "managed": { + "roles": [ + { + "name": "otel_monitor", + "ensure": "absent", + "connectionLimit": -1, + "inherit": true + } + ] + }, + "plugins": [ + { + "name": "cnpg-i-sidecar-injector.documentdb.io", + "enabled": true, + "parameters": { + "documentDbCredentialSecret": "documentdb-credentials", + "gatewayImage": "ghcr.io/documentdb/documentdb-kubernetes-operator/gateway:1.0.0" + } + } + ] + }, + "status": { + "managedRolesStatus": {}, + "topology": {}, + "secretsResourceVersion": {}, + "configMapResourceVersion": {}, + "certificates": {}, + "switchReplicaClusterStatus": {} + } +} From 63066998c20ce65c8eb8f828ceec1ceb8b7e5254 Mon Sep 17 00:00:00 2001 From: Rayhan Hossain Date: Mon, 19 Oct 2026 16:39:55 -0700 Subject: [PATCH 02/27] refactor(cnpg): split resources from a product-neutral model Adds product.Resources and ComputeResourceSplitFor/ValidateResourcesFor so the resource carve-out no longer requires a DocumentDB resource. The DocumentDB entry points delegate to them unchanged. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 33b3cb3c-1b25-4ffb-bff1-ebf0f3d04058 Signed-off-by: Rayhan Hossain --- operator/src/internal/cnpg/resource_split.go | 16 ++++++++++--- .../src/internal/cnpg/resource_split_test.go | 18 +++++++++++++++ .../src/internal/cnpg/resource_validation.go | 23 +++++++++++-------- operator/src/internal/product/documentdb.go | 20 ++++++++++++++++ operator/src/internal/product/intent.go | 19 +++++++++++++++ 5 files changed, 83 insertions(+), 13 deletions(-) diff --git a/operator/src/internal/cnpg/resource_split.go b/operator/src/internal/cnpg/resource_split.go index 961695f1..94295bf6 100644 --- a/operator/src/internal/cnpg/resource_split.go +++ b/operator/src/internal/cnpg/resource_split.go @@ -10,6 +10,7 @@ import ( "k8s.io/apimachinery/pkg/api/resource" dbpreview "github.com/documentdb/documentdb-operator/api/preview" + "github.com/documentdb/documentdb-operator/internal/product" util "github.com/documentdb/documentdb-operator/internal/utils" ) @@ -103,9 +104,13 @@ func DefaultSplitConfig() SplitConfig { // Legacy behavior is preserved: when neither the envelope nor any per-container // value is set for a dimension, that dimension is left unmanaged (no limits). func ComputeResourceSplit(documentdb *dbpreview.DocumentDB, cfg SplitConfig) ResourceSplit { - res := documentdb.Spec.Resource - monitoring := documentdb.Spec.Monitoring != nil && documentdb.Spec.Monitoring.Enabled + return ComputeResourceSplitFor(product.DocumentDBResources(documentdb), monitoringEnabled(documentdb.Spec.Monitoring), cfg) +} +// ComputeResourceSplitFor is the product-neutral form of ComputeResourceSplit: +// it splits the given resource model, carving the OTel collector only when +// monitoring is enabled. +func ComputeResourceSplitFor(res product.Resources, monitoring bool, cfg SplitConfig) ResourceSplit { envelopeBytes := parseMemoryToBytes(res.Memory) split := ResourceSplit{MonitoringEnabled: monitoring} @@ -219,8 +224,13 @@ func subtractCPU(envelope string, reserved ...string) string { // --- helpers --- +// monitoringEnabled reports whether the monitoring sidecar is requested. +func monitoringEnabled(m *dbpreview.MonitoringSpec) bool { + return m != nil && m.Enabled +} + // componentCPU returns the component's CPU override, or "" when unset/zero. -func componentCPU(c *dbpreview.ComponentResources) string { +func componentCPU(c *product.ComponentResources) string { if c == nil { return "" } diff --git a/operator/src/internal/cnpg/resource_split_test.go b/operator/src/internal/cnpg/resource_split_test.go index 0a25b828..b03d1e03 100644 --- a/operator/src/internal/cnpg/resource_split_test.go +++ b/operator/src/internal/cnpg/resource_split_test.go @@ -4,9 +4,13 @@ package cnpg import ( + "strings" "testing" + "k8s.io/apimachinery/pkg/util/validation/field" + dbpreview "github.com/documentdb/documentdb-operator/api/preview" + "github.com/documentdb/documentdb-operator/internal/product" ) // prodSplitConfig mirrors the documented production defaults (18.75%, cap 32Gi, @@ -250,3 +254,17 @@ func TestValidateResources(t *testing.T) { t.Errorf("omitted+fully-specified cpu: unexpected errors %v", errs) } } + +func TestValidateResourcesFor_UsesProductFieldPath(t *testing.T) { + res := product.Resources{Gateway: &product.ComponentResources{Memory: "512Mi"}} + errs := ValidateResourcesFor(res, false, field.NewPath("spec", "compute"), prodSplitConfig()) + if len(errs) != 1 { + t.Fatalf("expected one error, got %v", errs) + } + if errs[0].Field != "spec.compute.memory" { + t.Errorf("field = %q, want spec.compute.memory", errs[0].Field) + } + if !strings.Contains(errs[0].Detail, "spec.compute.gateway and spec.compute.database") { + t.Errorf("detail %q does not reference the product field path", errs[0].Detail) + } +} diff --git a/operator/src/internal/cnpg/resource_validation.go b/operator/src/internal/cnpg/resource_validation.go index 55b169ab..6b9bc621 100644 --- a/operator/src/internal/cnpg/resource_validation.go +++ b/operator/src/internal/cnpg/resource_validation.go @@ -10,6 +10,7 @@ import ( "k8s.io/apimachinery/pkg/util/validation/field" dbpreview "github.com/documentdb/documentdb-operator/api/preview" + "github.com/documentdb/documentdb-operator/internal/product" ) // ValidateResources checks that spec.resource is internally consistent under the @@ -28,10 +29,12 @@ import ( // - If neither the envelope nor any container sets the dimension, it is left // unmanaged (no error). func ValidateResources(documentdb *dbpreview.DocumentDB, cfg SplitConfig) field.ErrorList { - res := documentdb.Spec.Resource - monitoring := documentdb.Spec.Monitoring != nil && documentdb.Spec.Monitoring.Enabled - base := field.NewPath("spec", "resource") + return ValidateResourcesFor(product.DocumentDBResources(documentdb), monitoringEnabled(documentdb.Spec.Monitoring), field.NewPath("spec", "resource"), cfg) +} +// ValidateResourcesFor is the product-neutral form of ValidateResources. base is +// the field path of the resource stanza on the product's custom resource. +func ValidateResourcesFor(res product.Resources, monitoring bool, base *field.Path, cfg SplitConfig) field.ErrorList { // Memory reservations, with defaults applied when a container omits the // value (gateway always defaults to its fraction of the envelope). memEnv := parseMemoryToBytes(res.Memory) @@ -112,7 +115,7 @@ func validateDimension(base *field.Path, d dimension) field.ErrorList { // database pin the dimension; any other partial configuration is invalid. if (d.gwSet || d.dbSet || d.otelSet) && !(d.gwSet && d.dbSet) { errs = append(errs, field.Required(base.Child(d.noun), - fmt.Sprintf("pod %s envelope is required unless %s is set on both spec.resource.gateway and spec.resource.database", d.noun, d.noun))) + fmt.Sprintf("pod %s envelope is required unless %s is set on both %s and %s", d.noun, d.noun, base.Child("gateway"), base.Child("database")))) } return errs } @@ -136,29 +139,29 @@ func validateDimension(base *field.Path, d dimension) field.ErrorList { func isSet(s string) bool { return s != "" && s != "0" } -func componentMemSet(c *dbpreview.ComponentResources) bool { +func componentMemSet(c *product.ComponentResources) bool { return c != nil && isSet(c.Memory) } -func componentCPUSet(c *dbpreview.ComponentResources) bool { +func componentCPUSet(c *product.ComponentResources) bool { return c != nil && isSet(c.CPU) } -func componentMemBytes(c *dbpreview.ComponentResources) int64 { +func componentMemBytes(c *product.ComponentResources) int64 { if c == nil { return 0 } return parseMemoryToBytes(c.Memory) } -func componentMemoryValue(c *dbpreview.ComponentResources) string { +func componentMemoryValue(c *product.ComponentResources) string { if c == nil { return "" } return c.Memory } -func componentCPUValue(c *dbpreview.ComponentResources) string { +func componentCPUValue(c *product.ComponentResources) string { if c == nil { return "" } @@ -168,7 +171,7 @@ func componentCPUValue(c *dbpreview.ComponentResources) string { // milliCPUString renders a milli-CPU count as a Kubernetes quantity (e.g. "500m"). func milliCPUString(m int64) string { return fmt.Sprintf("%dm", m) } -func componentCPUMilli(c *dbpreview.ComponentResources) int64 { +func componentCPUMilli(c *product.ComponentResources) int64 { if c == nil { return 0 } diff --git a/operator/src/internal/product/documentdb.go b/operator/src/internal/product/documentdb.go index eec74391..337dc8ba 100644 --- a/operator/src/internal/product/documentdb.go +++ b/operator/src/internal/product/documentdb.go @@ -73,6 +73,26 @@ func (a DocumentDBAdapter) GatewayImage(db *dbpreview.DocumentDB) string { ) } +// DocumentDBResources maps spec.resource of a DocumentDB onto the neutral +// resource model consumed by the resource split and validation. +func DocumentDBResources(db *dbpreview.DocumentDB) Resources { + res := db.Spec.Resource + return Resources{ + Memory: res.Memory, + CPU: res.CPU, + Database: toComponentResources(res.Database), + Gateway: toComponentResources(res.Gateway), + OTel: toComponentResources(res.OTel), + } +} + +func toComponentResources(c *dbpreview.ComponentResources) *ComponentResources { + if c == nil { + return nil + } + return &ComponentResources{Memory: c.Memory, CPU: c.CPU} +} + // ToClusterIntent maps a DocumentDB custom resource onto the product-neutral // ClusterIntent, resolving product-varying images, credential secret, and plugin // names via the product profile. diff --git a/operator/src/internal/product/intent.go b/operator/src/internal/product/intent.go index 06ad7c5b..5cb90d37 100644 --- a/operator/src/internal/product/intent.go +++ b/operator/src/internal/product/intent.go @@ -37,6 +37,25 @@ type Storage struct { PvcSize string } +// ComponentResources is an optional per-container memory/CPU override. +// Empty or "0" values mean "unset". +type ComponentResources struct { + Memory string + CPU string +} + +// Resources is the pod resource envelope and optional per-container overrides +// the operator splits across the PostgreSQL, gateway, and OTel containers. +type Resources struct { + // Memory and CPU are the optional pod-level envelopes. + Memory string + CPU string + // Database, Gateway, and OTel are optional per-container overrides. + Database *ComponentResources + Gateway *ComponentResources + OTel *ComponentResources +} + // Identity carries the owning custom resource's identity for owner references // and resource labels. type Identity struct { From e592a227b8de5de486265d8124e0030918e566c3 Mon Sep 17 00:00:00 2001 From: Rayhan Hossain Date: Mon, 19 Oct 2026 16:41:09 -0700 Subject: [PATCH 03/27] feat(product): carry all renderer inputs on ClusterIntent Adds resources, monitoring, TLS, log level, stop delay, user parameters,the ChangeStreams gate and image pull policies to the intent and maps themfrom the DocumentDB custom resource. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 33b3cb3c-1b25-4ffb-bff1-ebf0f3d04058 Signed-off-by: Rayhan Hossain --- operator/src/internal/product/documentdb.go | 38 ++++++++---- .../src/internal/product/documentdb_test.go | 54 ++++++++++++++++ operator/src/internal/product/intent.go | 62 ++++++++++++++++--- 3 files changed, 136 insertions(+), 18 deletions(-) diff --git a/operator/src/internal/product/documentdb.go b/operator/src/internal/product/documentdb.go index 337dc8ba..a1cdc923 100644 --- a/operator/src/internal/product/documentdb.go +++ b/operator/src/internal/product/documentdb.go @@ -120,13 +120,23 @@ func (a DocumentDBAdapter) ToClusterIntent(db *dbpreview.DocumentDB) ClusterInte postgresImage = util.PostgresImage() } - var pg Postgres + pg := Postgres{ + LogLevel: db.Spec.LogLevel, + StopDelay: db.Spec.Timeouts.StopDelay, + } if db.Spec.Postgres != nil { - pg = Postgres{ - UID: db.Spec.Postgres.UID, - GID: db.Spec.Postgres.GID, - PostInitSQL: db.Spec.Postgres.PostInitSQL, - } + pg.UID = db.Spec.Postgres.UID + pg.GID = db.Spec.Postgres.GID + pg.PostInitSQL = db.Spec.Postgres.PostInitSQL + pg.Parameters = db.Spec.Postgres.Parameters + } + + var tls TLS + if db.Status.TLS != nil && db.Status.TLS.Ready { + tls.GatewaySecret = db.Status.TLS.SecretName + } + if db.Spec.TLS != nil { + tls.Postgres = db.Spec.TLS.Postgres } var bootstrap Bootstrap @@ -141,10 +151,12 @@ func (a DocumentDBAdapter) ToClusterIntent(db *dbpreview.DocumentDB) ClusterInte return ClusterIntent{ Images: Images{ - PostgresExtension: a.ExtensionImage(db), - Gateway: a.GatewayImage(db), - Postgres: postgresImage, - PullSecrets: db.Spec.ImagePullSecrets, + PostgresExtension: a.ExtensionImage(db), + PostgresExtensionPullPolicy: ParsePullPolicy(os.Getenv(util.DOCUMENTDB_IMAGE_PULL_POLICY_ENV)), + Gateway: a.GatewayImage(db), + GatewayPullPolicy: os.Getenv(util.GATEWAY_IMAGE_PULL_POLICY_ENV), + Postgres: postgresImage, + PullSecrets: db.Spec.ImagePullSecrets, }, Topology: Topology{ Instances: db.Spec.InstancesPerNode, @@ -153,6 +165,9 @@ func (a DocumentDBAdapter) ToClusterIntent(db *dbpreview.DocumentDB) ClusterInte Storage: Storage{ PvcSize: db.Spec.Resource.Storage.PvcSize, }, + Resources: DocumentDBResources(db), + Monitoring: db.Spec.Monitoring, + TLS: tls, Identity: Identity{ Name: db.Name, UID: db.UID, @@ -161,7 +176,8 @@ func (a DocumentDBAdapter) ToClusterIntent(db *dbpreview.DocumentDB) ClusterInte }, Postgres: pg, FeatureGates: FeatureGates{ - IOUring: dbpreview.IsFeatureGateEnabled(db, dbpreview.FeatureGateIOUring), + ChangeStreams: dbpreview.IsFeatureGateEnabled(db, dbpreview.FeatureGateChangeStreams), + IOUring: dbpreview.IsFeatureGateEnabled(db, dbpreview.FeatureGateIOUring), }, Bootstrap: bootstrap, CredentialSecret: credentialSecret, diff --git a/operator/src/internal/product/documentdb_test.go b/operator/src/internal/product/documentdb_test.go index 72fa0737..cf4a5bc4 100644 --- a/operator/src/internal/product/documentdb_test.go +++ b/operator/src/internal/product/documentdb_test.go @@ -6,6 +6,8 @@ package product import ( "testing" + cnpgv1 "github.com/cloudnative-pg/cloudnative-pg/api/v1" + dbpreview "github.com/documentdb/documentdb-operator/api/preview" util "github.com/documentdb/documentdb-operator/internal/utils" ) @@ -137,3 +139,55 @@ func TestDocumentDBAdapterPostgresImage(t *testing.T) { } }) } + +// TestDocumentDBAdapterRendererInputs covers the per-instance renderer inputs the +// adapter maps from the custom resource onto the intent. +func TestDocumentDBAdapterRendererInputs(t *testing.T) { + t.Setenv(util.DOCUMENTDB_IMAGE_PULL_POLICY_ENV, "bogus") + t.Setenv(util.GATEWAY_IMAGE_PULL_POLICY_ENV, "Never") + certs := &cnpgv1.CertificatesConfiguration{ServerTLSSecret: "pg-tls"} + db := &dbpreview.DocumentDB{Spec: dbpreview.DocumentDBSpec{ + Resource: dbpreview.Resource{ + Memory: "8Gi", + CPU: "2", + Gateway: &dbpreview.ComponentResources{Memory: "1Gi"}, + }, + Monitoring: &dbpreview.MonitoringSpec{Enabled: true}, + LogLevel: "debug", + Timeouts: dbpreview.Timeouts{StopDelay: 90}, + Postgres: &dbpreview.PostgresSpec{Parameters: map[string]string{"work_mem": "8MB"}}, + FeatureGates: map[string]bool{dbpreview.FeatureGateChangeStreams: true}, + TLS: &dbpreview.TLSConfiguration{Postgres: certs}, + }} + db.Status.TLS = &dbpreview.TLSStatus{Ready: false, SecretName: "gw-tls"} + + intent := DocumentDBAdapter{}.ToClusterIntent(db) + + if intent.Resources.Memory != "8Gi" || intent.Resources.CPU != "2" || + intent.Resources.Gateway == nil || intent.Resources.Gateway.Memory != "1Gi" || intent.Resources.Database != nil { + t.Errorf("Resources = %+v", intent.Resources) + } + if intent.Monitoring != db.Spec.Monitoring { + t.Errorf("Monitoring not passed through") + } + if intent.Postgres.LogLevel != "debug" || intent.Postgres.StopDelay != 90 || intent.Postgres.Parameters["work_mem"] != "8MB" { + t.Errorf("Postgres = %+v", intent.Postgres) + } + if !intent.FeatureGates.ChangeStreams || intent.FeatureGates.IOUring { + t.Errorf("FeatureGates = %+v", intent.FeatureGates) + } + if intent.TLS.GatewaySecret != "" { + t.Errorf("gateway TLS secret must stay empty until TLS is ready, got %q", intent.TLS.GatewaySecret) + } + if intent.TLS.Postgres != certs { + t.Errorf("Postgres certificates not passed through") + } + if intent.Images.PostgresExtensionPullPolicy != "" || intent.Images.GatewayPullPolicy != "Never" { + t.Errorf("pull policies = %q/%q", intent.Images.PostgresExtensionPullPolicy, intent.Images.GatewayPullPolicy) + } + + db.Status.TLS.Ready = true + if got := (DocumentDBAdapter{}).ToClusterIntent(db).TLS.GatewaySecret; got != "gw-tls" { + t.Errorf("ready gateway TLS secret = %q, want gw-tls", got) + } +} diff --git a/operator/src/internal/product/intent.go b/operator/src/internal/product/intent.go index 5cb90d37..dfbac4eb 100644 --- a/operator/src/internal/product/intent.go +++ b/operator/src/internal/product/intent.go @@ -7,14 +7,22 @@ import ( cnpgv1 "github.com/cloudnative-pg/cloudnative-pg/api/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" + + dbpreview "github.com/documentdb/documentdb-operator/api/preview" ) // Images holds the fully-resolved container images for a cluster. type Images struct { // PostgresExtension is the extension image mounted into PostgreSQL via ImageVolume. PostgresExtension string - // Gateway is the MongoDB wire-protocol gateway sidecar image. + // PostgresExtensionPullPolicy is the ImageVolume pull policy for the + // extension image; empty leaves the Kubernetes default. + PostgresExtensionPullPolicy corev1.PullPolicy + // Gateway is the product wire-protocol gateway sidecar image. Gateway string + // GatewayPullPolicy is passed through to the sidecar injector verbatim; + // empty lets the injector apply its default. + GatewayPullPolicy string // Postgres is the base PostgreSQL image. Postgres string // PullSecrets are the image pull secrets shared by all cluster containers. @@ -65,22 +73,41 @@ type Identity struct { Kind string } -// Postgres carries the operator-managed PostgreSQL process and init tuning taken -// from the custom resource. Parameter/GUC assembly stays product-specific in the -// builder and is not represented here yet. +// Postgres carries the per-instance PostgreSQL process, init, and parameter +// tuning taken from the custom resource. Product-wide defaults (extension, +// libraries, bootstrap SQL, protected GUCs) live on ProductProfile.Postgres. type Postgres struct { // UID and GID are the process identity overrides; nil leaves the CNPG default. UID *int64 GID *int64 - // PostInitSQL is appended to the mandatory bootstrap SQL. + // PostInitSQL is appended to the product's mandatory bootstrap SQL. PostInitSQL []string + // Parameters are user GUC overrides. They win over operator defaults but + // never over protected parameters. + Parameters map[string]string + // LogLevel is the CNPG instance manager log level; empty means "info". + LogLevel string + // StopDelay is the CNPG maxStopDelay in seconds; 0 uses the operator default. + StopDelay int32 } // FeatureGates carries the resolved feature-gate flags the builder acts on. type FeatureGates struct { + // ChangeStreams forces wal_level=logical. + ChangeStreams bool + // IOUring forces io_method=io_uring and relaxes the seccomp profile. IOUring bool } +// TLS carries the resolved TLS material for the cluster. +type TLS struct { + // GatewaySecret is the ready gateway certificate secret; empty when gateway + // TLS is not (yet) ready. + GatewaySecret string + // Postgres is the optional CNPG server/client certificate configuration. + Postgres *cnpgv1.CertificatesConfiguration +} + // Recovery describes a bootstrap-from-source request. A nil Recovery on Bootstrap // means default initialization. type Recovery struct { @@ -95,8 +122,8 @@ type Bootstrap struct { // ClusterIntent is the product-neutral desired state the reconciler renders into // a CNPG Cluster. Product adapters populate it from their custom resource; the -// reconciler consumes it without product-branding logic. Fields are added to -// this struct as the builder is progressively rewired onto the seam. +// renderer consumes only the intent (and the ProductProfile embedded in it), so +// it carries no product-branding logic. type ClusterIntent struct { // Images are the resolved extension, gateway, and postgres images. Images Images @@ -107,6 +134,16 @@ type ClusterIntent struct { // Storage is the persistent volume request. Storage Storage + // Resources is the pod resource envelope and per-container overrides. + Resources Resources + + // Monitoring is the optional OTel collector sidecar configuration. The + // MonitoringSpec API type is shared across products. + Monitoring *dbpreview.MonitoringSpec + + // TLS is the resolved gateway and PostgreSQL TLS material. + TLS TLS + // Identity is the owning custom resource's identity. Identity Identity @@ -130,3 +167,14 @@ type ClusterIntent struct { // Product is the profile this intent was produced from. Product ProductProfile } + +// ParsePullPolicy converts a string to a corev1.PullPolicy, returning "" for +// unrecognized values so callers fall back to the Kubernetes default. +func ParsePullPolicy(value string) corev1.PullPolicy { + switch corev1.PullPolicy(value) { + case corev1.PullAlways, corev1.PullNever, corev1.PullIfNotPresent: + return corev1.PullPolicy(value) + default: + return "" + } +} From ac942b5cd567cd190015bc115e2070582f252a18 Mon Sep 17 00:00:00 2001 From: Rayhan Hossain Date: Mon, 19 Oct 2026 16:42:52 -0700 Subject: [PATCH 04/27] refactor(cnpg): render CNPG clusters from the intent and product profile ProductProfile gains a PostgresProfile (extension, shared libraries, bootstrap SQL, pg_hba, protected GUCs, replication admin role) and GetCnpgClusterSpecFromIntent no longer takes a DocumentDB: every product-varying value comes from the ClusterIntent. Golden snapshots confirm the DocumentDB rendering is byte-identical. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 33b3cb3c-1b25-4ffb-bff1-ebf0f3d04058 Signed-off-by: Rayhan Hossain --- operator/src/internal/cnpg/cnpg_cluster.go | 130 +++++++----------- .../src/internal/cnpg/cnpg_intent_test.go | 4 +- operator/src/internal/cnpg/pg_defaults.go | 39 ++++-- .../controller/documentdb_controller.go | 2 +- operator/src/internal/product/documentdb.go | 25 ++++ operator/src/internal/product/profile.go | 35 +++++ 6 files changed, 137 insertions(+), 98 deletions(-) diff --git a/operator/src/internal/cnpg/cnpg_cluster.go b/operator/src/internal/cnpg/cnpg_cluster.go index b9759790..4269fbac 100644 --- a/operator/src/internal/cnpg/cnpg_cluster.go +++ b/operator/src/internal/cnpg/cnpg_cluster.go @@ -31,20 +31,20 @@ func GetCnpgClusterSpec(req ctrl.Request, documentdb *dbpreview.DocumentDB, docu if documentdbImage != "" { intent.Images.PostgresExtension = documentdbImage } - return GetCnpgClusterSpecFromIntent(req, documentdb, intent, serviceAccountName, storageClass, isPrimaryRegion, log) + return GetCnpgClusterSpecFromIntent(req, intent, serviceAccountName, storageClass, isPrimaryRegion, log) } // GetCnpgClusterSpecFromIntent renders a CNPG Cluster from a product-neutral -// ClusterIntent. This is the seam the reconciler drives: product-varying values -// (images, credential secret, plugin name) come from the intent rather than from -// product-specific lookups. -func GetCnpgClusterSpecFromIntent(req ctrl.Request, documentdb *dbpreview.DocumentDB, intent product.ClusterIntent, serviceAccountName, storageClass string, isPrimaryRegion bool, log logr.Logger) *cnpgv1.Cluster { - split := ComputeResourceSplit(documentdb, DefaultSplitConfig()) +// ClusterIntent. This is the seam every product drives: all product-varying +// values come from the intent and its embedded ProductProfile, so the renderer +// has no knowledge of any product custom resource. +func GetCnpgClusterSpecFromIntent(req ctrl.Request, intent product.ClusterIntent, serviceAccountName, storageClass string, isPrimaryRegion bool, log logr.Logger) *cnpgv1.Cluster { + split := ComputeResourceSplitFor(intent.Resources, monitoringEnabled(intent.Monitoring), DefaultSplitConfig()) sidecarPluginName := intent.SidecarInjectorPlugin gatewayImage := intent.Images.Gateway - log.Info("Creating CNPG cluster with gateway image", "gatewayImage", gatewayImage, "documentdbName", documentdb.Name, "specGatewayImage", imageGateway(documentdb)) + log.Info("Creating CNPG cluster with gateway image", "gatewayImage", gatewayImage, "name", intent.Identity.Name, "product", intent.Product.Name) credentialSecretName := intent.CredentialSecret @@ -58,9 +58,9 @@ func GetCnpgClusterSpecFromIntent(req ctrl.Request, documentdb *dbpreview.Docume // This addresses the fact that ImageVolume sources DO support pull policies // (via corev1.ImageVolumeSource.PullPolicy), unlike regular container images // which only support pull policies on container specs. - extensionImageSource := corev1.ImageVolumeSource{Reference: intent.Images.PostgresExtension} - if pullPolicy := parsePullPolicy(os.Getenv(util.DOCUMENTDB_IMAGE_PULL_POLICY_ENV)); pullPolicy != "" { - extensionImageSource.PullPolicy = pullPolicy + extensionImageSource := corev1.ImageVolumeSource{ + Reference: intent.Images.PostgresExtension, + PullPolicy: intent.Images.PostgresExtensionPullPolicy, } return &cnpgv1.Cluster{ @@ -73,8 +73,8 @@ func GetCnpgClusterSpecFromIntent(req ctrl.Request, documentdb *dbpreview.Docume Kind: intent.Identity.Kind, Name: intent.Identity.Name, UID: intent.Identity.UID, - Controller: &[]bool{true}[0], // This cluster is controlled by the DocumentDB instance - BlockOwnerDeletion: &[]bool{true}[0], // Block DocumentDB deletion until cluster is deleted + Controller: &[]bool{true}[0], // This cluster is controlled by the product instance + BlockOwnerDeletion: &[]bool{true}[0], // Block owner deletion until cluster is deleted }, }, }, @@ -94,7 +94,7 @@ func GetCnpgClusterSpecFromIntent(req ctrl.Request, documentdb *dbpreview.Docume "gatewayImage": gatewayImage, "documentDbCredentialSecret": credentialSecretName, } - if pullPolicy := os.Getenv(util.GATEWAY_IMAGE_PULL_POLICY_ENV); pullPolicy != "" { + if pullPolicy := intent.Images.GatewayPullPolicy; pullPolicy != "" { params["gatewayImagePullPolicy"] = pullPolicy } addPluginParamIfSet(params, util.PLUGIN_PARAM_GATEWAY_MEMORY_REQUEST, split.Gateway.MemoryRequest) @@ -102,26 +102,26 @@ func GetCnpgClusterSpecFromIntent(req ctrl.Request, documentdb *dbpreview.Docume addPluginParamIfSet(params, util.PLUGIN_PARAM_GATEWAY_CPU_REQUEST, split.Gateway.CPURequest) addPluginParamIfSet(params, util.PLUGIN_PARAM_GATEWAY_CPU_LIMIT, split.Gateway.CPULimit) // If TLS is ready, surface secret name to plugin so it can mount certs. - if documentdb.Status.TLS != nil && documentdb.Status.TLS.Ready && documentdb.Status.TLS.SecretName != "" { - params["gatewayTLSSecret"] = documentdb.Status.TLS.SecretName + if intent.TLS.GatewaySecret != "" { + params["gatewayTLSSecret"] = intent.TLS.GatewaySecret } // Pass monitoring parameters to plugin for OTel sidecar injection. // Sidecar is only injected when monitoring is enabled. // Config hash triggers operator-initiated rolling restart on config changes. if split.MonitoringEnabled { params["otelCollectorImage"] = util.OtelCollectorImage() - params["otelConfigMapName"] = otelcfg.ConfigMapName(documentdb.Name) + params["otelConfigMapName"] = otelcfg.ConfigMapName(intent.Identity.Name) addPluginParamIfSet(params, util.PLUGIN_PARAM_OTEL_MEMORY_REQUEST, split.OTel.MemoryRequest) addPluginParamIfSet(params, util.PLUGIN_PARAM_OTEL_MEMORY_LIMIT, split.OTel.MemoryLimit) addPluginParamIfSet(params, util.PLUGIN_PARAM_OTEL_CPU_REQUEST, split.OTel.CPURequest) addPluginParamIfSet(params, util.PLUGIN_PARAM_OTEL_CPU_LIMIT, split.OTel.CPULimit) - if promPort := otelcfg.ResolvePrometheusPort(documentdb.Spec.Monitoring); promPort > 0 { + if promPort := otelcfg.ResolvePrometheusPort(intent.Monitoring); promPort > 0 { params["prometheusPort"] = fmt.Sprintf("%d", promPort) } // Compute config hash for change detection. The operator triggers a // rolling restart (via restart annotation) when plugin parameters // change, ensuring pods pick up new config. - if configData, err := otelcfg.GenerateConfigMapData(documentdb.Name, req.Namespace, documentdb.Spec.Monitoring); err == nil { + if configData, err := otelcfg.GenerateConfigMapData(intent.Identity.Name, req.Namespace, intent.Monitoring); err == nil { params["otelConfigHash"] = otelcfg.HashConfigMapData(configData) } else { log.Error(err, "Failed to generate OTel config hash; config changes may not trigger rolling restart") @@ -133,10 +133,10 @@ func GetCnpgClusterSpecFromIntent(req ctrl.Request, documentdb *dbpreview.Docume Parameters: params, }} }(), - PostgresConfiguration: buildPostgresConfiguration(documentdb, extensionImageSource, split.PostgresMemoryBytes), + PostgresConfiguration: buildPostgresConfiguration(intent, extensionImageSource, split.PostgresMemoryBytes), Bootstrap: bootstrapConfigurationFromIntent(intent, isPrimaryRegion, log), - LogLevel: cmp.Or(documentdb.Spec.LogLevel, "info"), - Certificates: postgresCertificates(documentdb), + LogLevel: cmp.Or(intent.Postgres.LogLevel, "info"), + Certificates: intent.TLS.Postgres, Backup: &cnpgv1.BackupConfiguration{ VolumeSnapshot: &cnpgv1.VolumeSnapshotConfiguration{ SnapshotOwnerReference: "backup", // Set owner reference to 'backup' so that snapshots are deleted when Backup resource is deleted @@ -146,10 +146,10 @@ func GetCnpgClusterSpecFromIntent(req ctrl.Request, documentdb *dbpreview.Docume Affinity: intent.Topology.Affinity, Resources: buildResourceRequirements(split.Postgres), } - spec.MaxStopDelay = getMaxStopDelayOrDefault(documentdb) + spec.MaxStopDelay = stopDelayOrDefault(intent.Postgres.StopDelay) applyPostgresProcessIdentity(&spec, intent) applyIOUringSeccomp(&spec, intent) - applyOtelMonitorRole(&spec, documentdb) + applyOtelMonitorRole(&spec, intent.Monitoring) return spec }(), @@ -177,7 +177,7 @@ func bootstrapConfigurationFromIntent(intent product.ClusterIntent, isPrimaryReg // Handle backup recovery if recovery.BackupName != "" { - log.Info("DocumentDB cluster will be bootstrapped from backup", "backupName", recovery.BackupName) + log.Info("Cluster will be bootstrapped from backup", "product", intent.Product.Name, "backupName", recovery.BackupName) return &cnpgv1.BootstrapConfiguration{ Recovery: &cnpgv1.BootstrapRecovery{ Backup: &cnpgv1.BackupSource{ @@ -190,7 +190,7 @@ func bootstrapConfigurationFromIntent(intent product.ClusterIntent, isPrimaryReg // Handle PV recovery (via temporary PVC created by the controller) if recovery.PersistentVolumeName != "" { tempPVCName := util.TempPVCNameForPVRecovery(intent.Identity.Name) - log.Info("DocumentDB cluster will be bootstrapped from PV via temp PVC", + log.Info("Cluster will be bootstrapped from PV via temp PVC", "product", intent.Product.Name, "pvName", recovery.PersistentVolumeName, "tempPVC", tempPVCName) return &cnpgv1.BootstrapConfiguration{ Recovery: &cnpgv1.BootstrapRecovery{ @@ -216,11 +216,7 @@ func getBootstrapConfiguration(documentdb *dbpreview.DocumentDB, isPrimaryRegion } func defaultBootstrapConfigurationFromIntent(intent product.ClusterIntent) *cnpgv1.BootstrapConfiguration { - postInitSQL := []string{ - "CREATE EXTENSION documentdb CASCADE", - "CREATE ROLE documentdb WITH LOGIN PASSWORD 'Admin100'", - "ALTER ROLE documentdb WITH SUPERUSER CREATEDB CREATEROLE REPLICATION BYPASSRLS", - } + postInitSQL := append([]string{}, intent.Product.Postgres.BootstrapSQL...) if len(intent.Postgres.PostInitSQL) > 0 { postInitSQL = append(postInitSQL, intent.Postgres.PostInitSQL...) } @@ -239,8 +235,13 @@ func getDefaultBootstrapConfiguration(documentdb *dbpreview.DocumentDB) *cnpgv1. // getMaxStopDelayOrDefault returns StopDelay if set, otherwise util.CNPG_DEFAULT_STOP_DELAY func getMaxStopDelayOrDefault(documentdb *dbpreview.DocumentDB) int32 { - if documentdb.Spec.Timeouts.StopDelay != 0 { - return documentdb.Spec.Timeouts.StopDelay + return stopDelayOrDefault(documentdb.Spec.Timeouts.StopDelay) +} + +// stopDelayOrDefault returns stopDelay if set, otherwise util.CNPG_DEFAULT_STOP_DELAY. +func stopDelayOrDefault(stopDelay int32) int32 { + if stopDelay != 0 { + return stopDelay } return util.CNPG_DEFAULT_STOP_DELAY } @@ -300,33 +301,6 @@ func parseResourceQuantity(value string) (resource.Quantity, bool) { return quantity, true } -// parsePullPolicy converts a string to a corev1.PullPolicy. -// Returns empty string for unrecognized values. -func parsePullPolicy(value string) corev1.PullPolicy { - switch corev1.PullPolicy(value) { - case corev1.PullAlways, corev1.PullNever, corev1.PullIfNotPresent: - return corev1.PullPolicy(value) - default: - return "" - } -} - -// imageGateway returns spec.image.gateway or empty string when unset. -// Nil-safe. -func imageGateway(documentdb *dbpreview.DocumentDB) string { - if documentdb == nil || documentdb.Spec.Image == nil { - return "" - } - return documentdb.Spec.Image.Gateway -} - -func postgresCertificates(documentdb *dbpreview.DocumentDB) *cnpgv1.CertificatesConfiguration { - if documentdb.Spec.TLS == nil { - return nil - } - return documentdb.Spec.TLS.Postgres -} - // toCNPGImagePullSecrets translates a list of corev1.LocalObjectReference // (the Kubernetes-native shape used on spec.imagePullSecrets) into the // CNPG-flavoured cnpgv1.LocalObjectReference shape that @@ -387,15 +361,12 @@ func applyIOUringSeccomp(spec *cnpgv1.ClusterSpec, intent product.ClusterIntent) // PostgreSQL host authentication currently uses trust, so a generated password // would not be checked. Disable the role password explicitly until authentication // is tightened rather than creating an unused credential and widening Secret RBAC. -func applyOtelMonitorRole(spec *cnpgv1.ClusterSpec, documentdb *dbpreview.DocumentDB) { - if documentdb == nil { - return - } +func applyOtelMonitorRole(spec *cnpgv1.ClusterSpec, monitoring *dbpreview.MonitoringSpec) { if spec.Managed == nil { spec.Managed = &cnpgv1.ManagedConfiguration{} } role := absentOtelMonitorRole() - if documentdb.Spec.Monitoring != nil && documentdb.Spec.Monitoring.Enabled { + if monitoringEnabled(monitoring) { // The current health query is SELECT 1, so the role needs LOGIN only // and is not granted broad monitoring memberships. role = cnpgv1.RoleConfiguration{ @@ -423,30 +394,25 @@ func absentOtelMonitorRole() cnpgv1.RoleConfiguration { } } -// for the cluster. -// -// The operator declares the DocumentDB extension via CNPG's Extensions -// stanza (mounted from spec.image.documentDB as an ImageVolumeSource), -// sets a fixed AdditionalLibraries list, and applies a small set of +// buildPostgresConfiguration renders the PostgreSQL configuration for the +// cluster from the product profile: the product extension is declared via CNPG's +// Extensions stanza (mounted from the extension image as an ImageVolumeSource), +// alongside the product's AdditionalLibraries and pg_hba, plus the merged // operator-managed GUCs. -func buildPostgresConfiguration(documentdb *dbpreview.DocumentDB, extensionImageSource corev1.ImageVolumeSource, pgMemoryBytes int64) cnpgv1.PostgresConfiguration { - pgHBA := []string{ - "host all all localhost trust", - "hostssl replication streaming_replica all cert", - } - +func buildPostgresConfiguration(intent product.ClusterIntent, extensionImageSource corev1.ImageVolumeSource, pgMemoryBytes int64) cnpgv1.PostgresConfiguration { + pg := intent.Product.Postgres return cnpgv1.PostgresConfiguration{ Extensions: []cnpgv1.ExtensionConfiguration{ { - Name: "documentdb", + Name: pg.Extension.Name, ImageVolumeSource: extensionImageSource, - DynamicLibraryPath: []string{"lib"}, - ExtensionControlPath: []string{"share"}, - LdLibraryPath: []string{"lib", "system"}, + DynamicLibraryPath: pg.Extension.DynamicLibraryPath, + ExtensionControlPath: pg.Extension.ExtensionControlPath, + LdLibraryPath: pg.Extension.LdLibraryPath, }, }, - AdditionalLibraries: []string{"pg_cron", "pg_documentdb_core", "pg_documentdb"}, - Parameters: MergeParameters(documentdb, pgMemoryBytes), - PgHBA: pgHBA, + AdditionalLibraries: pg.AdditionalLibraries, + Parameters: MergeParametersFor(intent, pgMemoryBytes), + PgHBA: pg.PgHBA, } } diff --git a/operator/src/internal/cnpg/cnpg_intent_test.go b/operator/src/internal/cnpg/cnpg_intent_test.go index 1aee58ad..86c36880 100644 --- a/operator/src/internal/cnpg/cnpg_intent_test.go +++ b/operator/src/internal/cnpg/cnpg_intent_test.go @@ -44,7 +44,7 @@ var _ = Describe("GetCnpgClusterSpecFromIntent", func() { SidecarInjectorPlugin: "custom-injector.example.io", } - result := GetCnpgClusterSpecFromIntent(newRequest(), newDocumentDB(), intent, "test-sa", "", true, log) + result := GetCnpgClusterSpecFromIntent(newRequest(), intent, "test-sa", "", true, log) Expect(result.Spec.PostgresConfiguration.Extensions[0].ImageVolumeSource.Reference).To(Equal("reg/ext:test")) Expect(result.Spec.Plugins[0].Name).To(Equal("custom-injector.example.io")) @@ -56,7 +56,7 @@ var _ = Describe("GetCnpgClusterSpecFromIntent", func() { documentdb := newDocumentDB() intent := product.DocumentDBAdapter{}.ToClusterIntent(documentdb) - fromIntent := GetCnpgClusterSpecFromIntent(newRequest(), documentdb, intent, "test-sa", "", true, log) + fromIntent := GetCnpgClusterSpecFromIntent(newRequest(), intent, "test-sa", "", true, log) fromWrapper := GetCnpgClusterSpec(newRequest(), documentdb, "", "test-sa", "", true, log) Expect(fromIntent.Spec.PostgresConfiguration.Extensions[0].ImageVolumeSource.Reference). diff --git a/operator/src/internal/cnpg/pg_defaults.go b/operator/src/internal/cnpg/pg_defaults.go index 54d8eb51..78a23994 100644 --- a/operator/src/internal/cnpg/pg_defaults.go +++ b/operator/src/internal/cnpg/pg_defaults.go @@ -7,6 +7,7 @@ import ( "fmt" dbpreview "github.com/documentdb/documentdb-operator/api/preview" + "github.com/documentdb/documentdb-operator/internal/product" ) // formatMB formats a megabyte value as a PostgreSQL size string. @@ -79,29 +80,43 @@ func StaticDefaults() map[string]string { } // ProtectedParameters returns parameters that are always force-set by the -// operator and cannot be overridden by users. +// operator for a DocumentDB instance and cannot be overridden by users. func ProtectedParameters(documentdb *dbpreview.DocumentDB) map[string]string { + return ProtectedParametersFor(product.DocumentDBAdapter{}.ToClusterIntent(documentdb)) +} + +// ProtectedParametersFor returns the core protected parameters, overlaid with +// the product's protected parameters and the feature-gate driven GUCs. +func ProtectedParametersFor(intent product.ClusterIntent) map[string]string { params := map[string]string{ - "cron.database_name": "postgres", "max_replication_slots": "10", "max_wal_senders": "10", "max_prepared_transactions": "100", } - if dbpreview.IsFeatureGateEnabled(documentdb, dbpreview.FeatureGateChangeStreams) { + for k, v := range intent.Product.Postgres.ProtectedParameters { + params[k] = v + } + if intent.FeatureGates.ChangeStreams { params["wal_level"] = "logical" } - if dbpreview.IsFeatureGateEnabled(documentdb, dbpreview.FeatureGateIOUring) { + if intent.FeatureGates.IOUring { params["io_method"] = "io_uring" } return params } -// MergeParameters merges all parameter sources in priority order (last write wins): +// MergeParameters merges all parameter sources for a DocumentDB instance; see +// MergeParametersFor. +func MergeParameters(documentdb *dbpreview.DocumentDB, memoryLimitBytes int64) map[string]string { + return MergeParametersFor(product.DocumentDBAdapter{}.ToClusterIntent(documentdb), memoryLimitBytes) +} + +// MergeParametersFor merges all parameter sources in priority order (last write wins): // 1. StaticDefaults // 2. ComputeMemoryAwareDefaults -// 3. User overrides (documentdb.Spec.Postgres.Parameters) -// 4. ProtectedParameters (always wins) -func MergeParameters(documentdb *dbpreview.DocumentDB, memoryLimitBytes int64) map[string]string { +// 3. User overrides (intent.Postgres.Parameters) +// 4. ProtectedParametersFor (always wins) +func MergeParametersFor(intent product.ClusterIntent, memoryLimitBytes int64) map[string]string { result := make(map[string]string) for k, v := range StaticDefaults() { @@ -110,12 +125,10 @@ func MergeParameters(documentdb *dbpreview.DocumentDB, memoryLimitBytes int64) m for k, v := range ComputeMemoryAwareDefaults(memoryLimitBytes) { result[k] = v } - if documentdb.Spec.Postgres != nil { - for k, v := range documentdb.Spec.Postgres.Parameters { - result[k] = v - } + for k, v := range intent.Postgres.Parameters { + result[k] = v } - for k, v := range ProtectedParameters(documentdb) { + for k, v := range ProtectedParametersFor(intent) { result[k] = v } diff --git a/operator/src/internal/controller/documentdb_controller.go b/operator/src/internal/controller/documentdb_controller.go index 9dd8ed85..235d82b9 100644 --- a/operator/src/internal/controller/documentdb_controller.go +++ b/operator/src/internal/controller/documentdb_controller.go @@ -155,7 +155,7 @@ func (r *DocumentDBReconciler) Reconcile(ctx context.Context, req ctrl.Request) intent := product.DocumentDBAdapter{}.ToClusterIntent(documentdb) currentCnpgCluster := &cnpgv1.Cluster{} - desiredCnpgCluster := cnpg.GetCnpgClusterSpecFromIntent(req, documentdb, intent, documentdb.Name, replicationContext.StorageClass, replicationContext.IsPrimary(), logger) + desiredCnpgCluster := cnpg.GetCnpgClusterSpecFromIntent(req, intent, documentdb.Name, replicationContext.StorageClass, replicationContext.IsPrimary(), logger) if replicationContext.IsReplicating() { err = r.AddClusterReplicationToClusterSpec(ctx, documentdb, replicationContext, desiredCnpgCluster) diff --git a/operator/src/internal/product/documentdb.go b/operator/src/internal/product/documentdb.go index a1cdc923..fe5f5d1b 100644 --- a/operator/src/internal/product/documentdb.go +++ b/operator/src/internal/product/documentdb.go @@ -22,9 +22,34 @@ func DocumentDBProfile() ProductProfile { DefaultCredentialSecret: util.DEFAULT_DOCUMENTDB_CREDENTIALS_SECRET, SidecarInjectorPlugin: util.DEFAULT_SIDECAR_INJECTOR_PLUGIN, WALReplicaPlugin: util.DEFAULT_WAL_REPLICA_PLUGIN, + Postgres: PostgresProfile{ + Extension: ExtensionSpec{ + Name: DocumentDBExtensionName, + DynamicLibraryPath: []string{"lib"}, + ExtensionControlPath: []string{"share"}, + LdLibraryPath: []string{"lib", "system"}, + }, + AdditionalLibraries: []string{"pg_cron", "pg_documentdb_core", "pg_documentdb"}, + BootstrapSQL: []string{ + "CREATE EXTENSION documentdb CASCADE", + "CREATE ROLE documentdb WITH LOGIN PASSWORD 'Admin100'", + "ALTER ROLE documentdb WITH SUPERUSER CREATEDB CREATEROLE REPLICATION BYPASSRLS", + }, + PgHBA: []string{ + "host all all localhost trust", + "hostssl replication streaming_replica all cert", + }, + ProtectedParameters: map[string]string{ + "cron.database_name": "postgres", + }, + ReplicationAdminRole: "documentdb_admin_role", + }, } } +// DocumentDBExtensionName is the PostgreSQL extension backing DocumentDB. +const DocumentDBExtensionName = "documentdb" + // DocumentDBAdapter is the first product adapter. It maps the DocumentDB custom // resource onto the product-neutral model consumed by the reconciler. type DocumentDBAdapter struct{} diff --git a/operator/src/internal/product/profile.go b/operator/src/internal/product/profile.go index 4936c367..44ffc31a 100644 --- a/operator/src/internal/product/profile.go +++ b/operator/src/internal/product/profile.go @@ -35,6 +35,41 @@ type ProductProfile struct { // operator wires onto the rendered Cluster. SidecarInjectorPlugin string WALReplicaPlugin string + + // Postgres is the product's PostgreSQL contract: extension, libraries, + // bootstrap SQL, host-based auth, and operator-forced GUCs. + Postgres PostgresProfile +} + +// ExtensionSpec describes the product's PostgreSQL extension, mounted from the +// extension image through CNPG's ImageVolume extension support. +type ExtensionSpec struct { + // Name is the CNPG extension entry name and the PostgreSQL extension name + // used for CREATE/ALTER EXTENSION and version discovery. + Name string + // DynamicLibraryPath, ExtensionControlPath, and LdLibraryPath are the + // image-relative paths CNPG adds to the PostgreSQL search paths. + DynamicLibraryPath []string + ExtensionControlPath []string + LdLibraryPath []string +} + +// PostgresProfile captures the product-wide PostgreSQL contract. +type PostgresProfile struct { + // Extension is the product extension mounted from the extension image. + Extension ExtensionSpec + // AdditionalLibraries are added to shared_preload_libraries. + AdditionalLibraries []string + // BootstrapSQL runs once at initdb, before any user PostInitSQL. + BootstrapSQL []string + // PgHBA is the host-based authentication configuration. + PgHBA []string + // ProtectedParameters are product GUCs the operator always forces, on top + // of the core protected set. + ProtectedParameters map[string]string + // ReplicationAdminRole, when set, is granted to CNPG's streaming_replica + // user so physical replicas can read product-owned objects. + ReplicationAdminRole string } // Adapter is the product-neutral contract shared by every product integration: From 3cd651962d4e3d0e0769fa281bdbfdf1494d0994 Mon Sep 17 00:00:00 2001 From: Rayhan Hossain Date: Mon, 19 Oct 2026 16:43:48 -0700 Subject: [PATCH 05/27] refactor(cnpg): sync the product extension by name SyncCnpgCluster now identifies the product extension from the desired cluster rather than the hardcoded documentdb name, and sidecar plugin parameter keys are shared constants used by the renderer and the sync. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 33b3cb3c-1b25-4ffb-bff1-ebf0f3d04058 Signed-off-by: Rayhan Hossain --- operator/src/internal/cnpg/cnpg_cluster.go | 16 +++--- operator/src/internal/cnpg/cnpg_sync.go | 46 +++++++++++------ operator/src/internal/cnpg/cnpg_sync_test.go | 52 +++++++++++++++++++- operator/src/internal/utils/constants.go | 10 ++++ 4 files changed, 99 insertions(+), 25 deletions(-) diff --git a/operator/src/internal/cnpg/cnpg_cluster.go b/operator/src/internal/cnpg/cnpg_cluster.go index 4269fbac..676cc32f 100644 --- a/operator/src/internal/cnpg/cnpg_cluster.go +++ b/operator/src/internal/cnpg/cnpg_cluster.go @@ -91,11 +91,11 @@ func GetCnpgClusterSpecFromIntent(req ctrl.Request, intent product.ClusterIntent InheritedMetadata: getInheritedMetadataLabels(intent.Identity.Name), Plugins: func() []cnpgv1.PluginConfiguration { params := map[string]string{ - "gatewayImage": gatewayImage, - "documentDbCredentialSecret": credentialSecretName, + util.PLUGIN_PARAM_GATEWAY_IMAGE: gatewayImage, + util.PLUGIN_PARAM_CREDENTIAL_SECRET: credentialSecretName, } if pullPolicy := intent.Images.GatewayPullPolicy; pullPolicy != "" { - params["gatewayImagePullPolicy"] = pullPolicy + params[util.PLUGIN_PARAM_GATEWAY_IMAGE_PULL_POLICY] = pullPolicy } addPluginParamIfSet(params, util.PLUGIN_PARAM_GATEWAY_MEMORY_REQUEST, split.Gateway.MemoryRequest) addPluginParamIfSet(params, util.PLUGIN_PARAM_GATEWAY_MEMORY_LIMIT, split.Gateway.MemoryLimit) @@ -103,26 +103,26 @@ func GetCnpgClusterSpecFromIntent(req ctrl.Request, intent product.ClusterIntent addPluginParamIfSet(params, util.PLUGIN_PARAM_GATEWAY_CPU_LIMIT, split.Gateway.CPULimit) // If TLS is ready, surface secret name to plugin so it can mount certs. if intent.TLS.GatewaySecret != "" { - params["gatewayTLSSecret"] = intent.TLS.GatewaySecret + params[util.PLUGIN_PARAM_GATEWAY_TLS_SECRET] = intent.TLS.GatewaySecret } // Pass monitoring parameters to plugin for OTel sidecar injection. // Sidecar is only injected when monitoring is enabled. // Config hash triggers operator-initiated rolling restart on config changes. if split.MonitoringEnabled { - params["otelCollectorImage"] = util.OtelCollectorImage() - params["otelConfigMapName"] = otelcfg.ConfigMapName(intent.Identity.Name) + params[util.PLUGIN_PARAM_OTEL_COLLECTOR_IMAGE] = util.OtelCollectorImage() + params[util.PLUGIN_PARAM_OTEL_CONFIG_MAP_NAME] = otelcfg.ConfigMapName(intent.Identity.Name) addPluginParamIfSet(params, util.PLUGIN_PARAM_OTEL_MEMORY_REQUEST, split.OTel.MemoryRequest) addPluginParamIfSet(params, util.PLUGIN_PARAM_OTEL_MEMORY_LIMIT, split.OTel.MemoryLimit) addPluginParamIfSet(params, util.PLUGIN_PARAM_OTEL_CPU_REQUEST, split.OTel.CPURequest) addPluginParamIfSet(params, util.PLUGIN_PARAM_OTEL_CPU_LIMIT, split.OTel.CPULimit) if promPort := otelcfg.ResolvePrometheusPort(intent.Monitoring); promPort > 0 { - params["prometheusPort"] = fmt.Sprintf("%d", promPort) + params[util.PLUGIN_PARAM_PROMETHEUS_PORT] = fmt.Sprintf("%d", promPort) } // Compute config hash for change detection. The operator triggers a // rolling restart (via restart annotation) when plugin parameters // change, ensuring pods pick up new config. if configData, err := otelcfg.GenerateConfigMapData(intent.Identity.Name, req.Namespace, intent.Monitoring); err == nil { - params["otelConfigHash"] = otelcfg.HashConfigMapData(configData) + params[util.PLUGIN_PARAM_OTEL_CONFIG_HASH] = otelcfg.HashConfigMapData(configData) } else { log.Error(err, "Failed to generate OTel config hash; config changes may not trigger rolling restart") } diff --git a/operator/src/internal/cnpg/cnpg_sync.go b/operator/src/internal/cnpg/cnpg_sync.go index b971db69..3bd36865 100644 --- a/operator/src/internal/cnpg/cnpg_sync.go +++ b/operator/src/internal/cnpg/cnpg_sync.go @@ -43,12 +43,14 @@ func SyncCnpgCluster( extensionUpdated := false gatewayUpdated := false - // Extension image - currentExtIndex, currentExtImage := findExtensionImage(current) - _, desiredExtImage := findExtensionImage(desired) + // Extension image. The product extension is identified by name from the + // desired spec, so the sync works for any product's extension. + extensionName := productExtensionName(desired) + currentExtIndex, currentExtImage := findExtensionImage(current, extensionName) + _, desiredExtImage := findExtensionImage(desired, extensionName) if currentExtImage != desiredExtImage { if currentExtIndex == -1 { - return fmt.Errorf("documentdb extension not found in current CNPG cluster spec") + return fmt.Errorf("%s extension not found in current CNPG cluster spec", extensionName) } patchOps = append(patchOps, JSONPatch{ Op: PatchOpReplace, @@ -65,8 +67,8 @@ func SyncCnpgCluster( pluginIdx, currentPlugin := findPlugin(current, desiredPlugin.Name) if pluginIdx != -1 { // Gateway image - desiredGwImage := getParam(desiredPlugin.Parameters, "gatewayImage") - currentGwImage := getParam(currentPlugin.Parameters, "gatewayImage") + desiredGwImage := getParam(desiredPlugin.Parameters, util.PLUGIN_PARAM_GATEWAY_IMAGE) + currentGwImage := getParam(currentPlugin.Parameters, util.PLUGIN_PARAM_GATEWAY_IMAGE) if desiredGwImage != "" && currentGwImage != desiredGwImage { patchOps = append(patchOps, JSONPatch{ Op: PatchOpReplace, @@ -87,12 +89,12 @@ func SyncCnpgCluster( } // TLS secret (only synced when BuildDesiredCnpgCluster sets it, i.e. TLS is ready) - desiredTLS := getParam(desiredPlugin.Parameters, "gatewayTLSSecret") - currentTLS := getParam(currentPlugin.Parameters, "gatewayTLSSecret") + desiredTLS := getParam(desiredPlugin.Parameters, util.PLUGIN_PARAM_GATEWAY_TLS_SECRET) + currentTLS := getParam(currentPlugin.Parameters, util.PLUGIN_PARAM_GATEWAY_TLS_SECRET) if desiredTLS != "" && currentTLS != desiredTLS { patchOps = append(patchOps, JSONPatch{ Op: PatchOpReplace, - Path: fmt.Sprintf("/spec/plugins/%d/parameters/gatewayTLSSecret", pluginIdx), + Path: fmt.Sprintf(PatchPathPluginParamFmt, pluginIdx, util.PLUGIN_PARAM_GATEWAY_TLS_SECRET), Value: desiredTLS, }) pluginParamsChanged = true @@ -112,10 +114,10 @@ func SyncCnpgCluster( util.PLUGIN_PARAM_GATEWAY_MEMORY_LIMIT, util.PLUGIN_PARAM_GATEWAY_CPU_REQUEST, util.PLUGIN_PARAM_GATEWAY_CPU_LIMIT, - "otelCollectorImage", - "otelConfigMapName", - "prometheusPort", - "otelConfigHash", + util.PLUGIN_PARAM_OTEL_COLLECTOR_IMAGE, + util.PLUGIN_PARAM_OTEL_CONFIG_MAP_NAME, + util.PLUGIN_PARAM_PROMETHEUS_PORT, + util.PLUGIN_PARAM_OTEL_CONFIG_HASH, util.PLUGIN_PARAM_OTEL_MEMORY_REQUEST, util.PLUGIN_PARAM_OTEL_MEMORY_LIMIT, util.PLUGIN_PARAM_OTEL_CPU_REQUEST, @@ -313,10 +315,22 @@ func SyncCnpgCluster( return nil } -// findExtensionImage returns the index and image reference for the documentdb extension. -func findExtensionImage(cluster *cnpgv1.Cluster) (int, string) { +// productExtensionName returns the name of the product extension the renderer +// declares on the desired cluster (always the first Extensions entry), or "". +func productExtensionName(desired *cnpgv1.Cluster) string { + if desired == nil || len(desired.Spec.PostgresConfiguration.Extensions) == 0 { + return "" + } + return desired.Spec.PostgresConfiguration.Extensions[0].Name +} + +// findExtensionImage returns the index and image reference for the named extension. +func findExtensionImage(cluster *cnpgv1.Cluster, name string) (int, string) { + if name == "" { + return -1, "" + } for i, ext := range cluster.Spec.PostgresConfiguration.Extensions { - if ext.Name == "documentdb" { + if ext.Name == name { return i, ext.ImageVolumeSource.Reference } } diff --git a/operator/src/internal/cnpg/cnpg_sync_test.go b/operator/src/internal/cnpg/cnpg_sync_test.go index 42f3b276..5f76e81e 100644 --- a/operator/src/internal/cnpg/cnpg_sync_test.go +++ b/operator/src/internal/cnpg/cnpg_sync_test.go @@ -786,6 +786,38 @@ var _ = Describe("SyncCnpgCluster - managed roles", func() { Expect(updated.Annotations).ToNot(HaveKey("kubectl.kubernetes.io/restartedAt")) }) + It("patches the image of a non-documentdb product extension", func() { + current := baseCluster("test-cluster", namespace) + current.Spec.PostgresConfiguration.Extensions = []cnpgv1.ExtensionConfiguration{ + {Name: "pg_cron"}, + {Name: "pgcosmos", ImageVolumeSource: corev1.ImageVolumeSource{Reference: "reg/pgcosmos:1"}}, + } + desired := current.DeepCopy() + desired.Spec.PostgresConfiguration.Extensions = []cnpgv1.ExtensionConfiguration{ + {Name: "pgcosmos", ImageVolumeSource: corev1.ImageVolumeSource{Reference: "reg/pgcosmos:2"}}, + } + desired.Spec.Managed = &cnpgv1.ManagedConfiguration{Roles: []cnpgv1.RoleConfiguration{absentOtelMonitorRole()}} + current.Spec.Managed = desired.Spec.Managed.DeepCopy() + c := buildFakeClient(current).Build() + + Expect(SyncCnpgCluster(context.Background(), c, current, desired, nil)).To(Succeed()) + + updated := &cnpgv1.Cluster{} + Expect(c.Get(context.Background(), types.NamespacedName{Name: "test-cluster", Namespace: namespace}, updated)).To(Succeed()) + Expect(updated.Spec.PostgresConfiguration.Extensions[0].Name).To(Equal("pg_cron")) + Expect(updated.Spec.PostgresConfiguration.Extensions[1].ImageVolumeSource.Reference).To(Equal("reg/pgcosmos:2")) + }) + + It("fails when the desired product extension is missing from the current cluster", func() { + current := baseCluster("test-cluster", namespace) + desired := current.DeepCopy() + desired.Spec.PostgresConfiguration.Extensions[0].Name = "pgcosmos" + c := buildFakeClient(current).Build() + + err := SyncCnpgCluster(context.Background(), c, current, desired, nil) + Expect(err).To(MatchError(ContainSubstring("pgcosmos extension not found"))) + }) + It("defaults a missing desired monitor role to absent", func() { current := baseCluster("test-cluster", namespace) desired := current.DeepCopy() @@ -803,11 +835,29 @@ var _ = Describe("Helper functions", func() { PostgresConfiguration: cnpgv1.PostgresConfiguration{}, }, } - idx, img := findExtensionImage(cluster) + idx, img := findExtensionImage(cluster, "documentdb") Expect(idx).To(Equal(-1)) Expect(img).To(BeEmpty()) }) + It("findExtensionImage matches the requested extension name", func() { + cluster := &cnpgv1.Cluster{ + Spec: cnpgv1.ClusterSpec{ + PostgresConfiguration: cnpgv1.PostgresConfiguration{ + Extensions: []cnpgv1.ExtensionConfiguration{ + {Name: "other", ImageVolumeSource: corev1.ImageVolumeSource{Reference: "other:1"}}, + {Name: "pgcosmos", ImageVolumeSource: corev1.ImageVolumeSource{Reference: "pgcosmos:1"}}, + }, + }, + }, + } + idx, img := findExtensionImage(cluster, "pgcosmos") + Expect(idx).To(Equal(1)) + Expect(img).To(Equal("pgcosmos:1")) + idx, _ = findExtensionImage(cluster, "") + Expect(idx).To(Equal(-1)) + }) + It("findPlugin returns -1 when plugin not found", func() { cluster := &cnpgv1.Cluster{ Spec: cnpgv1.ClusterSpec{ diff --git a/operator/src/internal/utils/constants.go b/operator/src/internal/utils/constants.go index 2f8fb437..0df0a984 100644 --- a/operator/src/internal/utils/constants.go +++ b/operator/src/internal/utils/constants.go @@ -106,6 +106,16 @@ const ( // The operator passes the resolved per-container requests/limits to the // sidecar-injector plugin via these CNPG plugin parameters; the plugin sets // them on the gateway and otel-collector container Resources. + PLUGIN_PARAM_GATEWAY_IMAGE = "gatewayImage" + PLUGIN_PARAM_GATEWAY_IMAGE_PULL_POLICY = "gatewayImagePullPolicy" + PLUGIN_PARAM_GATEWAY_TLS_SECRET = "gatewayTLSSecret" + // PLUGIN_PARAM_CREDENTIAL_SECRET keeps its historical key: it is a wire + // contract with the sidecar injector and existing clusters. + PLUGIN_PARAM_CREDENTIAL_SECRET = "documentDbCredentialSecret" + PLUGIN_PARAM_OTEL_COLLECTOR_IMAGE = "otelCollectorImage" + PLUGIN_PARAM_OTEL_CONFIG_MAP_NAME = "otelConfigMapName" + PLUGIN_PARAM_OTEL_CONFIG_HASH = "otelConfigHash" + PLUGIN_PARAM_PROMETHEUS_PORT = "prometheusPort" PLUGIN_PARAM_GATEWAY_MEMORY_REQUEST = "gatewayMemoryRequest" PLUGIN_PARAM_GATEWAY_MEMORY_LIMIT = "gatewayMemoryLimit" PLUGIN_PARAM_GATEWAY_CPU_REQUEST = "gatewayCpuRequest" From 1709693a943db7f5250df79435647d4a13f2f900 Mon Sep 17 00:00:00 2001 From: Rayhan Hossain Date: Mon, 19 Oct 2026 16:44:47 -0700 Subject: [PATCH 06/27] refactor(controller): drive extension lifecycle SQL from the product profile Adds product-neutral helpers for the extension version query, ALTER EXTENSION, the streaming_replica role grant and current image lookups. The DocumentDB reconciler now takes the extension name and replication admin role from its product profile; the generated SQL is unchanged. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 33b3cb3c-1b25-4ffb-bff1-ebf0f3d04058 Signed-off-by: Rayhan Hossain --- operator/src/internal/cnpg/extension_sql.go | 61 +++++++++++++++++++ .../src/internal/cnpg/extension_sql_test.go | 49 +++++++++++++++ .../controller/documentdb_controller.go | 59 ++++++++---------- 3 files changed, 136 insertions(+), 33 deletions(-) create mode 100644 operator/src/internal/cnpg/extension_sql.go create mode 100644 operator/src/internal/cnpg/extension_sql_test.go diff --git a/operator/src/internal/cnpg/extension_sql.go b/operator/src/internal/cnpg/extension_sql.go new file mode 100644 index 00000000..a87e924f --- /dev/null +++ b/operator/src/internal/cnpg/extension_sql.go @@ -0,0 +1,61 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package cnpg + +import ( + "fmt" + + cnpgv1 "github.com/cloudnative-pg/cloudnative-pg/api/v1" + + util "github.com/documentdb/documentdb-operator/internal/utils" +) + +// The helpers below build the SQL and read the cluster state the extension +// lifecycle needs for any product. Extension and role names come from the +// product profile (operator constants, never user input), so they are +// interpolated as bare identifiers. + +// ExtensionVersionSQL returns the query reporting the default (binary) and +// installed versions of the named extension. +func ExtensionVersionSQL(extension string) string { + return fmt.Sprintf("SELECT default_version, installed_version FROM pg_available_extensions WHERE name = '%s'", extension) +} + +// ExtensionUpdateSQL returns the ALTER EXTENSION statement for the named +// extension. An empty targetVersion updates to the binary's default version. +// targetVersion must already be validated (digits, dots, and hyphens only). +func ExtensionUpdateSQL(extension, targetVersion string) string { + if targetVersion == "" { + return fmt.Sprintf("ALTER EXTENSION %s UPDATE", extension) + } + return fmt.Sprintf("ALTER EXTENSION %s UPDATE TO '%s'", extension, targetVersion) +} + +// ReplicationRoleCheckSQL returns the query that yields one row when CNPG's +// streaming_replica user already holds the given role. +func ReplicationRoleCheckSQL(role string) string { + return fmt.Sprintf("SELECT 1 FROM pg_roles WHERE rolname = 'streaming_replica' AND pg_has_role('streaming_replica', '%s', 'USAGE');", role) +} + +// ReplicationRoleGrantSQL returns the statement granting the given role to +// CNPG's streaming_replica user. +func ReplicationRoleGrantSQL(role string) string { + return fmt.Sprintf("GRANT %s TO streaming_replica;", role) +} + +// ExtensionImage returns the image currently declared for the named extension +// on the cluster, or "" when absent. +func ExtensionImage(cluster *cnpgv1.Cluster, extension string) string { + _, image := findExtensionImage(cluster, extension) + return image +} + +// GatewayImage returns the gateway image currently passed to the sidecar +// injector plugin (the first plugin), or "" when absent. +func GatewayImage(cluster *cnpgv1.Cluster) string { + if len(cluster.Spec.Plugins) == 0 { + return "" + } + return getParam(cluster.Spec.Plugins[0].Parameters, util.PLUGIN_PARAM_GATEWAY_IMAGE) +} diff --git a/operator/src/internal/cnpg/extension_sql_test.go b/operator/src/internal/cnpg/extension_sql_test.go new file mode 100644 index 00000000..fa5da7ef --- /dev/null +++ b/operator/src/internal/cnpg/extension_sql_test.go @@ -0,0 +1,49 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package cnpg + +import ( + "testing" + + cnpgv1 "github.com/cloudnative-pg/cloudnative-pg/api/v1" + corev1 "k8s.io/api/core/v1" + + util "github.com/documentdb/documentdb-operator/internal/utils" +) + +func TestExtensionLifecycleSQLUsesProductNames(t *testing.T) { + cases := map[string]string{ + ExtensionVersionSQL("pgcosmos"): "SELECT default_version, installed_version FROM pg_available_extensions WHERE name = 'pgcosmos'", + ExtensionUpdateSQL("pgcosmos", ""): "ALTER EXTENSION pgcosmos UPDATE", + ExtensionUpdateSQL("pgcosmos", "1.2-0"): "ALTER EXTENSION pgcosmos UPDATE TO '1.2-0'", + ReplicationRoleCheckSQL("cosmos_admin"): "SELECT 1 FROM pg_roles WHERE rolname = 'streaming_replica' AND pg_has_role('streaming_replica', 'cosmos_admin', 'USAGE');", + ReplicationRoleGrantSQL("cosmos_admin"): "GRANT cosmos_admin TO streaming_replica;", + } + for got, want := range cases { + if got != want { + t.Errorf("got %q, want %q", got, want) + } + } +} + +func TestClusterImageReaders(t *testing.T) { + cluster := &cnpgv1.Cluster{Spec: cnpgv1.ClusterSpec{ + PostgresConfiguration: cnpgv1.PostgresConfiguration{Extensions: []cnpgv1.ExtensionConfiguration{ + {Name: "pgcosmos", ImageVolumeSource: corev1.ImageVolumeSource{Reference: "reg/pgcosmos:1"}}, + }}, + Plugins: []cnpgv1.PluginConfiguration{{Parameters: map[string]string{util.PLUGIN_PARAM_GATEWAY_IMAGE: "reg/gw:1"}}}, + }} + if got := ExtensionImage(cluster, "pgcosmos"); got != "reg/pgcosmos:1" { + t.Errorf("ExtensionImage = %q", got) + } + if got := ExtensionImage(cluster, "documentdb"); got != "" { + t.Errorf("ExtensionImage(other) = %q, want empty", got) + } + if got := GatewayImage(cluster); got != "reg/gw:1" { + t.Errorf("GatewayImage = %q", got) + } + if got := GatewayImage(&cnpgv1.Cluster{}); got != "" { + t.Errorf("GatewayImage(no plugins) = %q, want empty", got) + } +} diff --git a/operator/src/internal/controller/documentdb_controller.go b/operator/src/internal/controller/documentdb_controller.go index 235d82b9..d305cd52 100644 --- a/operator/src/internal/controller/documentdb_controller.go +++ b/operator/src/internal/controller/documentdb_controller.go @@ -225,20 +225,19 @@ func (r *DocumentDBReconciler) Reconcile(ctx context.Context, req ctrl.Request) } if slices.Contains(currentCnpgCluster.Status.InstancesStatus[cnpgv1.PodHealthy], currentCnpgCluster.Status.CurrentPrimary) && replicationContext.IsPrimary() { - // Check if permissions have already been granted - checkCommand := "SELECT 1 FROM pg_roles WHERE rolname = 'streaming_replica' AND pg_has_role('streaming_replica', 'documentdb_admin_role', 'USAGE');" - output, err := r.SQLExecutor(ctx, currentCnpgCluster, checkCommand) - if err != nil { - logger.Error(err, "Failed to check if permissions already granted") - return ctrl.Result{RequeueAfter: RequeueAfterLong}, nil - } - - if !strings.Contains(output, "(1 row)") { - grantCommand := "GRANT documentdb_admin_role TO streaming_replica;" + if role := intent.Product.Postgres.ReplicationAdminRole; role != "" { + // Check if permissions have already been granted + output, err := r.SQLExecutor(ctx, currentCnpgCluster, cnpg.ReplicationRoleCheckSQL(role)) + if err != nil { + logger.Error(err, "Failed to check if permissions already granted") + return ctrl.Result{RequeueAfter: RequeueAfterLong}, nil + } - if _, err := r.SQLExecutor(ctx, currentCnpgCluster, grantCommand); err != nil { - logger.Error(err, "Failed to grant permissions to streaming_replica") - return ctrl.Result{RequeueAfter: RequeueAfterShort}, nil + if !strings.Contains(output, "(1 row)") { + if _, err := r.SQLExecutor(ctx, currentCnpgCluster, cnpg.ReplicationRoleGrantSQL(role)); err != nil { + logger.Error(err, "Failed to grant permissions to streaming_replica") + return ctrl.Result{RequeueAfter: RequeueAfterShort}, nil + } } } } @@ -801,10 +800,10 @@ func (r *DocumentDBReconciler) handleExtensionUpgrade(ctx context.Context, curre } // Check if ALTER EXTENSION UPDATE is needed - checkVersionSQL := "SELECT default_version, installed_version FROM pg_available_extensions WHERE name = 'documentdb'" - output, err := r.SQLExecutor(ctx, currentCluster, checkVersionSQL) + extension := documentDBExtensionName() + output, err := r.SQLExecutor(ctx, currentCluster, cnpg.ExtensionVersionSQL(extension)) if err != nil { - return fmt.Errorf("failed to check documentdb extension versions: %w", err) + return fmt.Errorf("failed to check %s extension versions: %w", extension, err) } defaultVersion, installedVersion, ok := parseExtensionVersionsFromOutput(output) @@ -878,7 +877,7 @@ func (r *DocumentDBReconciler) handleExtensionUpgrade(ctx context.Context, curre "toVersion", schemaTarget) if _, err := r.SQLExecutor(ctx, currentCluster, updateSQL); err != nil { - return fmt.Errorf("failed to run ALTER EXTENSION documentdb UPDATE: %w", err) + return fmt.Errorf("failed to run ALTER EXTENSION %s UPDATE: %w", extension, err) } logger.Info("Successfully upgraded DocumentDB extension", @@ -935,7 +934,7 @@ func (r *DocumentDBReconciler) determineSchemaTarget( case specSchemaVersion == "auto": // Auto-finalize: update schema to match binary version - return binaryVersion, "ALTER EXTENSION documentdb UPDATE" + return binaryVersion, cnpg.ExtensionUpdateSQL(documentDBExtensionName(), "") default: // Explicit version: update to that specific version. @@ -977,7 +976,7 @@ func (r *DocumentDBReconciler) determineSchemaTarget( // which produces a "Major.Minor-Patch" string (e.g., "0.112-0"). The input is validated // by the CRD schema regex pattern and the validating webhook's version parsing, so it // contains only digits, dots, and hyphens — no SQL injection risk. - return targetPgVersion, fmt.Sprintf("ALTER EXTENSION documentdb UPDATE TO '%s'", targetPgVersion) + return targetPgVersion, cnpg.ExtensionUpdateSQL(documentDBExtensionName(), targetPgVersion) } } @@ -985,20 +984,8 @@ func (r *DocumentDBReconciler) determineSchemaTarget( // and persists them into the DocumentDB status fields. This is a no-op if both fields // are already up to date. func (r *DocumentDBReconciler) updateImageStatus(ctx context.Context, documentdb *dbpreview.DocumentDB, cluster *cnpgv1.Cluster) error { - // Read current extension image - currentExtImage := "" - for _, ext := range cluster.Spec.PostgresConfiguration.Extensions { - if ext.Name == "documentdb" { - currentExtImage = ext.ImageVolumeSource.Reference - break - } - } - - // Read current gateway image - currentGwImage := "" - if len(cluster.Spec.Plugins) > 0 && cluster.Spec.Plugins[0].Parameters != nil { - currentGwImage = cluster.Spec.Plugins[0].Parameters["gatewayImage"] - } + currentExtImage := cnpg.ExtensionImage(cluster, documentDBExtensionName()) + currentGwImage := cnpg.GatewayImage(cluster) // Only update if something changed if documentdb.Status.DocumentDBImage == currentExtImage && documentdb.Status.GatewayImage == currentGwImage { @@ -1067,3 +1054,9 @@ func (r *DocumentDBReconciler) deleteOtelConfigMap(ctx context.Context, clusterN logger.Info("OTel ConfigMap deleted", "name", cmName) return nil } + +// documentDBExtensionName is the PostgreSQL extension whose lifecycle this +// reconciler manages, taken from the DocumentDB product profile. +func documentDBExtensionName() string { + return product.DocumentDBProfile().Postgres.Extension.Name +} From 5a89551837d7990a66f7df4ef596f766265195fc Mon Sep 17 00:00:00 2001 From: Rayhan Hossain Date: Mon, 19 Oct 2026 16:53:30 -0700 Subject: [PATCH 07/27] feat(sidecar-injector): make the gateway container contract configurable Adds optional gatewayContainerName, gatewayPort, gatewayArgs (JSON), gatewayUsernameEnv and gatewayPasswordEnv plugin parameters. They default to the DocumentDB gateway and are only serialized when they differ, so existing clusters see no change. The operator forwards them from ProductProfile.Gateway, which is empty for DocumentDB. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 33b3cb3c-1b25-4ffb-bff1-ebf0f3d04058 Signed-off-by: Rayhan Hossain --- .../internal/config/config.go | 88 ++++++++++++++++++ .../internal/config/config_test.go | 57 ++++++++++++ .../internal/lifecycle/lifecycle.go | 25 +++-- .../internal/lifecycle/lifecycle_test.go | 91 +++++++++++++++++-- operator/src/internal/cnpg/cnpg_cluster.go | 20 ++++ .../internal/cnpg/gateway_contract_test.go | 55 +++++++++++ operator/src/internal/product/profile.go | 20 ++++ operator/src/internal/utils/constants.go | 6 ++ 8 files changed, 346 insertions(+), 16 deletions(-) create mode 100644 operator/src/internal/cnpg/gateway_contract_test.go diff --git a/operator/cnpg-plugins/sidecar-injector/internal/config/config.go b/operator/cnpg-plugins/sidecar-injector/internal/config/config.go index 0083a6ed..a4720dd2 100644 --- a/operator/cnpg-plugins/sidecar-injector/internal/config/config.go +++ b/operator/cnpg-plugins/sidecar-injector/internal/config/config.go @@ -33,6 +33,23 @@ const ( otelCPURequestParameter = "otelCpuRequest" otelCPULimitParameter = "otelCpuLimit" prometheusPortParameter = "prometheusPort" + + // Gateway contract parameters. They are optional and default to the + // DocumentDB gateway so other products can reuse this injector with a + // different gateway binary. + gatewayContainerNameParameter = "gatewayContainerName" + gatewayPortParameter = "gatewayPort" + gatewayArgsParameter = "gatewayArgs" + gatewayUsernameEnvParameter = "gatewayUsernameEnv" + gatewayPasswordEnvParameter = "gatewayPasswordEnv" +) + +// Defaults for the gateway contract (the DocumentDB gateway). +const ( + DefaultGatewayContainerName = "documentdb-gateway" + DefaultGatewayPort = int32(10260) + DefaultGatewayUsernameEnv = "USERNAME" + DefaultGatewayPasswordEnv = "PASSWORD" ) // Configuration represents the plugin configuration parameters @@ -54,6 +71,16 @@ type Configuration struct { OTelCPURequest string OTelCPULimit string PrometheusPort int32 + // GatewayContainerName and GatewayPort name and expose the gateway sidecar. + GatewayContainerName string + GatewayPort int32 + // GatewayArgs, when non-nil, replaces the DocumentDB gateway arguments + // (--create-user/--start-pg/--pg-port and the TLS file flags) verbatim. + GatewayArgs []string + // GatewayUsernameEnv and GatewayPasswordEnv are the env var names the + // credential secret's username/password keys are exposed as. + GatewayUsernameEnv string + GatewayPasswordEnv string } // FromParameters builds a plugin configuration from the configuration parameters @@ -99,6 +126,29 @@ func FromParameters( otelCPULimitParameter, ) + var gatewayArgs []string + if raw := helper.Parameters[gatewayArgsParameter]; raw != "" { + if err := json.Unmarshal([]byte(raw), &gatewayArgs); err != nil { + validationErrors = append( + validationErrors, + validation.BuildErrorForParameter(helper, gatewayArgsParameter, "must be a JSON array of strings: "+err.Error()), + ) + } + } + + var gatewayPort int32 + if portStr := helper.Parameters[gatewayPortParameter]; portStr != "" { + p, err := strconv.ParseInt(portStr, 10, 32) + if err != nil || p <= 0 || p > 65535 { + validationErrors = append( + validationErrors, + validation.BuildErrorForParameter(helper, gatewayPortParameter, "invalid port number: "+portStr), + ) + } else { + gatewayPort = int32(p) + } + } + var prometheusPort int32 if portStr := helper.Parameters[prometheusPortParameter]; portStr != "" { p, err := strconv.ParseInt(portStr, 10, 32) @@ -161,6 +211,11 @@ func FromParameters( OTelCPURequest: helper.Parameters[otelCPURequestParameter], OTelCPULimit: helper.Parameters[otelCPULimitParameter], PrometheusPort: prometheusPort, + GatewayContainerName: helper.Parameters[gatewayContainerNameParameter], + GatewayPort: gatewayPort, + GatewayArgs: gatewayArgs, + GatewayUsernameEnv: helper.Parameters[gatewayUsernameEnvParameter], + GatewayPasswordEnv: helper.Parameters[gatewayPasswordEnvParameter], } configuration.applyDefaults() @@ -228,6 +283,18 @@ func (config *Configuration) applyDefaults() { if config.DocumentDbCredentialSecret == "" { config.DocumentDbCredentialSecret = "documentdb-credentials" } + if config.GatewayContainerName == "" { + config.GatewayContainerName = DefaultGatewayContainerName + } + if config.GatewayPort == 0 { + config.GatewayPort = DefaultGatewayPort + } + if config.GatewayUsernameEnv == "" { + config.GatewayUsernameEnv = DefaultGatewayUsernameEnv + } + if config.GatewayPasswordEnv == "" { + config.GatewayPasswordEnv = DefaultGatewayPasswordEnv + } } // parsePullPolicy converts a string to a corev1.PullPolicy. @@ -277,6 +344,27 @@ func (config *Configuration) ToParameters() (map[string]string, error) { if config.PrometheusPort > 0 { result[prometheusPortParameter] = strconv.FormatInt(int64(config.PrometheusPort), 10) } + // Gateway contract parameters are only serialized when they differ from + // the DocumentDB defaults, keeping existing parameter maps unchanged. + if config.GatewayContainerName != DefaultGatewayContainerName { + setIfNotEmpty(gatewayContainerNameParameter, config.GatewayContainerName) + } + if config.GatewayPort > 0 && config.GatewayPort != DefaultGatewayPort { + result[gatewayPortParameter] = strconv.FormatInt(int64(config.GatewayPort), 10) + } + if config.GatewayArgs != nil { + serializedArgs, err := json.Marshal(config.GatewayArgs) + if err != nil { + return nil, err + } + result[gatewayArgsParameter] = string(serializedArgs) + } + if config.GatewayUsernameEnv != DefaultGatewayUsernameEnv { + setIfNotEmpty(gatewayUsernameEnvParameter, config.GatewayUsernameEnv) + } + if config.GatewayPasswordEnv != DefaultGatewayPasswordEnv { + setIfNotEmpty(gatewayPasswordEnvParameter, config.GatewayPasswordEnv) + } return result, nil } diff --git a/operator/cnpg-plugins/sidecar-injector/internal/config/config_test.go b/operator/cnpg-plugins/sidecar-injector/internal/config/config_test.go index 42228e56..3763aed7 100644 --- a/operator/cnpg-plugins/sidecar-injector/internal/config/config_test.go +++ b/operator/cnpg-plugins/sidecar-injector/internal/config/config_test.go @@ -279,3 +279,60 @@ func TestToParametersRoundTrip(t *testing.T) { t.Errorf("round-trip otel cpu request = %q, want %q", restored.OTelCPURequest, original.OTelCPURequest) } } + +func TestGatewayContract(t *testing.T) { + t.Run("defaults to the DocumentDB gateway and is not serialized", func(t *testing.T) { + config, errs := FromParameters(&common.Plugin{Parameters: map[string]string{}}) + if len(errs) != 0 { + t.Fatalf("unexpected validation errors: %v", errs) + } + if config.GatewayContainerName != "documentdb-gateway" || config.GatewayPort != 10260 || + config.GatewayUsernameEnv != "USERNAME" || config.GatewayPasswordEnv != "PASSWORD" || config.GatewayArgs != nil { + t.Errorf("unexpected defaults: %+v", config) + } + params, err := config.ToParameters() + if err != nil { + t.Fatalf("ToParameters() error: %v", err) + } + for _, key := range []string{"gatewayContainerName", "gatewayPort", "gatewayArgs", "gatewayUsernameEnv", "gatewayPasswordEnv"} { + if _, ok := params[key]; ok { + t.Errorf("default contract serialized %q", key) + } + } + }) + + t.Run("custom contract round-trips", func(t *testing.T) { + params := map[string]string{ + "gatewayContainerName": "pgcosmos-gateway", + "gatewayPort": "8081", + "gatewayArgs": `["--listen","0.0.0.0:8081"]`, + "gatewayUsernameEnv": "COSMOS_USER", + "gatewayPasswordEnv": "COSMOS_KEY", + } + config, errs := FromParameters(&common.Plugin{Parameters: params}) + if len(errs) != 0 { + t.Fatalf("unexpected validation errors: %v", errs) + } + out, err := config.ToParameters() + if err != nil { + t.Fatalf("ToParameters() error: %v", err) + } + for k, v := range params { + if out[k] != v { + t.Errorf("%s = %q, want %q", k, out[k], v) + } + } + }) + + t.Run("rejects invalid values", func(t *testing.T) { + for _, params := range []map[string]string{ + {"gatewayPort": "0"}, + {"gatewayPort": "abc"}, + {"gatewayArgs": "--not-json"}, + } { + if _, errs := FromParameters(&common.Plugin{Parameters: params}); len(errs) == 0 { + t.Errorf("expected validation error for %v", params) + } + } + }) +} diff --git a/operator/cnpg-plugins/sidecar-injector/internal/lifecycle/lifecycle.go b/operator/cnpg-plugins/sidecar-injector/internal/lifecycle/lifecycle.go index dfa88a94..edaf7a5f 100644 --- a/operator/cnpg-plugins/sidecar-injector/internal/lifecycle/lifecycle.go +++ b/operator/cnpg-plugins/sidecar-injector/internal/lifecycle/lifecycle.go @@ -134,10 +134,10 @@ func (impl Implementation) reconcileMetadata( // Add USERNAME and PASSWORD environment variables from secret defined in configuration credentialSecretName := configuration.DocumentDbCredentialSecret - log.Printf("Adding USERNAME and PASSWORD environment variables from secret '%s'", credentialSecretName) + log.Printf("Adding %s and %s environment variables from secret '%s'", configuration.GatewayUsernameEnv, configuration.GatewayPasswordEnv, credentialSecretName) envVars = append(envVars, corev1.EnvVar{ - Name: "USERNAME", + Name: configuration.GatewayUsernameEnv, ValueFrom: &corev1.EnvVarSource{ SecretKeyRef: &corev1.SecretKeySelector{ LocalObjectReference: corev1.LocalObjectReference{ @@ -148,7 +148,7 @@ func (impl Implementation) reconcileMetadata( }, }, corev1.EnvVar{ - Name: "PASSWORD", + Name: configuration.GatewayPasswordEnv, ValueFrom: &corev1.EnvVarSource{ SecretKeyRef: &corev1.SecretKeySelector{ LocalObjectReference: corev1.LocalObjectReference{ @@ -162,12 +162,12 @@ func (impl Implementation) reconcileMetadata( // Initialize the sidecar container with configurable gateway image sidecar := &corev1.Container{ - Name: "documentdb-gateway", + Name: configuration.GatewayContainerName, Image: configuration.GatewayImage, ImagePullPolicy: configuration.GatewayImagePullPolicy, Ports: []corev1.ContainerPort{ { - ContainerPort: 10260, + ContainerPort: configuration.GatewayPort, }, }, Env: envVars, @@ -230,6 +230,11 @@ func (impl Implementation) reconcileMetadata( // Pass cert and key via CLI args to align with emulator_entrypoint.sh interface args = append(args, "--cert-path", "/tls/tls.crt", "--key-file", "/tls/tls.key") } + if configuration.GatewayArgs != nil { + // A product-specific gateway supplies its own arguments; TLS material is + // still mounted and advertised through the env vars above. + args = configuration.GatewayArgs + } sidecar.Args = args // Inject the sidecar container @@ -317,7 +322,7 @@ func (impl Implementation) reconcileMetadata( // sidecar and so that every signal carries a per-pod service.instance.id // resource attribute. Only set when the sidecar is present to avoid // connection errors and resource-attribute drift. - injectGatewayOTelEnv(mutatedPod) + injectGatewayOTelEnv(mutatedPod, configuration.GatewayContainerName) log.Printf("OTel Collector sidecar injected successfully") } @@ -341,10 +346,10 @@ func (impl Implementation) reconcileMetadata( }, nil } -// gatewayContainerName is the name of the documentdb gateway container that +// gatewayContainerName is the default name of the gateway container that // pushes OTLP metrics and traces to the co-located OTel Collector sidecar. // Kept as a package-level constant so tests can reference it. -const gatewayContainerName = "documentdb-gateway" +const gatewayContainerName = config.DefaultGatewayContainerName func buildResources(cpuReq, cpuLim, memReq, memLim string) corev1.ResourceRequirements { var requests corev1.ResourceList @@ -462,10 +467,10 @@ func gatewayOTelEnvVars() []corev1.EnvVar { // Idempotency keeps pod construction stable if a caller passes an already // mutated pod. Container injection itself runs only for CREATE because // Kubernetes forbids adding or removing containers from an existing pod. -func injectGatewayOTelEnv(pod *corev1.Pod) { +func injectGatewayOTelEnv(pod *corev1.Pod, containerName string) { envs := gatewayOTelEnvVars() for i := range pod.Spec.Containers { - if pod.Spec.Containers[i].Name != gatewayContainerName { + if pod.Spec.Containers[i].Name != containerName { continue } existing := make(map[string]bool, len(pod.Spec.Containers[i].Env)) diff --git a/operator/cnpg-plugins/sidecar-injector/internal/lifecycle/lifecycle_test.go b/operator/cnpg-plugins/sidecar-injector/internal/lifecycle/lifecycle_test.go index 94b4328b..2e3474d6 100644 --- a/operator/cnpg-plugins/sidecar-injector/internal/lifecycle/lifecycle_test.go +++ b/operator/cnpg-plugins/sidecar-injector/internal/lifecycle/lifecycle_test.go @@ -98,7 +98,7 @@ func TestLifecycleHookPatchDoesNotMutatePod(t *testing.T) { // so we don't need POD_NAME / OTEL_RESOURCE_ATTRIBUTES on the gateway. func TestInjectGatewayOTelEnv_AllAppended(t *testing.T) { pod := gatewayContainer() - injectGatewayOTelEnv(pod) + injectGatewayOTelEnv(pod, gatewayContainerName) got := envNames(pod.Spec.Containers[1].Env) want := []string{"OTEL_EXPORTER_OTLP_ENDPOINT", "OTEL_METRICS_ENABLED"} @@ -112,7 +112,7 @@ func TestInjectGatewayOTelEnv_AllAppended(t *testing.T) { // silently disables OTLP metrics export. func TestInjectGatewayOTelEnv_MetricsEnabledPresent(t *testing.T) { pod := gatewayContainer() - injectGatewayOTelEnv(pod) + injectGatewayOTelEnv(pod, gatewayContainerName) for _, e := range pod.Spec.Containers[1].Env { if e.Name == "OTEL_METRICS_ENABLED" { if e.Value != "true" { @@ -129,7 +129,7 @@ func TestInjectGatewayOTelEnv_MetricsEnabledPresent(t *testing.T) { // untouched (we don't overwrite), and the others are still appended. func TestInjectGatewayOTelEnv_PreservesExisting(t *testing.T) { pod := gatewayContainer(corev1.EnvVar{Name: "OTEL_EXPORTER_OTLP_ENDPOINT", Value: "http://custom:4317"}) - injectGatewayOTelEnv(pod) + injectGatewayOTelEnv(pod, gatewayContainerName) env := pod.Spec.Containers[1].Env if len(env) != 2 { @@ -147,10 +147,10 @@ func TestInjectGatewayOTelEnv_PreservesExisting(t *testing.T) { // not duplicate environment variables on an already-mutated pod. func TestInjectGatewayOTelEnv_Idempotent(t *testing.T) { pod := gatewayContainer() - injectGatewayOTelEnv(pod) + injectGatewayOTelEnv(pod, gatewayContainerName) first := append([]corev1.EnvVar(nil), pod.Spec.Containers[1].Env...) - injectGatewayOTelEnv(pod) + injectGatewayOTelEnv(pod, gatewayContainerName) second := pod.Spec.Containers[1].Env if !reflect.DeepEqual(first, second) { @@ -165,7 +165,7 @@ func TestInjectGatewayOTelEnv_NoGatewayContainer(t *testing.T) { Containers: []corev1.Container{{Name: "postgres"}}, }, } - injectGatewayOTelEnv(pod) + injectGatewayOTelEnv(pod, gatewayContainerName) if len(pod.Spec.Containers[0].Env) != 0 { t.Errorf("expected no envs on non-gateway container, got %v", envNames(pod.Spec.Containers[0].Env)) } @@ -435,3 +435,82 @@ func TestNewOtelCollectorSidecar_MonitorUser(t *testing.T) { } t.Fatal("missing PGUSER env var") } + +func runCreateHook(t *testing.T, params map[string]string) map[string]corev1.Container { + t.Helper() + enabled := true + cluster := &apiv1.Cluster{ + TypeMeta: metav1.TypeMeta{APIVersion: apiv1.SchemeGroupVersion.String(), Kind: apiv1.ClusterKind}, + ObjectMeta: metav1.ObjectMeta{Name: "cluster", Namespace: "default"}, + Spec: apiv1.ClusterSpec{Plugins: []apiv1.PluginConfiguration{ + {Name: pluginmetadata.PluginName, Enabled: &enabled, Parameters: params}, + }}, + Status: apiv1.ClusterStatus{TargetPrimary: "cluster-1"}, + } + pod := &corev1.Pod{ + TypeMeta: metav1.TypeMeta{APIVersion: "v1", Kind: "Pod"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster-1", Namespace: "default", + Labels: map[string]string{"cnpg.io/cluster": "cluster"}, + Annotations: map[string]string{"cnpg.io/operatorVersion": "test"}, + }, + Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: "postgres"}}}, + } + response, err := Implementation{}.LifecycleHook(context.Background(), &cnpglifecycle.OperatorLifecycleRequest{ + OperationType: &cnpglifecycle.OperatorOperationType{Type: cnpglifecycle.OperatorOperationType_TYPE_CREATE}, + ClusterDefinition: mustMarshal(t, cluster), + ObjectDefinition: mustMarshal(t, pod), + }) + if err != nil { + t.Fatalf("LifecycleHook() error: %v", err) + } + return addedContainersFromPatch(t, response.JsonPatch) +} + +func TestLifecycleHookDefaultGatewayContract(t *testing.T) { + gateway, ok := runCreateHook(t, map[string]string{"gatewayImage": "gw:1"})[gatewayContainerName] + if !ok { + t.Fatal("documentdb-gateway container missing") + } + if len(gateway.Ports) != 1 || gateway.Ports[0].ContainerPort != 10260 { + t.Errorf("ports = %+v, want 10260", gateway.Ports) + } + if got := envNames(gateway.Env); len(got) < 2 || got[0] != "USERNAME" || got[1] != "PASSWORD" { + t.Errorf("env = %v, want USERNAME/PASSWORD first", got) + } + want := []string{"--create-user", "true", "--start-pg", "false", "--pg-port", "5432"} + if !reflect.DeepEqual(gateway.Args, want) { + t.Errorf("args = %v, want %v", gateway.Args, want) + } +} + +func TestLifecycleHookCustomGatewayContract(t *testing.T) { + containers := runCreateHook(t, map[string]string{ + "gatewayImage": "cosmos-gw:1", + "gatewayContainerName": "pgcosmos-gateway", + "gatewayPort": "8081", + "gatewayArgs": `["--port","8081"]`, + "gatewayUsernameEnv": "COSMOS_USER", + "gatewayPasswordEnv": "COSMOS_KEY", + "otelCollectorImage": "otel:1", + "otelConfigMapName": "otel-config", + }) + if _, ok := containers[gatewayContainerName]; ok { + t.Fatal("default gateway container name leaked") + } + gateway, ok := containers["pgcosmos-gateway"] + if !ok { + t.Fatalf("pgcosmos-gateway missing; got %v", containers) + } + if gateway.Ports[0].ContainerPort != 8081 { + t.Errorf("port = %d, want 8081", gateway.Ports[0].ContainerPort) + } + if !reflect.DeepEqual(gateway.Args, []string{"--port", "8081"}) { + t.Errorf("args = %v", gateway.Args) + } + names := envNames(gateway.Env) + if names[0] != "COSMOS_USER" || names[1] != "COSMOS_KEY" { + t.Errorf("env = %v", names) + } + assertEnvValue(t, gateway.Env, "OTEL_METRICS_ENABLED", "true") +} diff --git a/operator/src/internal/cnpg/cnpg_cluster.go b/operator/src/internal/cnpg/cnpg_cluster.go index 676cc32f..2b1417a2 100644 --- a/operator/src/internal/cnpg/cnpg_cluster.go +++ b/operator/src/internal/cnpg/cnpg_cluster.go @@ -5,8 +5,10 @@ package cnpg import ( "cmp" + "encoding/json" "fmt" "os" + "strconv" cnpgv1 "github.com/cloudnative-pg/cloudnative-pg/api/v1" "github.com/go-logr/logr" @@ -101,6 +103,7 @@ func GetCnpgClusterSpecFromIntent(req ctrl.Request, intent product.ClusterIntent addPluginParamIfSet(params, util.PLUGIN_PARAM_GATEWAY_MEMORY_LIMIT, split.Gateway.MemoryLimit) addPluginParamIfSet(params, util.PLUGIN_PARAM_GATEWAY_CPU_REQUEST, split.Gateway.CPURequest) addPluginParamIfSet(params, util.PLUGIN_PARAM_GATEWAY_CPU_LIMIT, split.Gateway.CPULimit) + addGatewayContractParams(params, intent.Product.Gateway) // If TLS is ready, surface secret name to plugin so it can mount certs. if intent.TLS.GatewaySecret != "" { params[util.PLUGIN_PARAM_GATEWAY_TLS_SECRET] = intent.TLS.GatewaySecret @@ -416,3 +419,20 @@ func buildPostgresConfiguration(intent product.ClusterIntent, extensionImageSour PgHBA: pg.PgHBA, } } + +// addGatewayContractParams forwards a product's non-default gateway contract to +// the sidecar injector. Unset fields are omitted so the injector keeps its +// DocumentDB gateway defaults and existing parameter maps are unchanged. +func addGatewayContractParams(params map[string]string, gw product.GatewayProfile) { + addPluginParamIfSet(params, util.PLUGIN_PARAM_GATEWAY_CONTAINER_NAME, gw.ContainerName) + if gw.Port > 0 { + params[util.PLUGIN_PARAM_GATEWAY_PORT] = strconv.FormatInt(int64(gw.Port), 10) + } + if gw.Args != nil { + // json.Marshal of a []string cannot fail. + args, _ := json.Marshal(gw.Args) + params[util.PLUGIN_PARAM_GATEWAY_ARGS] = string(args) + } + addPluginParamIfSet(params, util.PLUGIN_PARAM_GATEWAY_USERNAME_ENV, gw.UsernameEnv) + addPluginParamIfSet(params, util.PLUGIN_PARAM_GATEWAY_PASSWORD_ENV, gw.PasswordEnv) +} diff --git a/operator/src/internal/cnpg/gateway_contract_test.go b/operator/src/internal/cnpg/gateway_contract_test.go new file mode 100644 index 00000000..201f7f14 --- /dev/null +++ b/operator/src/internal/cnpg/gateway_contract_test.go @@ -0,0 +1,55 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package cnpg + +import ( + "testing" + + "github.com/go-logr/logr" + ctrl "sigs.k8s.io/controller-runtime" + + "github.com/documentdb/documentdb-operator/internal/product" + util "github.com/documentdb/documentdb-operator/internal/utils" +) + +func TestGatewayContractPluginParams(t *testing.T) { + gatewayKeys := []string{ + util.PLUGIN_PARAM_GATEWAY_CONTAINER_NAME, + util.PLUGIN_PARAM_GATEWAY_PORT, + util.PLUGIN_PARAM_GATEWAY_ARGS, + util.PLUGIN_PARAM_GATEWAY_USERNAME_ENV, + util.PLUGIN_PARAM_GATEWAY_PASSWORD_ENV, + } + req := ctrl.Request{} + req.Name, req.Namespace = "golden", "golden-ns" + + intent := product.DocumentDBAdapter{}.ToClusterIntent(goldenBaseDocumentDB()) + params := GetCnpgClusterSpecFromIntent(req, intent, "sa", "", true, logr.Discard()).Spec.Plugins[0].Parameters + for _, key := range gatewayKeys { + if _, ok := params[key]; ok { + t.Errorf("DocumentDB profile emitted %q", key) + } + } + + intent.Product.Gateway = product.GatewayProfile{ + ContainerName: "pgcosmos-gateway", + Port: 8081, + Args: []string{"--port", "8081"}, + UsernameEnv: "COSMOS_USER", + PasswordEnv: "COSMOS_KEY", + } + params = GetCnpgClusterSpecFromIntent(req, intent, "sa", "", true, logr.Discard()).Spec.Plugins[0].Parameters + want := map[string]string{ + util.PLUGIN_PARAM_GATEWAY_CONTAINER_NAME: "pgcosmos-gateway", + util.PLUGIN_PARAM_GATEWAY_PORT: "8081", + util.PLUGIN_PARAM_GATEWAY_ARGS: `["--port","8081"]`, + util.PLUGIN_PARAM_GATEWAY_USERNAME_ENV: "COSMOS_USER", + util.PLUGIN_PARAM_GATEWAY_PASSWORD_ENV: "COSMOS_KEY", + } + for k, v := range want { + if params[k] != v { + t.Errorf("%s = %q, want %q", k, params[k], v) + } + } +} diff --git a/operator/src/internal/product/profile.go b/operator/src/internal/product/profile.go index 44ffc31a..3139c7d4 100644 --- a/operator/src/internal/product/profile.go +++ b/operator/src/internal/product/profile.go @@ -39,6 +39,26 @@ type ProductProfile struct { // Postgres is the product's PostgreSQL contract: extension, libraries, // bootstrap SQL, host-based auth, and operator-forced GUCs. Postgres PostgresProfile + + // Gateway is the product's gateway sidecar contract. Zero values keep the + // sidecar injector's DocumentDB gateway defaults. + Gateway GatewayProfile +} + +// GatewayProfile describes how the sidecar injector runs the product gateway. +// Every field is optional; an empty value is not sent to the plugin so the +// injector falls back to its DocumentDB gateway defaults. +type GatewayProfile struct { + // ContainerName is the injected gateway container name. + ContainerName string + // Port is the gateway container port. + Port int32 + // Args, when non-nil, replaces the DocumentDB gateway arguments verbatim. + Args []string + // UsernameEnv and PasswordEnv are the env var names the credential + // secret's username/password keys are exposed as. + UsernameEnv string + PasswordEnv string } // ExtensionSpec describes the product's PostgreSQL extension, mounted from the diff --git a/operator/src/internal/utils/constants.go b/operator/src/internal/utils/constants.go index 0df0a984..0b59e39a 100644 --- a/operator/src/internal/utils/constants.go +++ b/operator/src/internal/utils/constants.go @@ -124,6 +124,12 @@ const ( PLUGIN_PARAM_OTEL_MEMORY_LIMIT = "otelMemoryLimit" PLUGIN_PARAM_OTEL_CPU_REQUEST = "otelCpuRequest" PLUGIN_PARAM_OTEL_CPU_LIMIT = "otelCpuLimit" + // Optional gateway contract overrides; omitted for the DocumentDB gateway. + PLUGIN_PARAM_GATEWAY_CONTAINER_NAME = "gatewayContainerName" + PLUGIN_PARAM_GATEWAY_PORT = "gatewayPort" + PLUGIN_PARAM_GATEWAY_ARGS = "gatewayArgs" + PLUGIN_PARAM_GATEWAY_USERNAME_ENV = "gatewayUsernameEnv" + PLUGIN_PARAM_GATEWAY_PASSWORD_ENV = "gatewayPasswordEnv" // TODO: remove these constants once change stream support is included in the official images. CHANGESTREAM_DOCUMENTDB_IMAGE_REPOSITORY = "ghcr.io/wentingwu666666/documentdb-kubernetes-operator" From 93cc2d9c482177ba94497722643e9a74dc53d049 Mon Sep 17 00:00:00 2001 From: Rayhan Hossain Date: Mon, 19 Oct 2026 16:54:25 -0700 Subject: [PATCH 08/27] test(cnpg): prove a second product renders without DocumentDB literals Renders a synthetic pgcosmos intent end to end and asserts the CNPG Cluster carries only pgcosmos extension, libraries, bootstrap SQL, pg_hba, protected GUCs, plugin names, gateway contract and owner kind. The only DocumentDB string allowed is the documented documentDbCredentialSecret plugin key. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 33b3cb3c-1b25-4ffb-bff1-ebf0f3d04058 Signed-off-by: Rayhan Hossain --- .../src/internal/cnpg/second_product_test.go | 134 ++++++++++++++++++ 1 file changed, 134 insertions(+) create mode 100644 operator/src/internal/cnpg/second_product_test.go diff --git a/operator/src/internal/cnpg/second_product_test.go b/operator/src/internal/cnpg/second_product_test.go new file mode 100644 index 00000000..a5df8d4f --- /dev/null +++ b/operator/src/internal/cnpg/second_product_test.go @@ -0,0 +1,134 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package cnpg + +import ( + "encoding/json" + "regexp" + "slices" + "testing" + + "github.com/go-logr/logr" + "k8s.io/apimachinery/pkg/util/validation/field" + ctrl "sigs.k8s.io/controller-runtime" + + dbpreview "github.com/documentdb/documentdb-operator/api/preview" + "github.com/documentdb/documentdb-operator/internal/product" + util "github.com/documentdb/documentdb-operator/internal/utils" +) + +// pgcosmosIntent is a synthetic second product used to prove the renderer has +// no DocumentDB-specific knowledge beyond documented wire contracts. +func pgcosmosIntent() product.ClusterIntent { + return product.ClusterIntent{ + Images: product.Images{ + PostgresExtension: "reg/pgcosmos:1", + Gateway: "reg/pgcosmos-gateway:1", + Postgres: "reg/postgres:17", + }, + Topology: product.Topology{Instances: 3}, + Storage: product.Storage{PvcSize: "20Gi"}, + Resources: product.Resources{Memory: "4Gi", CPU: "2"}, + Monitoring: &dbpreview.MonitoringSpec{Enabled: true}, + TLS: product.TLS{GatewaySecret: "acct-tls"}, + Identity: product.Identity{Name: "acct", UID: "uid-1", APIVersion: "cosmos.azure.com/v1", Kind: "CosmosAccount"}, + Postgres: product.Postgres{Parameters: map[string]string{"work_mem": "8MB", "pgcosmos.locked": "user"}}, + CredentialSecret: "cosmos-credentials", + SidecarInjectorPlugin: "sidecar.cosmos.azure.com", + WALReplicaPlugin: "wal.cosmos.azure.com", + Product: product.ProductProfile{ + Name: "Cosmos DB", + Postgres: product.PostgresProfile{ + Extension: product.ExtensionSpec{ + Name: "pgcosmos", + DynamicLibraryPath: []string{"lib"}, + ExtensionControlPath: []string{"share"}, + }, + AdditionalLibraries: []string{"pgcosmos_core"}, + BootstrapSQL: []string{"CREATE EXTENSION pgcosmos CASCADE"}, + PgHBA: []string{"host all all 0.0.0.0/0 scram-sha-256"}, + ProtectedParameters: map[string]string{"pgcosmos.locked": "operator"}, + ReplicationAdminRole: "cosmos_admin", + }, + Gateway: product.GatewayProfile{ContainerName: "pgcosmos-gateway", Port: 8081}, + }, + } +} + +func TestSecondProductRendersWithoutDocumentDBLiterals(t *testing.T) { + req := ctrl.Request{} + req.Name, req.Namespace = "acct", "ns" + intent := pgcosmosIntent() + cluster := GetCnpgClusterSpecFromIntent(req, intent, "acct-sa", "", true, logr.Discard()) + + raw, err := json.Marshal(cluster) + if err != nil { + t.Fatal(err) + } + // documentDbCredentialSecret is the sidecar plugin's parameter key, a + // documented wire contract shared by every product. + scrubbed := regexp.MustCompile(regexp.QuoteMeta(util.PLUGIN_PARAM_CREDENTIAL_SECRET)).ReplaceAll(raw, nil) + if loc := regexp.MustCompile(`(?i)documentdb`).FindIndex(scrubbed); loc != nil { + lo, hi := max(loc[0]-80, 0), min(loc[1]+80, len(scrubbed)) + t.Fatalf("rendered pgcosmos cluster contains a DocumentDB literal: ...%s...", scrubbed[lo:hi]) + } + + pg := cluster.Spec.PostgresConfiguration + if len(pg.Extensions) != 1 || pg.Extensions[0].Name != "pgcosmos" || pg.Extensions[0].ImageVolumeSource.Reference != "reg/pgcosmos:1" { + t.Errorf("extensions = %+v", pg.Extensions) + } + if !slices.Contains(pg.AdditionalLibraries, "pgcosmos_core") { + t.Errorf("additional libraries = %v", pg.AdditionalLibraries) + } + if !slices.Equal(pg.PgHBA, intent.Product.Postgres.PgHBA) { + t.Errorf("pg_hba = %v", pg.PgHBA) + } + if got := pg.Parameters["pgcosmos.locked"]; got != "operator" { + t.Errorf("protected product GUC = %q, want operator", got) + } + if got := pg.Parameters["work_mem"]; got != "8MB" { + t.Errorf("user GUC = %q, want 8MB", got) + } + if _, ok := pg.Parameters["cron.database_name"]; ok { + t.Error("DocumentDB-only cron.database_name leaked into pgcosmos parameters") + } + if cluster.Spec.Bootstrap == nil || cluster.Spec.Bootstrap.InitDB == nil || + len(cluster.Spec.Bootstrap.InitDB.PostInitSQL) == 0 || + cluster.Spec.Bootstrap.InitDB.PostInitSQL[0] != "CREATE EXTENSION pgcosmos CASCADE" { + t.Errorf("bootstrap = %+v", cluster.Spec.Bootstrap) + } + + if len(cluster.Spec.Plugins) == 0 || cluster.Spec.Plugins[0].Name != "sidecar.cosmos.azure.com" { + t.Fatalf("plugins = %+v", cluster.Spec.Plugins) + } + params := cluster.Spec.Plugins[0].Parameters + for k, want := range map[string]string{ + util.PLUGIN_PARAM_GATEWAY_IMAGE: "reg/pgcosmos-gateway:1", + util.PLUGIN_PARAM_CREDENTIAL_SECRET: "cosmos-credentials", + util.PLUGIN_PARAM_GATEWAY_TLS_SECRET: "acct-tls", + util.PLUGIN_PARAM_GATEWAY_CONTAINER_NAME: "pgcosmos-gateway", + util.PLUGIN_PARAM_GATEWAY_PORT: "8081", + } { + if params[k] != want { + t.Errorf("plugin param %s = %q, want %q", k, params[k], want) + } + } + + if len(cluster.OwnerReferences) != 1 || cluster.OwnerReferences[0].Kind != "CosmosAccount" { + t.Errorf("owner references = %+v", cluster.OwnerReferences) + } +} + +func TestSecondProductResourceValidationUsesProductPath(t *testing.T) { + res := product.Resources{Memory: "not-a-quantity"} + errs := ValidateResourcesFor(res, false, field.NewPath("spec", "compute"), DefaultSplitConfig()) + if len(errs) == 0 { + t.Fatal("expected a validation error") + } + for _, e := range errs { + if regexp.MustCompile(`^spec\.compute`).FindString(e.Field) == "" { + t.Errorf("error field %q is not under spec.compute", e.Field) + } + } +} From 0d733c86ee8650be13ce1228855458d1051087af Mon Sep 17 00:00:00 2001 From: Rayhan Hossain Date: Mon, 19 Oct 2026 16:55:23 -0700 Subject: [PATCH 09/27] feat(core): add public pkg/core facade for product-neutral orchestration Go internal/ packages cannot be imported from another module, so pkg/core re-exports the intent/profile types and wraps the renderer, sync, parameter merge, resource split/validation and extension lifecycle helpers. It adds no behavior. Verified by building and running a throwaway external module that uses a replace directive. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 33b3cb3c-1b25-4ffb-bff1-ebf0f3d04058 Signed-off-by: Rayhan Hossain --- operator/src/pkg/core/core.go | 125 ++++++++++++++++++++++++++ operator/src/pkg/core/example_test.go | 44 +++++++++ 2 files changed, 169 insertions(+) create mode 100644 operator/src/pkg/core/core.go create mode 100644 operator/src/pkg/core/example_test.go diff --git a/operator/src/pkg/core/core.go b/operator/src/pkg/core/core.go new file mode 100644 index 00000000..b6d32708 --- /dev/null +++ b/operator/src/pkg/core/core.go @@ -0,0 +1,125 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +// Package core is the public, product-neutral surface of the operator's +// CloudNative-PG orchestration. Another operator module (for example an +// operator for a different PostgreSQL-extension product) builds a +// ProductProfile and a ClusterIntent from its own custom resource and drives +// the same renderer, sync, parameter, resource and extension-lifecycle logic +// the DocumentDB operator uses. +// +// The package only re-exports types and functions from the operator's internal +// packages; it adds no behavior of its own, so both products share one +// implementation. +package core + +import ( + "context" + + cnpgv1 "github.com/cloudnative-pg/cloudnative-pg/api/v1" + "github.com/go-logr/logr" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/util/validation/field" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/documentdb/documentdb-operator/internal/cnpg" + "github.com/documentdb/documentdb-operator/internal/product" +) + +// Product-neutral seam. +type ( + ProductProfile = product.ProductProfile + PostgresProfile = product.PostgresProfile + ExtensionSpec = product.ExtensionSpec + GatewayProfile = product.GatewayProfile + Adapter = product.Adapter + + ClusterIntent = product.ClusterIntent + Images = product.Images + Topology = product.Topology + Storage = product.Storage + Resources = product.Resources + ComponentResources = product.ComponentResources + Identity = product.Identity + Postgres = product.Postgres + FeatureGates = product.FeatureGates + TLS = product.TLS + Recovery = product.Recovery + Bootstrap = product.Bootstrap +) + +// Rendering and sync. +type ( + SplitConfig = cnpg.SplitConfig + ResourceSplit = cnpg.ResourceSplit + ComponentResource = cnpg.ComponentResource + JSONPatch = cnpg.JSONPatch +) + +// RenderCluster renders the CNPG Cluster for a product-neutral intent. +func RenderCluster(req ctrl.Request, intent ClusterIntent, serviceAccountName, storageClass string, isPrimaryRegion bool, log logr.Logger) *cnpgv1.Cluster { + return cnpg.GetCnpgClusterSpecFromIntent(req, intent, serviceAccountName, storageClass, isPrimaryRegion, log) +} + +// SyncCluster patches the mutable fields of current towards desired, including +// the product extension and gateway images, plus any extraOps. +func SyncCluster(ctx context.Context, c client.Client, current, desired *cnpgv1.Cluster, extraOps []JSONPatch) error { + return cnpg.SyncCnpgCluster(ctx, c, current, desired, extraOps) +} + +// MergeParameters returns the PostgreSQL parameters for an intent: memory-aware +// defaults, user overrides, then core and product protected parameters. +func MergeParameters(intent ClusterIntent, memoryLimitBytes int64) map[string]string { + return cnpg.MergeParametersFor(intent, memoryLimitBytes) +} + +// ProtectedParameters returns the parameters the operator always forces. +func ProtectedParameters(intent ClusterIntent) map[string]string { + return cnpg.ProtectedParametersFor(intent) +} + +// DefaultSplitConfig returns the resource split configuration, honoring the +// operator's environment overrides. +func DefaultSplitConfig() SplitConfig { return cnpg.DefaultSplitConfig() } + +// ComputeResourceSplit divides a pod-level resource envelope across +// PostgreSQL, the gateway and (when monitoring) the OTel collector. +func ComputeResourceSplit(res Resources, monitoring bool, cfg SplitConfig) ResourceSplit { + return cnpg.ComputeResourceSplitFor(res, monitoring, cfg) +} + +// ValidateResources validates a resource envelope; errors are reported under +// base, the product's field path for its resources. +func ValidateResources(res Resources, monitoring bool, base *field.Path, cfg SplitConfig) field.ErrorList { + return cnpg.ValidateResourcesFor(res, monitoring, base, cfg) +} + +// ExtensionVersionSQL queries the default and installed versions of extension. +func ExtensionVersionSQL(extension string) string { return cnpg.ExtensionVersionSQL(extension) } + +// ExtensionUpdateSQL upgrades extension, to targetVersion when set. +func ExtensionUpdateSQL(extension, targetVersion string) string { + return cnpg.ExtensionUpdateSQL(extension, targetVersion) +} + +// ReplicationRoleCheckSQL checks whether streaming_replica is a member of role. +func ReplicationRoleCheckSQL(role string) string { return cnpg.ReplicationRoleCheckSQL(role) } + +// ReplicationRoleGrantSQL grants role to streaming_replica. +func ReplicationRoleGrantSQL(role string) string { return cnpg.ReplicationRoleGrantSQL(role) } + +// ExtensionImage returns the image currently mounted for extension in cluster. +func ExtensionImage(cluster *cnpgv1.Cluster, extension string) string { + return cnpg.ExtensionImage(cluster, extension) +} + +// GatewayImage returns the gateway image currently configured on cluster. +func GatewayImage(cluster *cnpgv1.Cluster) string { return cnpg.GatewayImage(cluster) } + +// ParsePullPolicy returns value as a PullPolicy, or "" when it is not valid. +func ParsePullPolicy(value string) corev1.PullPolicy { return product.ParsePullPolicy(value) } + +// DocumentDBProfile returns the DocumentDB product profile, a reference for +// what a complete profile contains. +func DocumentDBProfile() ProductProfile { return product.DocumentDBProfile() } diff --git a/operator/src/pkg/core/example_test.go b/operator/src/pkg/core/example_test.go new file mode 100644 index 00000000..d8749794 --- /dev/null +++ b/operator/src/pkg/core/example_test.go @@ -0,0 +1,44 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package core_test + +import ( + "fmt" + + "github.com/go-logr/logr" + ctrl "sigs.k8s.io/controller-runtime" + + "github.com/documentdb/documentdb-operator/pkg/core" +) + +// Example renders a CNPG Cluster for a product other than DocumentDB. +func Example() { + intent := core.ClusterIntent{ + Images: core.Images{PostgresExtension: "reg/pgcosmos:1", Gateway: "reg/pgcosmos-gateway:1"}, + Topology: core.Topology{Instances: 1}, + Storage: core.Storage{PvcSize: "10Gi"}, + Identity: core.Identity{Name: "acct", APIVersion: "cosmos.azure.com/v1", Kind: "CosmosAccount"}, + CredentialSecret: "cosmos-credentials", + SidecarInjectorPlugin: "sidecar.cosmos.azure.com", + Product: core.ProductProfile{ + Name: "Cosmos DB", + Postgres: core.PostgresProfile{ + Extension: core.ExtensionSpec{Name: "pgcosmos"}, + ProtectedParameters: map[string]string{"pgcosmos.mode": "operator"}, + }, + Gateway: core.GatewayProfile{ContainerName: "pgcosmos-gateway"}, + }, + } + req := ctrl.Request{} + req.Name, req.Namespace = "acct", "ns" + + cluster := core.RenderCluster(req, intent, "acct-sa", "", true, logr.Discard()) + fmt.Println(cluster.Spec.PostgresConfiguration.Extensions[0].Name) + fmt.Println(cluster.Spec.PostgresConfiguration.Parameters["pgcosmos.mode"]) + fmt.Println(core.ExtensionUpdateSQL("pgcosmos", "")) + // Output: + // pgcosmos + // operator + // ALTER EXTENSION pgcosmos UPDATE +} From 18e8b7a0c26fbc3ce4e4fc44f28e442576197ec1 Mon Sep 17 00:00:00 2001 From: Rayhan Hossain Date: Mon, 19 Oct 2026 16:56:21 -0700 Subject: [PATCH 10/27] docs: describe the product-neutral operator core Adds a design doc for the ProductProfile/ClusterIntent seam, the pkg/core facade, retained wire contracts and follow-ups; documents the new sidecar-injector gateway parameters; points AGENTS.md at the core; adds an Unreleased CHANGELOG entry. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 33b3cb3c-1b25-4ffb-bff1-ebf0f3d04058 Signed-off-by: Rayhan Hossain --- AGENTS.md | 5 + CHANGELOG.md | 3 + docs/designs/product-neutral-core.md | 138 ++++++++++++++++++ .../sidecar-injector-plugin-configuration.md | 26 ++++ 4 files changed, 172 insertions(+) create mode 100644 docs/designs/product-neutral-core.md diff --git a/AGENTS.md b/AGENTS.md index 92e5e1d6..725ab0d3 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -20,6 +20,11 @@ The DocumentDB Kubernetes Operator is a Kubernetes operator that manages Documen - CLI plugin for managing DocumentDB resources - Provides commands for status, health checks, and operations +- **Product-neutral core** (in `operator/src/internal/product`, `operator/src/internal/cnpg`, public facade `operator/src/pkg/core`) + - The CNPG renderer, sync, PostgreSQL parameters, resource split and extension lifecycle are driven by a `ProductProfile` + `ClusterIntent`; DocumentDB is the first product adapter + - Keep DocumentDB-specific values in `internal/product/documentdb.go`; golden tests in `internal/cnpg/testdata/golden` must stay byte-identical + - See [docs/designs/product-neutral-core.md](docs/designs/product-neutral-core.md) + - **Helm Chart** (in `operator/documentdb-helm-chart/`) - Packages operator deployment configuration - Manages CRDs and RBAC resources diff --git a/CHANGELOG.md b/CHANGELOG.md index 3ddded59..66f12be3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,9 @@ ### Security - Require Go 1.26.10 across all six modules and pin Go builder images to that patch release to address [GO-2026-6809](https://pkg.go.dev/vuln/GO-2026-6809) in `crypto/x509`. Unit-test workflows now resolve Go from the operator module instead of an older hard-coded version. +### Major Features +- **Product-neutral operator core**: The CNPG `Cluster` renderer, sync, PostgreSQL parameter merge, resource split/validation and extension lifecycle SQL are now driven by a `ProductProfile` and `ClusterIntent` instead of the `DocumentDB` custom resource, and are exported for other operator modules through `operator/src/pkg/core`. The sidecar injector accepts optional `gatewayContainerName`, `gatewayPort`, `gatewayArgs`, `gatewayUsernameEnv` and `gatewayPasswordEnv` parameters for non-DocumentDB gateways. Rendered DocumentDB clusters are byte-identical to before, so existing deployments see no change. See [Product-neutral operator core](docs/designs/product-neutral-core.md). + ## [1.0.0] - 2026-10-19 First generally available operator release, for DocumentDB 1.0. The operator and the DocumentDB engine keep separate version numbers; this chart deploys engine `1.0.0` (extension schema `1.0-0`) by default. See [Upgrading to DocumentDB 1.0](docs/operator-public-documentation/preview/operations/upgrades.md#upgrading-to-documentdb-10). diff --git a/docs/designs/product-neutral-core.md b/docs/designs/product-neutral-core.md new file mode 100644 index 00000000..16ee8167 --- /dev/null +++ b/docs/designs/product-neutral-core.md @@ -0,0 +1,138 @@ +# Product-neutral operator core + +## Summary + +The operator manages a PostgreSQL extension plus a protocol gateway on top of +[CloudNative-PG](https://cloudnative-pg.io/) (CNPG). DocumentDB is one such +product: the `documentdb` extension and the DocumentDB gateway. Other products +follow the same pattern with a different extension and gateway. + +This design separates the CNPG orchestration (rendering the `Cluster`, +syncing it, PostgreSQL parameters, the resource split, and the extension +lifecycle) from the DocumentDB product, so a second operator can reuse the same +code. DocumentDB behavior is unchanged: golden snapshots of the rendered +`Cluster` are byte-identical before and after the refactor. + +## Layers + +``` + product CR (DocumentDB, ...) + │ Adapter.ToClusterIntent + ▼ + ClusterIntent + ProductProfile internal/product (public: pkg/core) + │ + ▼ + renderer · sync · params · split · internal/cnpg (public: pkg/core) + extension lifecycle SQL + │ + ▼ + CNPG Cluster ──► sidecar-injector plugin (gateway contract via plugin params) +``` + +- **`ProductProfile`** (`internal/product/profile.go`) holds what is fixed for + a product: name, image repositories and default tag, default credential + secret, CNPG plugin names, the PostgreSQL contract (`PostgresProfile`), and + the gateway contract (`GatewayProfile`). +- **`PostgresProfile`** holds the extension (`ExtensionSpec`: name and + ImageVolume library/control/ld paths), the extra + `shared_preload_libraries`, the bootstrap SQL, `pg_hba`, product-protected + GUCs, and the role granted to `streaming_replica`. +- **`GatewayProfile`** holds the sidecar container name, port, args, and the + credential env var names. Empty fields keep the DocumentDB gateway defaults. +- **`ClusterIntent`** (`internal/product/intent.go`) is the per-instance + desired state: images, topology, storage, resources, monitoring, TLS, + identity (owner reference), PostgreSQL settings, feature gates, bootstrap, + and the embedded `ProductProfile`. +- **`DocumentDBAdapter`** (`internal/product/documentdb.go`) maps a `DocumentDB` + custom resource to a `ClusterIntent`. It is the only place that reads the + DocumentDB spec for rendering. + +The renderer (`cnpg.GetCnpgClusterSpecFromIntent`) only reads the intent. The +old `GetCnpgClusterSpec`, `MergeParameters`, `ProtectedParameters`, +`ComputeResourceSplit`, and `ValidateResources` entry points remain as thin +DocumentDB wrappers. + +## What moved where + +| Concern | Product-neutral | DocumentDB value | +| --- | --- | --- | +| Extension mount and name | `PostgresProfile.Extension` | `documentdb` | +| `shared_preload_libraries` | `PostgresProfile.AdditionalLibraries` | `pg_cron`, `pg_documentdb_core`, `pg_documentdb` | +| initdb SQL | `PostgresProfile.BootstrapSQL` | `CREATE EXTENSION documentdb CASCADE`, ... | +| Forced GUCs | core: `max_replication_slots`, `max_wal_senders`, `max_prepared_transactions`; product: `PostgresProfile.ProtectedParameters` | `cron.database_name` | +| Replication grant | `PostgresProfile.ReplicationAdminRole` (skipped when empty) | `documentdb_admin_role` | +| Extension upgrade SQL | `cnpg.ExtensionVersionSQL` / `ExtensionUpdateSQL` | name from the profile | +| Image sync | `SyncCnpgCluster` reads the extension by name | first `Extensions` entry | +| Resource validation path | `ValidateResourcesFor(..., base)` | `spec.resource` | +| Gateway sidecar | `GatewayProfile` → plugin params | injector defaults | + +## Sidecar-injector gateway contract + +The injector accepts five optional plugin parameters: `gatewayContainerName`, +`gatewayPort`, `gatewayArgs` (a JSON array that replaces the DocumentDB args +verbatim), `gatewayUsernameEnv`, and `gatewayPasswordEnv`. They default to the +DocumentDB gateway (`documentdb-gateway`, `10260`, `--create-user ... +--start-pg false --pg-port 5432`, `USERNAME`, `PASSWORD`). The operator sends +them only when `ProductProfile.Gateway` sets them, and the injector serializes +them only when they differ from the defaults, so existing clusters see no +parameter change and no rolling restart. See +[the plugin configuration guide](../developer-guides/sidecar-injector-plugin-configuration.md). + +## Wire contracts kept on purpose + +These names are visible to running clusters or users. Renaming them would +restart pods or break configuration, so they stay as they are: + +- The plugin parameter key `documentDbCredentialSecret` carries the credential + secret name for every product. +- Operator environment variables keep their `DOCUMENTDB_*` names (for example + `DOCUMENTDB_GATEWAY_MEMORY_FRACTION` and `DOCUMENTDB_IOURING_SECCOMP_PROFILE`). +- The credential secret keys are `username` and `password`, and the gateway + TLS secret is mounted at `/tls`. + +## Using the core from another operator + +Go does not allow importing `internal/` packages from another module, so +`operator/src/pkg/core` re-exports the seam: type aliases for the intent and +profile types plus `RenderCluster`, `SyncCluster`, `MergeParameters`, +`ProtectedParameters`, `ComputeResourceSplit`, `ValidateResources`, +`DefaultSplitConfig`, the extension SQL helpers, and `DocumentDBProfile` as a +reference profile. It adds no behavior of its own. + +A second product: + +1. Defines its own CRD and an adapter that returns a `core.ProductProfile` and + builds a `core.ClusterIntent` from its custom resource. +2. Writes its own reconciler. It calls `core.RenderCluster` to build the + desired `Cluster` and `core.SyncCluster` to apply changes, and runs + `core.ExtensionUpdateSQL` / `core.ReplicationRoleGrantSQL` for the + extension lifecycle. +3. Ships the shared sidecar-injector with its gateway image, setting + `ProductProfile.Gateway` when its gateway contract differs. + +`pkg/core/example_test.go` renders a non-DocumentDB cluster through the facade. +`internal/cnpg/second_product_test.go` renders a synthetic `pgcosmos` product +end to end and fails if any DocumentDB string other than the documented +`documentDbCredentialSecret` key appears in the output. + +## Testing guarantees + +- `internal/cnpg/cnpg_golden_test.go` pins the rendered `Cluster` for seven + DocumentDB scenarios. Any change to DocumentDB output fails the test. + Regenerate the snapshots with `UPDATE_GOLDEN=1 go test ./internal/cnpg` only + for intentional changes. +- Unit tests cover the `...For` functions with a non-DocumentDB field path and + extension name, sync of a `pgcosmos` extension, the gateway contract on both + the operator and the injector side, and the facade. + +## Not in scope / follow-ups + +- `DocumentDBReconciler`, the webhook, backup CRDs, and the kubectl plugin stay + DocumentDB-specific. A second product writes its own reconciler on top of + `pkg/core`. +- The client `Service` port comes from the `GATEWAY_PORT` environment variable + (default `10260`), not from `GatewayProfile.Port`. +- The OTel collector ConfigMap labels metrics with `documentdb.cluster`. + Making the attribute name a profile field is a follow-up. +- `MonitoringSpec` and `CertificatesConfiguration` are reused from the + DocumentDB API and CNPG respectively as shared types. diff --git a/docs/developer-guides/sidecar-injector-plugin-configuration.md b/docs/developer-guides/sidecar-injector-plugin-configuration.md index 2dc7b8de..698e12f5 100644 --- a/docs/developer-guides/sidecar-injector-plugin-configuration.md +++ b/docs/developer-guides/sidecar-injector-plugin-configuration.md @@ -78,6 +78,32 @@ plugins: annotations: '{"prometheus.io/scrape":"true","prometheus.io/port":"8080"}' ``` +### 4. Gateway Container Contract + +These optional parameters let a product other than DocumentDB run its own +gateway with the same injector. The DocumentDB operator never sets them, so the +defaults below apply. + +| Parameter | Default | Description | +| --- | --- | --- | +| `gatewayContainerName` | `documentdb-gateway` | Name of the injected gateway container | +| `gatewayPort` | `10260` | Gateway container port | +| `gatewayArgs` | DocumentDB gateway args | JSON array that replaces `--create-user --start-pg false --pg-port 5432` and the TLS `--cert-path`/`--key-file` flags verbatim | +| `gatewayUsernameEnv` | `USERNAME` | Env var that receives the credential secret's `username` key | +| `gatewayPasswordEnv` | `PASSWORD` | Env var that receives the credential secret's `password` key | + +The operator forwards these from `ProductProfile.Gateway` (see +[Product-neutral operator core](../designs/product-neutral-core.md)). + +```yaml +plugins: + - name: cnpg-i-sidecar-injector.documentdb.io + parameters: + gatewayContainerName: "pgcosmos-gateway" + gatewayPort: "8081" + gatewayArgs: '["--port","8081"]' +``` + ## CNPG Plugin Parameters The DocumentDB controller automatically passes all configuration parameters to the sidecar injector plugin via CNPG's plugin parameter mechanism: From 2725c115b53923ed03c1f90918e312804811d55f Mon Sep 17 00:00:00 2001 From: Rayhan Hossain Date: Mon, 19 Oct 2026 17:01:13 -0700 Subject: [PATCH 11/27] test(cnpg): use k8s.io/utils/ptr in golden test Full validation for the product-neutral core: every Go module builds and vets, unit tests pass (with -race for the operator and sidecar injector), and golangci-lint reports no new findings against main. Replaces the deprecated pointer helpers the new golden test used. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 33b3cb3c-1b25-4ffb-bff1-ebf0f3d04058 Signed-off-by: Rayhan Hossain --- operator/src/internal/cnpg/cnpg_golden_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/operator/src/internal/cnpg/cnpg_golden_test.go b/operator/src/internal/cnpg/cnpg_golden_test.go index e0fce879..16ba1a51 100644 --- a/operator/src/internal/cnpg/cnpg_golden_test.go +++ b/operator/src/internal/cnpg/cnpg_golden_test.go @@ -12,7 +12,7 @@ import ( cnpgv1 "github.com/cloudnative-pg/cloudnative-pg/api/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/utils/pointer" + "k8s.io/utils/ptr" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/log/zap" @@ -87,8 +87,8 @@ func goldenCases() []goldenCase { d.Spec.DocumentDbCredentialSecret = "creds" d.Spec.Plugins = &dbpreview.PluginsSpec{SidecarInjectorName: "custom.injector.io"} d.Spec.Postgres = &dbpreview.PostgresSpec{ - UID: pointer.Int64(26), - GID: pointer.Int64(26), + UID: ptr.To[int64](26), + GID: ptr.To[int64](26), PostInitSQL: []string{"SELECT 1"}, Parameters: map[string]string{"work_mem": "64MB", "max_wal_senders": "99"}, } @@ -110,7 +110,7 @@ func goldenCases() []goldenCase { }, } d.Spec.Affinity = cnpgv1.AffinityConfiguration{ - EnablePodAntiAffinity: pointer.Bool(true), + EnablePodAntiAffinity: ptr.To(true), TopologyKey: "kubernetes.io/hostname", } }, From 8d8f1a15f8ae7fc521ffd219bfca3ffcbfc9cf18 Mon Sep 17 00:00:00 2001 From: Rayhan Hossain Date: Mon, 19 Oct 2026 17:19:50 -0700 Subject: [PATCH 12/27] feat(product): add ClusterAdapter, neutral spec/status views and naming profile Introduce the operational contract a product implements to run on shared reconcilers: ClusterSpec (read-only operational view), ClusterStatus (neutral status fields), NamingProfile (label/annotation/finalizer domain, service prefix, metric attribute) and the generic ClusterAdapter[T]. DocumentDBAdapter implements it with values identical to today's constants; util.MongoConnectionString is extracted from GenerateConnectionString. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 33b3cb3c-1b25-4ffb-bff1-ebf0f3d04058 Signed-off-by: Rayhan Hossain --- operator/src/internal/product/cluster.go | 128 ++++++++++++++++++ operator/src/internal/product/cluster_test.go | 72 ++++++++++ operator/src/internal/product/documentdb.go | 67 +++++++++ operator/src/internal/product/profile.go | 4 + operator/src/internal/utils/util.go | 9 +- 5 files changed, 279 insertions(+), 1 deletion(-) create mode 100644 operator/src/internal/product/cluster.go create mode 100644 operator/src/internal/product/cluster_test.go diff --git a/operator/src/internal/product/cluster.go b/operator/src/internal/product/cluster.go new file mode 100644 index 00000000..9c25bd87 --- /dev/null +++ b/operator/src/internal/product/cluster.go @@ -0,0 +1,128 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package product + +import ( + "sigs.k8s.io/controller-runtime/pkg/client" + + dbpreview "github.com/documentdb/documentdb-operator/api/preview" +) + +// NamingProfile carries the product's Kubernetes naming conventions: the domain +// used for labels, annotations, and finalizers, the gateway Service name +// prefix, and the OTel resource attribute that identifies the cluster. +type NamingProfile struct { + // Domain prefixes product labels, annotations, and finalizers + // (for example "documentdb.io"). + Domain string + // ServicePrefix is prepended to the custom resource name to form the + // gateway Service name. + ServicePrefix string + // MetricClusterAttribute is the OTel resource attribute key carrying the + // cluster name. + MetricClusterAttribute string +} + +// ClusterLabel is the PV/PVC label holding the owning custom resource name. +func (n NamingProfile) ClusterLabel() string { return n.Domain + "/cluster" } + +// NamespaceLabel is the PV label holding the owning custom resource namespace. +func (n NamingProfile) NamespaceLabel() string { return n.Domain + "/namespace" } + +// RecoveryTempLabel marks the temporary PVC created for PV recovery. +func (n NamingProfile) RecoveryTempLabel() string { return n.Domain + "/recovery-temp" } + +// SchemaVersionAnnotation records the installed extension schema version on a PV. +func (n NamingProfile) SchemaVersionAnnotation() string { return n.Domain + "/schema-version" } + +// Finalizer is the custom resource finalizer used to warn about retained PVs. +func (n NamingProfile) Finalizer() string { return n.Domain + "/pv-retention-finalizer" } + +// ServiceName is the gateway Service name for a custom resource, truncated to +// the 63-character Kubernetes name limit. +func (n NamingProfile) ServiceName(clusterName string) string { + name := n.ServicePrefix + clusterName + if len(name) > 63 { + name = name[:63] + } + return name +} + +// ClusterSpec is the product-neutral, read-only view of the operational inputs +// the shared reconcilers act on. Products typically reuse the api/preview +// sub-types in their own custom resources and copy them here. +type ClusterSpec struct { + // ServiceType is "", "ClusterIP", or "LoadBalancer"; empty means no + // gateway Service is managed. + ServiceType string + // Environment is the cloud environment ("aks", "eks", "gke", or empty) + // used for LoadBalancer annotations and VolumeSnapshotClass defaults. + Environment string + // StorageClass is the PVC storage class; empty uses the cluster default. + StorageClass string + // PersistentVolumeReclaimPolicy is "Retain" (default when empty) or "Delete". + PersistentVolumeReclaimPolicy string + // RecoveryPersistentVolume, when set, bootstraps the cluster from a + // retained PersistentVolume. + RecoveryPersistentVolume string + // GatewayTLS is the gateway TLS configuration; nil means self-signed. + GatewayTLS *dbpreview.GatewayTLS + // SchemaVersion controls the extension upgrade: "" (two-phase, manual), + // "auto", or an explicit semantic version. + SchemaVersion string + // Monitoring is the optional OTel collector configuration. + Monitoring *dbpreview.MonitoringSpec + // Backup is the cluster-level backup configuration. + Backup *dbpreview.BackupConfiguration +} + +// MonitoringEnabled reports whether the OTel collector sidecar is enabled. +func (s ClusterSpec) MonitoringEnabled() bool { + return s.Monitoring != nil && s.Monitoring.Enabled +} + +// RetainsPVs reports whether PVs outlive the cluster (Retain is the default). +func (s ClusterSpec) RetainsPVs() bool { + return s.PersistentVolumeReclaimPolicy == "" || s.PersistentVolumeReclaimPolicy == "Retain" +} + +// ClusterStatus is the product-neutral view of the status fields the shared +// reconcilers own. Product status types keep their own field names; adapters +// map between them. SetStatus must only write these fields. +type ClusterStatus struct { + // Phase mirrors the CNPG cluster phase. + Phase string + // ConnectionString is the client connection string for the gateway. + ConnectionString string + // SchemaVersion is the installed extension schema version (semver). + SchemaVersion string + // ExtensionImage and GatewayImage are the images currently applied to the + // CNPG cluster. + ExtensionImage string + GatewayImage string + // TLS is the gateway TLS status. + TLS *dbpreview.TLSStatus +} + +// ClusterAdapter is the full contract a product implements to run on the shared +// reconcilers: it creates its custom resource type, translates it into the +// rendering intent and operational view, and maps the neutral status. +type ClusterAdapter[T client.Object] interface { + Adapter + // New returns an empty custom resource, used for Get and watches. + New() T + // Intent translates the custom resource into the CNPG rendering intent. + Intent(obj T) ClusterIntent + // Spec returns the operational view of the custom resource. + Spec(obj T) ClusterSpec + // Status reads the neutral status from the custom resource. + Status(obj T) ClusterStatus + // SetStatus writes the neutral status into the custom resource, leaving + // product-only status fields untouched. + SetStatus(obj T, status ClusterStatus) + // ConnectionString builds the client connection string for a gateway + // reachable at host:port. trustTLS is true when the gateway certificate + // is ready and trusted. + ConnectionString(obj T, host string, port int32, trustTLS bool) string +} diff --git a/operator/src/internal/product/cluster_test.go b/operator/src/internal/product/cluster_test.go new file mode 100644 index 00000000..bf349143 --- /dev/null +++ b/operator/src/internal/product/cluster_test.go @@ -0,0 +1,72 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package product + +import ( + "strings" + "testing" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + dbpreview "github.com/documentdb/documentdb-operator/api/preview" + util "github.com/documentdb/documentdb-operator/internal/utils" +) + +func TestDocumentDBNamingMatchesWireContract(t *testing.T) { + n := DocumentDBProfile().Naming + for got, want := range map[string]string{ + n.ClusterLabel(): util.LabelCluster, + n.NamespaceLabel(): util.LabelNamespace, + n.RecoveryTempLabel(): util.LabelRecoveryTemp, + n.SchemaVersionAnnotation(): util.AnnotationSchemaVersion, + n.Finalizer(): "documentdb.io/pv-retention-finalizer", + n.ServicePrefix: util.DOCUMENTDB_SERVICE_PREFIX, + } { + if got != want { + t.Errorf("got %q, want %q", got, want) + } + } + if long := n.ServiceName(strings.Repeat("a", 80)); len(long) != 63 { + t.Errorf("service name not truncated to 63: %d", len(long)) + } +} + +func TestDocumentDBAdapterStatusRoundTrip(t *testing.T) { + a := DocumentDBAdapter{} + db := &dbpreview.DocumentDB{} + db.Status.TargetPrimary = "keep-me" + db.Status.LocalPrimary = "keep-me-too" + want := ClusterStatus{ + Phase: "Cluster in healthy state", ConnectionString: "conn", SchemaVersion: "0.110.0", + ExtensionImage: "ext:1", GatewayImage: "gw:1", TLS: &dbpreview.TLSStatus{Ready: true}, + } + a.SetStatus(db, want) + if got := a.Status(db); got != want { + t.Fatalf("round trip mismatch: %+v", got) + } + if db.Status.TargetPrimary != "keep-me" || db.Status.LocalPrimary != "keep-me-too" { + t.Fatal("SetStatus must not touch product-only status fields") + } +} + +func TestDocumentDBAdapterSpecAndConnectionString(t *testing.T) { + a := DocumentDBAdapter{} + db := &dbpreview.DocumentDB{ObjectMeta: metav1.ObjectMeta{Name: "db", Namespace: "ns"}} + db.Spec.ExposeViaService.ServiceType = "LoadBalancer" + db.Spec.Resource.Storage.StorageClass = "fast" + db.Spec.TLS = &dbpreview.TLSConfiguration{Gateway: &dbpreview.GatewayTLS{Mode: "Provided"}} + db.Spec.Bootstrap = &dbpreview.BootstrapConfiguration{Recovery: &dbpreview.RecoveryConfiguration{ + PersistentVolume: &dbpreview.PVRecoveryConfiguration{Name: "pv-1"}}} + + spec := a.Spec(db) + if spec.ServiceType != "LoadBalancer" || spec.StorageClass != "fast" || + spec.GatewayTLS.Mode != "Provided" || spec.RecoveryPersistentVolume != "pv-1" || !spec.RetainsPVs() { + t.Fatalf("unexpected spec view: %+v", spec) + } + + if got, want := a.ConnectionString(db, "1.2.3.4", util.GetPortFor(util.GATEWAY_PORT), false), + util.GenerateConnectionString(db, "1.2.3.4", false); got != want { + t.Fatalf("connection string drift:\n got %s\nwant %s", got, want) + } +} diff --git a/operator/src/internal/product/documentdb.go b/operator/src/internal/product/documentdb.go index fe5f5d1b..49c34b7d 100644 --- a/operator/src/internal/product/documentdb.go +++ b/operator/src/internal/product/documentdb.go @@ -44,6 +44,11 @@ func DocumentDBProfile() ProductProfile { }, ReplicationAdminRole: "documentdb_admin_role", }, + Naming: NamingProfile{ + Domain: "documentdb.io", + ServicePrefix: util.DOCUMENTDB_SERVICE_PREFIX, + MetricClusterAttribute: "documentdb.cluster", + }, } } @@ -214,3 +219,65 @@ func (a DocumentDBAdapter) ToClusterIntent(db *dbpreview.DocumentDB) ClusterInte // compile-time assertion that DocumentDBAdapter satisfies the Adapter seam. var _ Adapter = DocumentDBAdapter{} + +// DocumentDBAdapter implements the full shared-reconciler contract. +var _ ClusterAdapter[*dbpreview.DocumentDB] = DocumentDBAdapter{} + +// New returns an empty DocumentDB custom resource. +func (DocumentDBAdapter) New() *dbpreview.DocumentDB { return &dbpreview.DocumentDB{} } + +// Intent is ToClusterIntent under the ClusterAdapter contract. +func (a DocumentDBAdapter) Intent(db *dbpreview.DocumentDB) ClusterIntent { + return a.ToClusterIntent(db) +} + +// Spec returns the operational view of a DocumentDB custom resource. +func (DocumentDBAdapter) Spec(db *dbpreview.DocumentDB) ClusterSpec { + spec := ClusterSpec{ + ServiceType: db.Spec.ExposeViaService.ServiceType, + Environment: db.Spec.Environment, + StorageClass: db.Spec.Resource.Storage.StorageClass, + PersistentVolumeReclaimPolicy: db.Spec.Resource.Storage.PersistentVolumeReclaimPolicy, + RecoveryPersistentVolume: db.GetPVNameForRecovery(), + SchemaVersion: db.Spec.SchemaVersion, + Monitoring: db.Spec.Monitoring, + Backup: db.Spec.Backup, + } + if db.Spec.TLS != nil { + spec.GatewayTLS = db.Spec.TLS.Gateway + } + return spec +} + +// Status reads the neutral status fields from a DocumentDB custom resource. +func (DocumentDBAdapter) Status(db *dbpreview.DocumentDB) ClusterStatus { + return ClusterStatus{ + Phase: db.Status.Status, + ConnectionString: db.Status.ConnectionString, + SchemaVersion: db.Status.SchemaVersion, + ExtensionImage: db.Status.DocumentDBImage, + GatewayImage: db.Status.GatewayImage, + TLS: db.Status.TLS, + } +} + +// SetStatus writes the neutral status fields into a DocumentDB custom resource; +// multi-region fields (TargetPrimary, LocalPrimary) are left untouched. +func (DocumentDBAdapter) SetStatus(db *dbpreview.DocumentDB, s ClusterStatus) { + db.Status.Status = s.Phase + db.Status.ConnectionString = s.ConnectionString + db.Status.SchemaVersion = s.SchemaVersion + db.Status.DocumentDBImage = s.ExtensionImage + db.Status.GatewayImage = s.GatewayImage + db.Status.TLS = s.TLS +} + +// ConnectionString builds the MongoDB-protocol connection string for a +// DocumentDB gateway. +func (a DocumentDBAdapter) ConnectionString(db *dbpreview.DocumentDB, host string, port int32, trustTLS bool) string { + secretName := db.Spec.DocumentDbCredentialSecret + if secretName == "" { + secretName = a.Profile().DefaultCredentialSecret + } + return util.MongoConnectionString(secretName, db.Namespace, host, port, trustTLS) +} diff --git a/operator/src/internal/product/profile.go b/operator/src/internal/product/profile.go index 3139c7d4..182c02c2 100644 --- a/operator/src/internal/product/profile.go +++ b/operator/src/internal/product/profile.go @@ -43,6 +43,10 @@ type ProductProfile struct { // Gateway is the product's gateway sidecar contract. Zero values keep the // sidecar injector's DocumentDB gateway defaults. Gateway GatewayProfile + + // Naming is the product's label/annotation/finalizer domain and resource + // naming conventions used by the shared reconcilers. + Naming NamingProfile } // GatewayProfile describes how the sidecar injector runs the product gateway. diff --git a/operator/src/internal/utils/util.go b/operator/src/internal/utils/util.go index ed765031..6440eeea 100644 --- a/operator/src/internal/utils/util.go +++ b/operator/src/internal/utils/util.go @@ -414,7 +414,14 @@ func GenerateConnectionString(documentdb *dbpreview.DocumentDB, serviceIp string if secretName == "" { secretName = DEFAULT_DOCUMENTDB_CREDENTIALS_SECRET } - conn := fmt.Sprintf("mongodb://$(kubectl get secret %s -n %s -o jsonpath='{.data.username}' | base64 -d):$(kubectl get secret %s -n %s -o jsonpath='{.data.password}' | base64 -d)@%s:%d/?directConnection=true&authMechanism=SCRAM-SHA-256&tls=true", secretName, documentdb.Namespace, secretName, documentdb.Namespace, serviceIp, GetPortFor(GATEWAY_PORT)) + return MongoConnectionString(secretName, documentdb.Namespace, serviceIp, GetPortFor(GATEWAY_PORT), trustTLS) +} + +// MongoConnectionString builds the DocumentDB (MongoDB wire protocol) client +// connection string for a gateway at host:port, reading credentials from +// secretName in namespace. +func MongoConnectionString(secretName, namespace, host string, port int32, trustTLS bool) string { + conn := fmt.Sprintf("mongodb://$(kubectl get secret %s -n %s -o jsonpath='{.data.username}' | base64 -d):$(kubectl get secret %s -n %s -o jsonpath='{.data.password}' | base64 -d)@%s:%d/?directConnection=true&authMechanism=SCRAM-SHA-256&tls=true", secretName, namespace, secretName, namespace, host, port) if !trustTLS { conn += "&tlsAllowInvalidCertificates=true" } From 26b630d99c52cf06543585374284d767c4ed585e Mon Sep 17 00:00:00 2001 From: Rayhan Hossain Date: Mon, 19 Oct 2026 17:20:58 -0700 Subject: [PATCH 13/27] refactor: product-neutral gateway Service builder and OTel naming Extract util.BuildGatewayService (name, owner, port, environment) from GetDocumentDBServiceDefinition, and add otel.GenerateConfigMapDataWithOptions so the cluster resource attribute and generator header come from the product NamingProfile. DocumentDB output, including the OTel config hash, is unchanged. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 33b3cb3c-1b25-4ffb-bff1-ebf0f3d04058 Signed-off-by: Rayhan Hossain --- operator/src/internal/cnpg/cnpg_cluster.go | 10 ++- operator/src/internal/otel/config.go | 33 ++++++++- operator/src/internal/otel/options_test.go | 38 ++++++++++ operator/src/internal/product/cluster.go | 2 + operator/src/internal/product/documentdb.go | 1 + operator/src/internal/utils/util.go | 81 +++++++++++++++------ operator/src/internal/utils/util_test.go | 15 ++++ 7 files changed, 154 insertions(+), 26 deletions(-) create mode 100644 operator/src/internal/otel/options_test.go diff --git a/operator/src/internal/cnpg/cnpg_cluster.go b/operator/src/internal/cnpg/cnpg_cluster.go index 2b1417a2..1ab7194d 100644 --- a/operator/src/internal/cnpg/cnpg_cluster.go +++ b/operator/src/internal/cnpg/cnpg_cluster.go @@ -124,7 +124,7 @@ func GetCnpgClusterSpecFromIntent(req ctrl.Request, intent product.ClusterIntent // Compute config hash for change detection. The operator triggers a // rolling restart (via restart annotation) when plugin parameters // change, ensuring pods pick up new config. - if configData, err := otelcfg.GenerateConfigMapData(intent.Identity.Name, req.Namespace, intent.Monitoring); err == nil { + if configData, err := otelcfg.GenerateConfigMapDataWithOptions(OTelOptions(intent.Product), intent.Identity.Name, req.Namespace, intent.Monitoring); err == nil { params[util.PLUGIN_PARAM_OTEL_CONFIG_HASH] = otelcfg.HashConfigMapData(configData) } else { log.Error(err, "Failed to generate OTel config hash; config changes may not trigger rolling restart") @@ -436,3 +436,11 @@ func addGatewayContractParams(params map[string]string, gw product.GatewayProfil addPluginParamIfSet(params, util.PLUGIN_PARAM_GATEWAY_USERNAME_ENV, gw.UsernameEnv) addPluginParamIfSet(params, util.PLUGIN_PARAM_GATEWAY_PASSWORD_ENV, gw.PasswordEnv) } + +// OTelOptions maps the product naming onto the OTel collector config options. +func OTelOptions(p product.ProductProfile) otelcfg.Options { + return otelcfg.Options{ + ClusterAttribute: p.Naming.MetricClusterAttribute, + Generator: p.Naming.OperatorName, + } +} diff --git a/operator/src/internal/otel/config.go b/operator/src/internal/otel/config.go index e87eb7e6..3b5def04 100644 --- a/operator/src/internal/otel/config.go +++ b/operator/src/internal/otel/config.go @@ -67,21 +67,46 @@ func ConfigMapName(clusterName string) string { // then triggers a rolling restart (via restart annotation) so that CNPG // recreates pods without the sidecar. func GenerateConfigMapData(clusterName, namespace string, spec *dbpreview.MonitoringSpec) (map[string]string, error) { - dynamicYAML, err := generateDynamicConfig(clusterName, namespace, spec) + return GenerateConfigMapDataWithOptions(Options{}, clusterName, namespace, spec) +} + +// Options carries the product naming used in the generated collector config. +// Empty fields fall back to the DocumentDB defaults. +type Options struct { + // ClusterAttribute is the resource attribute key holding the cluster name. + ClusterAttribute string + // Generator is the operator name written in the generated-file header. + Generator string +} + +const ( + defaultClusterAttribute = "documentdb.cluster" + defaultGenerator = "documentdb-operator" +) + +// GenerateConfigMapDataWithOptions is GenerateConfigMapData with product naming. +func GenerateConfigMapDataWithOptions(opts Options, clusterName, namespace string, spec *dbpreview.MonitoringSpec) (map[string]string, error) { + if opts.ClusterAttribute == "" { + opts.ClusterAttribute = defaultClusterAttribute + } + if opts.Generator == "" { + opts.Generator = defaultGenerator + } + dynamicYAML, err := generateDynamicConfig(opts.ClusterAttribute, clusterName, namespace, spec) if err != nil { return nil, err } return map[string]string{ "static.yaml": string(baseConfigYAML), - "dynamic.yaml": "# Auto-generated by documentdb-operator. Do not edit.\n" + dynamicYAML, + "dynamic.yaml": "# Auto-generated by " + opts.Generator + ". Do not edit.\n" + dynamicYAML, }, nil } // generateDynamicConfig builds the per-cluster dynamic config (resource // processor, exporters, pipeline wiring) that the collector deep-merges // with the embedded base_config.yaml. -func generateDynamicConfig(clusterName, namespace string, spec *dbpreview.MonitoringSpec) (string, error) { +func generateDynamicConfig(clusterAttribute, clusterName, namespace string, spec *dbpreview.MonitoringSpec) (string, error) { cfg := collectorConfig{ Processors: map[string]any{ // `insert` (not `upsert`) so receivers that already emit their @@ -89,7 +114,7 @@ func generateDynamicConfig(clusterName, namespace string, spec *dbpreview.Monito // gap for receivers that don't (e.g. sqlquery). "resource": map[string]any{ "attributes": []map[string]any{ - {"key": "documentdb.cluster", "value": clusterName, "action": "insert"}, + {"key": clusterAttribute, "value": clusterName, "action": "insert"}, {"key": "k8s.namespace.name", "value": namespace, "action": "insert"}, {"key": "k8s.pod.name", "value": "${POD_NAME}", "action": "insert"}, }, diff --git a/operator/src/internal/otel/options_test.go b/operator/src/internal/otel/options_test.go new file mode 100644 index 00000000..a51f5c04 --- /dev/null +++ b/operator/src/internal/otel/options_test.go @@ -0,0 +1,38 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package otel + +import ( + "reflect" + "strings" + "testing" + + dbpreview "github.com/documentdb/documentdb-operator/api/preview" +) + +func TestGenerateConfigMapDataWithOptions(t *testing.T) { + spec := &dbpreview.MonitoringSpec{Enabled: true} + + def, err := GenerateConfigMapData("c", "ns", spec) + if err != nil { + t.Fatal(err) + } + empty, err := GenerateConfigMapDataWithOptions(Options{}, "c", "ns", spec) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(def, empty) { + t.Fatal("empty options must produce the DocumentDB config") + } + + custom, err := GenerateConfigMapDataWithOptions(Options{ClusterAttribute: "cosmos.cluster", Generator: "cosmos-operator"}, "c", "ns", spec) + if err != nil { + t.Fatal(err) + } + dyn := custom["dynamic.yaml"] + if !strings.HasPrefix(dyn, "# Auto-generated by cosmos-operator.") || !strings.Contains(dyn, "cosmos.cluster") || + strings.Contains(dyn, "documentdb") { + t.Fatalf("custom naming not applied:\n%s", dyn) + } +} diff --git a/operator/src/internal/product/cluster.go b/operator/src/internal/product/cluster.go index 9c25bd87..c68030a0 100644 --- a/operator/src/internal/product/cluster.go +++ b/operator/src/internal/product/cluster.go @@ -22,6 +22,8 @@ type NamingProfile struct { // MetricClusterAttribute is the OTel resource attribute key carrying the // cluster name. MetricClusterAttribute string + // OperatorName identifies the operator in generated artifacts. + OperatorName string } // ClusterLabel is the PV/PVC label holding the owning custom resource name. diff --git a/operator/src/internal/product/documentdb.go b/operator/src/internal/product/documentdb.go index 49c34b7d..8e262797 100644 --- a/operator/src/internal/product/documentdb.go +++ b/operator/src/internal/product/documentdb.go @@ -48,6 +48,7 @@ func DocumentDBProfile() ProductProfile { Domain: "documentdb.io", ServicePrefix: util.DOCUMENTDB_SERVICE_PREFIX, MetricClusterAttribute: "documentdb.cluster", + OperatorName: "documentdb-operator", }, } } diff --git a/operator/src/internal/utils/util.go b/operator/src/internal/utils/util.go index 6440eeea..fdfe07dd 100644 --- a/operator/src/internal/utils/util.go +++ b/operator/src/internal/utils/util.go @@ -30,51 +30,90 @@ import ( // GetDocumentDBServiceDefinition returns the LoadBalancer Service definition for a given DocumentDB instance func GetDocumentDBServiceDefinition(documentdb *dbpreview.DocumentDB, replicationContext *ReplicationContext, namespace string, serviceType corev1.ServiceType) *corev1.Service { + // Ensure service name doesn't exceed 63 characters (Kubernetes limit) + serviceName := DOCUMENTDB_SERVICE_PREFIX + documentdb.Name + if len(serviceName) > 63 { + serviceName = serviceName[:63] + } + return BuildGatewayService(GatewayServiceOptions{ + Name: serviceName, + Namespace: namespace, + ClusterName: documentdb.Name, + OwnerAPIVersion: documentdb.APIVersion, + OwnerKind: documentdb.Kind, + OwnerUID: documentdb.UID, + Port: GetPortFor(GATEWAY_PORT), + EndpointEnabled: replicationContext.EndpointEnabled(), + Environment: replicationContext.Environment, + ServiceType: serviceType, + }) +} + +// GatewayServiceOptions are the product-neutral inputs of the gateway Service. +type GatewayServiceOptions struct { + // Name and Namespace of the Service. + Name string + Namespace string + // ClusterName is the owning custom resource name; it is the owner + // reference name and the pod selector's app label value. + ClusterName string + // OwnerAPIVersion, OwnerKind, and OwnerUID identify the owning custom + // resource for the controller owner reference. + OwnerAPIVersion string + OwnerKind string + OwnerUID types.UID + // Port is the gateway port exposed by the Service and targeted on the pod. + Port int32 + // EndpointEnabled selects the CNPG primary; when false the Service selects + // nothing (for example a multi-region replica cluster). + EndpointEnabled bool + // Environment selects cloud-specific LoadBalancer annotations. + Environment string + ServiceType corev1.ServiceType +} + +// BuildGatewayService returns the gateway Service that forwards client traffic +// to the CNPG primary instance. +func BuildGatewayService(o GatewayServiceOptions) *corev1.Service { // If no local HA, these two should be empty selector := map[string]string{ "disabled": "true", } - if replicationContext.EndpointEnabled() { + if o.EndpointEnabled { selector = map[string]string{ - LABEL_APP: documentdb.Name, + LABEL_APP: o.ClusterName, "cnpg.io/instanceRole": "primary", // Service forwards traffic to CNPG primary instance } } - // Ensure service name doesn't exceed 63 characters (Kubernetes limit) - serviceName := DOCUMENTDB_SERVICE_PREFIX + documentdb.Name - if len(serviceName) > 63 { - serviceName = serviceName[:63] - } - service := &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ - Name: serviceName, - Namespace: namespace, - // CRITICAL: Set owner reference so service gets deleted when DocumentDB instance is deleted + Name: o.Name, + Namespace: o.Namespace, + // CRITICAL: Set owner reference so service gets deleted when the owning instance is deleted OwnerReferences: []metav1.OwnerReference{ { - APIVersion: documentdb.APIVersion, - Kind: documentdb.Kind, - Name: documentdb.Name, - UID: documentdb.UID, - Controller: &[]bool{true}[0], // This service is controlled by the DocumentDB instance - BlockOwnerDeletion: &[]bool{true}[0], // Block DocumentDB deletion until service is deleted + APIVersion: o.OwnerAPIVersion, + Kind: o.OwnerKind, + Name: o.ClusterName, + UID: o.OwnerUID, + Controller: &[]bool{true}[0], // This service is controlled by the owning instance + BlockOwnerDeletion: &[]bool{true}[0], // Block owner deletion until service is deleted }, }, }, Spec: corev1.ServiceSpec{ Selector: selector, Ports: []corev1.ServicePort{ - {Name: "gateway", Protocol: corev1.ProtocolTCP, Port: GetPortFor(GATEWAY_PORT), TargetPort: intstr.FromInt(int(GetPortFor(GATEWAY_PORT)))}, + {Name: "gateway", Protocol: corev1.ProtocolTCP, Port: o.Port, TargetPort: intstr.FromInt(int(o.Port))}, }, - Type: serviceType, + Type: o.ServiceType, }, } // Add environment-specific annotations for LoadBalancer services - if serviceType == corev1.ServiceTypeLoadBalancer { - service.ObjectMeta.Annotations = getEnvironmentSpecificAnnotations(replicationContext.Environment) + if o.ServiceType == corev1.ServiceTypeLoadBalancer { + service.ObjectMeta.Annotations = getEnvironmentSpecificAnnotations(o.Environment) } return service diff --git a/operator/src/internal/utils/util_test.go b/operator/src/internal/utils/util_test.go index f2207a7d..b069f44d 100644 --- a/operator/src/internal/utils/util_test.go +++ b/operator/src/internal/utils/util_test.go @@ -1002,3 +1002,18 @@ func TestSemverToExtensionVersion(t *testing.T) { }) } } + +func TestBuildGatewayService_ProductNeutral(t *testing.T) { + svc := BuildGatewayService(GatewayServiceOptions{ + Name: "cosmos-svc-x", Namespace: "ns", ClusterName: "x", + OwnerAPIVersion: "cosmos.example/v1", OwnerKind: "CosmosCluster", OwnerUID: "uid", + Port: 8081, EndpointEnabled: true, Environment: "aks", ServiceType: corev1.ServiceTypeLoadBalancer, + }) + if svc.Name != "cosmos-svc-x" || svc.Spec.Ports[0].Port != 8081 || svc.Spec.Ports[0].TargetPort.IntValue() != 8081 { + t.Fatalf("unexpected service: %+v", svc.Spec) + } + if svc.OwnerReferences[0].Kind != "CosmosCluster" || svc.Spec.Selector[LABEL_APP] != "x" || + svc.Annotations["service.beta.kubernetes.io/azure-load-balancer-external"] != "true" { + t.Fatalf("unexpected owner/selector/annotations: %+v", svc.ObjectMeta) + } +} From caa7ddabbfe94c96338d10d5ac2d7ae8c0b6a22c Mon Sep 17 00:00:00 2001 From: Rayhan Hossain Date: Mon, 19 Oct 2026 18:32:07 -0700 Subject: [PATCH 14/27] refactor: extract generic ClusterReconciler engine Move the DocumentDB cluster reconcile loop (service, RBAC, CNPG render/sync, PV retention and recovery, OTel config, extension upgrade, status) into internal/engine as ClusterReconciler[T] driven by product.ClusterAdapter. Multi-region replication is wired through the optional ReplicationHooks. DocumentDBReconciler keeps its fields and methods and delegates to the engine; behavior, events and controller wiring are unchanged. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 33b3cb3c-1b25-4ffb-bff1-ebf0f3d04058 Signed-off-by: Rayhan Hossain --- .../controller/documentdb_controller.go | 984 ++---------------- .../src/internal/engine/cluster_reconciler.go | 574 ++++++++++ operator/src/internal/engine/extension.go | 297 ++++++ operator/src/internal/engine/predicates.go | 53 + operator/src/internal/engine/pv_recovery.go | 121 +++ operator/src/internal/engine/sql.go | 80 ++ operator/src/internal/utils/pv_recovery.go | 10 +- .../src/internal/utils/replication_context.go | 21 +- 8 files changed, 1238 insertions(+), 902 deletions(-) create mode 100644 operator/src/internal/engine/cluster_reconciler.go create mode 100644 operator/src/internal/engine/extension.go create mode 100644 operator/src/internal/engine/predicates.go create mode 100644 operator/src/internal/engine/pv_recovery.go create mode 100644 operator/src/internal/engine/sql.go diff --git a/operator/src/internal/controller/documentdb_controller.go b/operator/src/internal/controller/documentdb_controller.go index d305cd52..b80d0ae3 100644 --- a/operator/src/internal/controller/documentdb_controller.go +++ b/operator/src/internal/controller/documentdb_controller.go @@ -4,11 +4,8 @@ package controller import ( - "bytes" "context" "fmt" - "slices" - "strings" "sync" "time" @@ -16,43 +13,32 @@ import ( "github.com/cloudnative-pg/cloudnative-pg/pkg/resources/status" pgTime "github.com/cloudnative-pg/machinery/pkg/postgres/time" corev1 "k8s.io/api/core/v1" - rbacv1 "k8s.io/api/rbac/v1" - "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes" - "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" "k8s.io/client-go/tools/record" - "k8s.io/client-go/tools/remotecommand" ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" - "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/log" - "sigs.k8s.io/controller-runtime/pkg/predicate" dbpreview "github.com/documentdb/documentdb-operator/api/preview" cnpg "github.com/documentdb/documentdb-operator/internal/cnpg" - otelcfg "github.com/documentdb/documentdb-operator/internal/otel" + "github.com/documentdb/documentdb-operator/internal/engine" "github.com/documentdb/documentdb-operator/internal/product" util "github.com/documentdb/documentdb-operator/internal/utils" ) const ( - RequeueAfterShort = 10 * time.Second - RequeueAfterLong = 30 * time.Second + RequeueAfterShort = engine.RequeueAfterShort + RequeueAfterLong = engine.RequeueAfterLong // documentDBFinalizer ensures we can emit PV retention warnings before deletion completes documentDBFinalizer = "documentdb.io/pv-retention-finalizer" - - // cnpgClusterHealthyPhase is the CNPG cluster status phase indicating a healthy cluster. - // This value is from CNPG's internal status representation. - cnpgClusterHealthyPhase = "Cluster in healthy state" ) -// DocumentDBReconciler reconciles a DocumentDB object +// DocumentDBReconciler reconciles a DocumentDB object. The single-region +// lifecycle is implemented by the product-neutral engine.ClusterReconciler; +// this type supplies the DocumentDB adapter and the multi-region hooks. type DocumentDBReconciler struct { client.Client Scheme *runtime.Scheme @@ -61,13 +47,28 @@ type DocumentDBReconciler struct { // Recorder emits Kubernetes events for this controller, including PV retention warnings during deletion. Recorder record.EventRecorder // SQLExecutor executes SQL commands against a CNPG cluster's primary pod. - // Defaults to executeSQLCommand (real pod exec via SPDY). Override in tests + // Defaults to engine.NewPodSQLExecutor (real pod exec via SPDY). Override in tests // to inject canned responses without requiring a live Kubernetes cluster. SQLExecutor func(ctx context.Context, cluster *cnpgv1.Cluster, sqlCommand string) (string, error) } var reconcileMutex sync.Mutex +// engine returns the product-neutral reconciler bound to this reconciler's +// clients. It is built per call so struct-literal construction (as in tests) +// keeps working. +func (r *DocumentDBReconciler) engine() *engine.ClusterReconciler[*dbpreview.DocumentDB] { + return &engine.ClusterReconciler[*dbpreview.DocumentDB]{ + Client: r.Client, + Scheme: r.Scheme, + Recorder: r.Recorder, + SQLExecutor: r.SQLExecutor, + Adapter: product.DocumentDBAdapter{}, + Replication: documentDBReplication{r}, + Mutex: &reconcileMutex, + } +} + // +kubebuilder:rbac:groups=documentdb.io,resources=dbs,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=documentdb.io,resources=dbs/status,verbs=get;update;patch // +kubebuilder:rbac:groups=documentdb.io,resources=dbs/finalizers,verbs=update @@ -75,449 +76,92 @@ var reconcileMutex sync.Mutex // +kubebuilder:rbac:groups="",resources=persistentvolumeclaims,verbs=get;list;watch;create;delete // +kubebuilder:rbac:groups="",resources=persistentvolumes,verbs=get;list;watch;update;patch func (r *DocumentDBReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { - reconcileMutex.Lock() - defer reconcileMutex.Unlock() - - logger := log.FromContext(ctx) - - // Fetch the DocumentDB instance - documentdb := &dbpreview.DocumentDB{} - err := r.Get(ctx, req.NamespacedName, documentdb) - if err != nil { - if errors.IsNotFound(err) { - // DocumentDB resource not found, handle cleanup - logger.Info("DocumentDB resource not found. Cleaning up associated resources.") - if err := r.cleanupResources(ctx, req); err != nil { - return ctrl.Result{}, err - } - return ctrl.Result{}, nil - } - logger.Error(err, "Failed to get DocumentDB resource") - return ctrl.Result{}, err - } - - // Handle finalizer lifecycle (add on create, remove on delete) - if done, result, err := r.reconcileFinalizer(ctx, documentdb); done || err != nil { - return result, err - } - - replicationContext, err := util.GetReplicationContext(ctx, r.Client, *documentdb) - if err != nil { - logger.Error(err, "Failed to determine replication context") - return ctrl.Result{}, err - } - - if replicationContext.IsNotPresent() { - logger.Info("DocumentDB instance is not part of the replication setup; skipping reconciliation and deleting any present resources") - if err := r.cleanupResources(ctx, req); err != nil { - return ctrl.Result{}, err - } - if err := util.DeleteOwnedResources(ctx, r.Client, documentdb.ObjectMeta); err != nil { - return ctrl.Result{}, err - } - return ctrl.Result{}, nil - } - - var documentDbServiceIp string - - // Only create/manage the service if ExposeViaService is configured - if documentdb.Spec.ExposeViaService.ServiceType != "" { - serviceType := corev1.ServiceTypeClusterIP - if documentdb.Spec.ExposeViaService.ServiceType == "LoadBalancer" { - serviceType = corev1.ServiceTypeLoadBalancer // Public LoadBalancer service - } - - // Define the Service for this DocumentDB instance - ddbService := util.GetDocumentDBServiceDefinition(documentdb, replicationContext, req.Namespace, serviceType) - - // Check if the DocumentDB Service already exists for this instance - foundService, err := util.UpsertService(ctx, r.Client, ddbService) - if err != nil { - logger.Error(err, "Failed to create DocumentDB Service; Requeuing.") - return ctrl.Result{RequeueAfter: RequeueAfterShort}, nil - } - - // Ensure DocumentDB Service has an IP assigned - documentDbServiceIp, err = util.EnsureServiceIP(ctx, foundService) - if err != nil { - logger.Info("DocumentDB Service IP not assigned, pausing until update posted.") - return ctrl.Result{}, nil - } - } - - // Ensure App ServiceAccount, Role and RoleBindings are created - if err := r.EnsureServiceAccountRoleAndRoleBinding(ctx, documentdb, req.Namespace); err != nil { - logger.Info("Failed to create ServiceAccount, Role and RoleBinding; Requeuing.") - return ctrl.Result{RequeueAfter: RequeueAfterShort}, nil - } - - // create the CNPG Cluster - intent := product.DocumentDBAdapter{}.ToClusterIntent(documentdb) - - currentCnpgCluster := &cnpgv1.Cluster{} - desiredCnpgCluster := cnpg.GetCnpgClusterSpecFromIntent(req, intent, documentdb.Name, replicationContext.StorageClass, replicationContext.IsPrimary(), logger) - - if replicationContext.IsReplicating() { - err = r.AddClusterReplicationToClusterSpec(ctx, documentdb, replicationContext, desiredCnpgCluster) - if err != nil { - logger.Error(err, "Failed to add physical replication features cnpg Cluster spec") - return ctrl.Result{RequeueAfter: RequeueAfterShort}, nil - } - } - - // Handle PV recovery lifecycle (create temp PVC before CNPG, cleanup after healthy) - if result, err := r.reconcilePVRecovery(ctx, documentdb, req.Namespace, desiredCnpgCluster.Name); err != nil { - logger.Error(err, "Failed to reconcile PV recovery") - return result, err - } else if result.Requeue || result.RequeueAfter > 0 { - return result, nil - } - - // Reconcile OTel resources when monitoring is enabled. - // The sidecar itself is added/removed via CNPG plugin parameters; - // the operator triggers a rolling restart (via restart annotation) - // and CNPG manages the pod rollout. - monitoringEnabled := documentdb.Spec.Monitoring != nil && documentdb.Spec.Monitoring.Enabled - if monitoringEnabled { - if err := r.reconcileOtelConfigMap(ctx, documentdb, req.Namespace); err != nil { - logger.Error(err, "Failed to reconcile OTel ConfigMap") - return ctrl.Result{RequeueAfter: RequeueAfterShort}, nil - } - } - - if err := r.Client.Get(ctx, types.NamespacedName{Name: desiredCnpgCluster.Name, Namespace: req.Namespace}, currentCnpgCluster); err != nil { - if errors.IsNotFound(err) { - if err := r.Client.Create(ctx, desiredCnpgCluster); err != nil { - logger.Error(err, "Failed to create CNPG Cluster") - return ctrl.Result{RequeueAfter: RequeueAfterShort}, nil - } - logger.Info("CNPG Cluster created successfully", "Cluster.Name", desiredCnpgCluster.Name, "Namespace", desiredCnpgCluster.Namespace) - return ctrl.Result{RequeueAfter: RequeueAfterLong}, nil - } - logger.Error(err, "Failed to get CNPG Cluster") - return ctrl.Result{RequeueAfter: RequeueAfterShort}, nil - } - - // Build replication patch ops (performs side effects: HTTP token reads, service creation). - // syncReplicationChanges handles non-replicating cases internally via nil checks. - replicationOps, err, requeueTime := r.syncReplicationChanges(ctx, currentCnpgCluster, desiredCnpgCluster, documentdb, replicationContext) - if err != nil { - logger.Error(err, "Failed to build replication patch ops") - } - if requeueTime > 0 { - return ctrl.Result{RequeueAfter: requeueTime}, nil - } - - // Sync all CNPG Cluster changes in one atomic patch (images + plugins + replication) - if err := cnpg.SyncCnpgCluster(ctx, r.Client, currentCnpgCluster, desiredCnpgCluster, replicationOps); err != nil { - logger.Error(err, "Failed to sync CNPG Cluster spec") - return ctrl.Result{RequeueAfter: RequeueAfterShort}, nil - } - - // Delete the OTel ConfigMap only after the CNPG Cluster spec no longer - // references it. If cluster sync fails, retaining it keeps the existing - // sidecar configuration functional and allows a subsequent reconcile to retry. - if !monitoringEnabled { - if err := r.deleteOtelConfigMap(ctx, documentdb.Name, req.Namespace); err != nil { - logger.Error(err, "Failed to clean up OTel ConfigMap") - return ctrl.Result{RequeueAfter: RequeueAfterShort}, nil - } - } + return r.engine().Reconcile(ctx, req) +} - if slices.Contains(currentCnpgCluster.Status.InstancesStatus[cnpgv1.PodHealthy], currentCnpgCluster.Status.CurrentPrimary) && replicationContext.IsPrimary() { - if role := intent.Product.Postgres.ReplicationAdminRole; role != "" { - // Check if permissions have already been granted - output, err := r.SQLExecutor(ctx, currentCnpgCluster, cnpg.ReplicationRoleCheckSQL(role)) - if err != nil { - logger.Error(err, "Failed to check if permissions already granted") - return ctrl.Result{RequeueAfter: RequeueAfterLong}, nil - } +// documentDBReplication wires DocumentDB multi-region replication into the engine. +type documentDBReplication struct{ r *DocumentDBReconciler } - if !strings.Contains(output, "(1 row)") { - if _, err := r.SQLExecutor(ctx, currentCnpgCluster, cnpg.ReplicationRoleGrantSQL(role)); err != nil { - logger.Error(err, "Failed to grant permissions to streaming_replica") - return ctrl.Result{RequeueAfter: RequeueAfterShort}, nil - } - } - } - } +func (h documentDBReplication) Context(ctx context.Context, documentdb *dbpreview.DocumentDB) (*util.ReplicationContext, error) { + return util.GetReplicationContext(ctx, h.r.Client, *documentdb) +} - if replicationContext.IsPrimary() && documentdb.Status.TargetPrimary != "" { - // If these are different, we need to initiate a failover - if documentdb.Status.TargetPrimary != currentCnpgCluster.Status.TargetPrimary { +func (h documentDBReplication) AddToClusterSpec(ctx context.Context, documentdb *dbpreview.DocumentDB, rc *util.ReplicationContext, desired *cnpgv1.Cluster) error { + return h.r.AddClusterReplicationToClusterSpec(ctx, documentdb, rc, desired) +} - if err = Promote(ctx, r.Client, currentCnpgCluster.Namespace, currentCnpgCluster.Name, documentdb.Status.TargetPrimary); err != nil { - logger.Error(err, "Failed to promote standby cluster to primary") - return ctrl.Result{RequeueAfter: RequeueAfterShort}, nil - } - } else if documentdb.Status.TargetPrimary != documentdb.Status.LocalPrimary && - documentdb.Status.TargetPrimary == currentCnpgCluster.Status.CurrentPrimary { +func (h documentDBReplication) SyncChanges(ctx context.Context, current, desired *cnpgv1.Cluster, documentdb *dbpreview.DocumentDB, rc *util.ReplicationContext) ([]cnpg.JSONPatch, error, time.Duration) { + return h.r.syncReplicationChanges(ctx, current, desired, documentdb, rc) +} - logger.Info("Marking failover as complete") - documentdb.Status.LocalPrimary = currentCnpgCluster.Status.CurrentPrimary - if err := r.Status().Update(ctx, documentdb); err != nil { - logger.Error(err, "Failed to update DocumentDB status") - return ctrl.Result{RequeueAfter: RequeueAfterShort}, nil - } - } +func (h documentDBReplication) ReconcilePrimary(ctx context.Context, documentdb *dbpreview.DocumentDB, rc *util.ReplicationContext, currentCnpgCluster *cnpgv1.Cluster) *ctrl.Result { + logger := log.FromContext(ctx) + if !rc.IsPrimary() || documentdb.Status.TargetPrimary == "" { + return nil } - - // Update DocumentDB status with CNPG Cluster phase and connection string - if err := r.Client.Get(ctx, types.NamespacedName{Name: desiredCnpgCluster.Name, Namespace: req.Namespace}, currentCnpgCluster); err == nil { - statusChanged := false - - // Update phase status from CNPG Cluster - if currentCnpgCluster.Status.Phase != "" && documentdb.Status.Status != currentCnpgCluster.Status.Phase { - documentdb.Status.Status = currentCnpgCluster.Status.Phase - statusChanged = true - } - - // Update connection string if primary and service IP available - if replicationContext.IsPrimary() && documentDbServiceIp != "" { - trustTLS := documentdb.Status.TLS != nil && documentdb.Status.TLS.Ready - newConnStr := util.GenerateConnectionString(documentdb, documentDbServiceIp, trustTLS) - if documentdb.Status.ConnectionString != newConnStr { - documentdb.Status.ConnectionString = newConnStr - statusChanged = true - } + // If these are different, we need to initiate a failover + if documentdb.Status.TargetPrimary != currentCnpgCluster.Status.TargetPrimary { + if err := Promote(ctx, h.r.Client, currentCnpgCluster.Namespace, currentCnpgCluster.Name, documentdb.Status.TargetPrimary); err != nil { + logger.Error(err, "Failed to promote standby cluster to primary") + return &ctrl.Result{RequeueAfter: RequeueAfterShort} } + } else if documentdb.Status.TargetPrimary != documentdb.Status.LocalPrimary && + documentdb.Status.TargetPrimary == currentCnpgCluster.Status.CurrentPrimary { - if statusChanged { - if err := r.Status().Update(ctx, documentdb); err != nil { - logger.Error(err, "Failed to update DocumentDB status") - } + logger.Info("Marking failover as complete") + documentdb.Status.LocalPrimary = currentCnpgCluster.Status.CurrentPrimary + if err := h.r.Status().Update(ctx, documentdb); err != nil { + logger.Error(err, "Failed to update DocumentDB status") + return &ctrl.Result{RequeueAfter: RequeueAfterShort} } } + return nil +} +func (h documentDBReplication) ReconcileNetworking(ctx context.Context, documentdb *dbpreview.DocumentDB, rc *util.ReplicationContext) *ctrl.Result { // Check for fleet-networking issues and attempt to remediate - if replicationContext.IsAzureFleetNetworking() { - deleted, imports, err := r.CleanupMismatchedServiceImports(ctx, documentdb.Namespace, replicationContext) - if err != nil { - log.Log.Error(err, "Failed to cleanup ServiceImports") - return ctrl.Result{RequeueAfter: RequeueAfterShort}, nil - } - if deleted { - log.Log.Info("Deleted mismatched ServiceImports; requeuing to allow for proper recreation") - return ctrl.Result{RequeueAfter: RequeueAfterShort}, nil - } - reconciled, err := r.ForceReconcileInternalServiceExports(ctx, documentdb.Namespace, replicationContext, imports) - if err != nil { - log.Log.Error(err, "Failed to force reconcile InternalServiceExports") - return ctrl.Result{RequeueAfter: RequeueAfterShort}, nil - } - if reconciled { - log.Log.Info("Annotated InternalServiceExports for reconciliation; requeuing to allow fleet-networking to recreate ServiceImports") - return ctrl.Result{RequeueAfter: RequeueAfterLong}, nil - } + if !rc.IsAzureFleetNetworking() { + return nil } - - // Check if documentdb extension needs ALTER EXTENSION UPDATE - if err := r.handleExtensionUpgrade(ctx, currentCnpgCluster, documentdb); err != nil { - logger.Error(err, "Failed to handle DocumentDB extension upgrade") - return ctrl.Result{RequeueAfter: RequeueAfterShort}, nil + deleted, imports, err := h.r.CleanupMismatchedServiceImports(ctx, documentdb.Namespace, rc) + if err != nil { + log.Log.Error(err, "Failed to cleanup ServiceImports") + return &ctrl.Result{RequeueAfter: RequeueAfterShort} } - - // Don't requeue again unless there is a change - return ctrl.Result{}, nil -} - -// cleanupResources handles the cleanup of associated resources when a DocumentDB resource is not found -func (r *DocumentDBReconciler) cleanupResources(ctx context.Context, req ctrl.Request) error { - log := log.FromContext(ctx) - - // Cleanup ServiceAccount, Role and RoleBinding - if err := util.DeleteRoleBinding(ctx, r.Client, req.Name, req.Namespace); err != nil { - log.Error(err, "Failed to delete RoleBinding during cleanup", "RoleBindingName", req.Name) - // Continue with other cleanup even if this fails + if deleted { + log.Log.Info("Deleted mismatched ServiceImports; requeuing to allow for proper recreation") + return &ctrl.Result{RequeueAfter: RequeueAfterShort} } - - if err := util.DeleteServiceAccount(ctx, r.Client, req.Name, req.Namespace); err != nil { - log.Error(err, "Failed to delete ServiceAccount during cleanup", "ServiceAccountName", req.Name) - // Continue with other cleanup even if this fails + reconciled, err := h.r.ForceReconcileInternalServiceExports(ctx, documentdb.Namespace, rc, imports) + if err != nil { + log.Log.Error(err, "Failed to force reconcile InternalServiceExports") + return &ctrl.Result{RequeueAfter: RequeueAfterShort} } - - if err := util.DeleteRole(ctx, r.Client, req.Name, req.Namespace); err != nil { - log.Error(err, "Failed to delete Role during cleanup", "RoleName", req.Name) - // Continue with other cleanup even if this fails + if reconciled { + log.Log.Info("Annotated InternalServiceExports for reconciliation; requeuing to allow fleet-networking to recreate ServiceImports") + return &ctrl.Result{RequeueAfter: RequeueAfterLong} } - - log.Info("Cleanup process completed", "DocumentDB", req.Name, "Namespace", req.Namespace) return nil } -// reconcileFinalizer handles the finalizer lifecycle: -// - If resource is being deleted: process deletion and remove finalizer -// - If finalizer is missing: add it -// - Otherwise: continue with normal reconciliation -// -// Returns (done, result, error) where done=true means reconciliation should stop. +// reconcileFinalizer handles the finalizer lifecycle; see engine.ClusterReconciler.ReconcileFinalizer. func (r *DocumentDBReconciler) reconcileFinalizer(ctx context.Context, documentdb *dbpreview.DocumentDB) (bool, ctrl.Result, error) { - logger := log.FromContext(ctx) - - // Handle deletion - if !documentdb.ObjectMeta.DeletionTimestamp.IsZero() { - if !controllerutil.ContainsFinalizer(documentdb, documentDBFinalizer) { - // Finalizer already removed, nothing to do - return true, ctrl.Result{}, nil - } - - // Check if PVs will be retained and emit warning - if documentdb.ShouldWarnAboutRetainedPVs() { - if err := r.emitPVRetentionWarning(ctx, documentdb); err != nil { - // Log but don't block deletion - logger.Error(err, "Failed to emit PV retention warning, continuing with deletion") - } - } - - // Remove finalizer to allow deletion to proceed - controllerutil.RemoveFinalizer(documentdb, documentDBFinalizer) - if err := r.Update(ctx, documentdb); err != nil { - logger.Error(err, "Failed to remove finalizer") - return true, ctrl.Result{}, err - } - - logger.Info("Removed finalizer, deletion will proceed") - return true, ctrl.Result{}, nil - } - - // Ensure finalizer is present for non-deleting resources - if !controllerutil.ContainsFinalizer(documentdb, documentDBFinalizer) { - controllerutil.AddFinalizer(documentdb, documentDBFinalizer) - if err := r.Update(ctx, documentdb); err != nil { - logger.Error(err, "Failed to add finalizer") - return true, ctrl.Result{}, err - } - logger.Info("Added finalizer to DocumentDB") - return true, ctrl.Result{Requeue: true}, nil - } - - // Finalizer is present and resource is not being deleted, continue reconciliation - return false, ctrl.Result{}, nil + return r.engine().ReconcileFinalizer(ctx, documentdb) } // emitPVRetentionWarning emits a warning event listing PVs that will be retained after deletion func (r *DocumentDBReconciler) emitPVRetentionWarning(ctx context.Context, documentdb *dbpreview.DocumentDB) error { - logger := log.FromContext(ctx) - - if r.Recorder == nil { - logger.Info("Event recorder not configured, skipping PV retention warning") - return nil - } - - // Find PVs associated with this DocumentDB - pvNames, err := r.findPVsForDocumentDB(ctx, documentdb) - if err != nil { - return fmt.Errorf("failed to find PVs: %w", err) - } - - if len(pvNames) == 0 { - logger.V(1).Info("No PVs found for DocumentDB") - return nil - } - - // Emit actionable warning event - message := fmt.Sprintf( - "PersistentVolumes retained after cluster deletion (policy=Retain). "+ - "To delete when no longer needed: kubectl delete pv %s", - strings.Join(pvNames, " ")) - - r.Recorder.Event(documentdb, corev1.EventTypeWarning, "PVsRetained", message) - logger.Info("Emitted PV retention warning", "pvCount", len(pvNames), "pvNames", pvNames) - - return nil + return r.engine().EmitPVRetentionWarning(ctx, documentdb) } -// findPVsForDocumentDB finds all PV names associated with a DocumentDB cluster. -// Uses the documentdb.io/cluster and documentdb.io/namespace labels on PVs, which is set by the PV controller. -// This works correctly in both single and multi-cluster scenarios where CNPG -// cluster names may differ from the DocumentDB name. +// findPVsForDocumentDB finds all PV names associated with a DocumentDB cluster +// through the documentdb.io/cluster and documentdb.io/namespace PV labels. func (r *DocumentDBReconciler) findPVsForDocumentDB(ctx context.Context, documentdb *dbpreview.DocumentDB) ([]string, error) { - pvList := &corev1.PersistentVolumeList{} - if err := r.List(ctx, pvList, - client.MatchingLabels{ - util.LabelCluster: documentdb.Name, - util.LabelNamespace: documentdb.Namespace, - }, - ); err != nil { - return nil, err - } - - pvNames := make([]string, 0, len(pvList.Items)) - for _, pv := range pvList.Items { - pvNames = append(pvNames, pv.Name) - } - - return pvNames, nil + return r.engine().FindPVNames(ctx, documentdb) } func (r *DocumentDBReconciler) EnsureServiceAccountRoleAndRoleBinding(ctx context.Context, documentdb *dbpreview.DocumentDB, namespace string) error { - log := log.FromContext(ctx) - - rules := []rbacv1.PolicyRule{ - { - APIGroups: []string{""}, - Resources: []string{"pods", "services", "endpoints"}, - Verbs: []string{"get", "list", "watch", "create", "update", "patch", "delete"}, - }, - } - - // Create Role - if err := util.CreateRole(ctx, r.Client, documentdb.Name, namespace, rules); err != nil { - log.Error(err, "Failed to create Role for DocumentDB", "DocumentDB.Name", documentdb.Name, "Namespace", namespace) - return err - } - - // Create ServiceAccount - if err := util.CreateServiceAccount(ctx, r.Client, documentdb.Name, namespace); err != nil { - log.Error(err, "Failed to create ServiceAccount for DocumentDB", "DocumentDB.Name", documentdb.Name, "Namespace", namespace) - return err - } - - // Create RoleBinding - if err := util.CreateRoleBinding(ctx, r.Client, documentdb.Name, namespace); err != nil { - log.Error(err, "Failed to create RoleBinding for DocumentDB", "DocumentDB.Name", documentdb.Name, "Namespace", namespace) - return err - } - - return nil -} - -// If you ever have another state from the cluster that you want to trigger on, add it here -func clusterInstanceStatusChangedPredicate() predicate.Predicate { - return predicate.Funcs{ - UpdateFunc: func(e event.UpdateEvent) bool { - oldCluster, ok := e.ObjectOld.(*cnpgv1.Cluster) - if !ok { - return true - } - newCluster, ok := e.ObjectNew.(*cnpgv1.Cluster) - if !ok { - return true - } - // Trigger on healthy instances change OR phase change - return !slices.Equal(oldCluster.Status.InstancesStatus[cnpgv1.PodHealthy], newCluster.Status.InstancesStatus[cnpgv1.PodHealthy]) || - oldCluster.Status.Phase != newCluster.Status.Phase - }, - } -} - -// documentDBServicePredicate returns a predicate that only triggers reconciliation -// for services created by the DocumentDB operator (with the documentdb-service- prefix) -func documentDBServicePredicate() predicate.Predicate { - return predicate.Funcs{ - CreateFunc: func(e event.CreateEvent) bool { - return strings.HasPrefix(e.Object.GetName(), util.DOCUMENTDB_SERVICE_PREFIX) - }, - UpdateFunc: func(e event.UpdateEvent) bool { - return strings.HasPrefix(e.ObjectNew.GetName(), util.DOCUMENTDB_SERVICE_PREFIX) - }, - DeleteFunc: func(e event.DeleteEvent) bool { - return strings.HasPrefix(e.Object.GetName(), util.DOCUMENTDB_SERVICE_PREFIX) - }, - GenericFunc: func(e event.GenericEvent) bool { - return strings.HasPrefix(e.Object.GetName(), util.DOCUMENTDB_SERVICE_PREFIX) - }, - } + return r.engine().EnsureRBAC(ctx, documentdb, namespace) } // SetupWithManager sets up the controller with the Manager. @@ -527,16 +171,12 @@ func (r *DocumentDBReconciler) SetupWithManager(mgr ctrl.Manager) error { } if r.SQLExecutor == nil { - r.SQLExecutor = r.executeSQLCommand + r.SQLExecutor = engine.NewPodSQLExecutor(r.Client, r.Clientset, r.Config) } - return ctrl.NewControllerManagedBy(mgr). - For(&dbpreview.DocumentDB{}). - Owns(&corev1.Service{}, builder.WithPredicates(documentDBServicePredicate())). - Owns(&cnpgv1.Cluster{}, builder.WithPredicates(clusterInstanceStatusChangedPredicate())). + return r.engine().ControllerBuilder(mgr, "documentdb-controller"). Owns(&cnpgv1.Publication{}). Owns(&cnpgv1.Subscription{}). - Named("documentdb-controller"). Complete(r) } @@ -587,476 +227,34 @@ func Promote(ctx context.Context, cli client.Client, return nil } -// executeSQLCommand executes SQL commands directly in the postgres container of a running pod -func (r *DocumentDBReconciler) executeSQLCommand(ctx context.Context, cluster *cnpgv1.Cluster, sqlCommand string) (string, error) { - logger := log.FromContext(ctx) - - var targetPod corev1.Pod - if err := r.Client.Get(ctx, types.NamespacedName{Name: cluster.Status.CurrentPrimary, Namespace: cluster.Namespace}, &targetPod); err != nil { - return "", fmt.Errorf("failed to get primary pod: %w", err) - } - - // Execute psql command in the postgres container - cmd := []string{ - "psql", - "-U", "postgres", - "-d", "postgres", - "-c", sqlCommand, - } - - req := r.Clientset.CoreV1().RESTClient().Post(). - Resource("pods"). - Name(targetPod.Name). - Namespace(cluster.Namespace). - SubResource("exec"). - VersionedParams(&corev1.PodExecOptions{ - Container: "postgres", - Command: cmd, - Stdin: false, - Stdout: true, - Stderr: true, - TTY: false, - }, scheme.ParameterCodec) - - exec, err := remotecommand.NewSPDYExecutor(r.Config, "POST", req.URL()) - if err != nil { - return "", fmt.Errorf("failed to create executor: %w", err) - } - - var stdout, stderr bytes.Buffer - err = exec.StreamWithContext(ctx, remotecommand.StreamOptions{ - Stdout: &stdout, - Stderr: &stderr, - }) - - if err != nil { - logger.Error(err, "Failed to execute SQL command", - "stdout", stdout.String(), - "stderr", stderr.String()) - return "", fmt.Errorf("failed to execute command: %w (stderr: %s)", err, stderr.String()) - } - - if stderr.Len() > 0 && !strings.Contains(stderr.String(), "GRANT") { - logger.Info("SQL command executed with warnings", "stderr", stderr.String()) - } - - return stdout.String(), nil -} - -// reconcilePVRecovery handles recovery from a retained PersistentVolume. -// -// CNPG only supports recovery from PVC (via VolumeSnapshots.Storage with Kind: PersistentVolumeClaim), -// not directly from PV. To bridge this gap, we create a temporary PVC that binds to the retained PV -// via spec.volumeName. CNPG then clones the data from this temp PVC to new cluster PVCs. -// After recovery completes (cluster healthy), we delete the temp PVC to release the source PV -// back to the user for manual cleanup or reuse. -// -// Flow: -// - If no PV recovery configured, return immediately -// - If CNPG exists and healthy, delete temp PVC (recovery complete) -// - If CNPG doesn't exist, validate PV and create temp PVC bound to it +// reconcilePVRecovery handles recovery from a retained PersistentVolume; see +// engine.ClusterReconciler.ReconcilePVRecovery. func (r *DocumentDBReconciler) reconcilePVRecovery(ctx context.Context, documentdb *dbpreview.DocumentDB, namespace, cnpgClusterName string) (ctrl.Result, error) { - logger := log.FromContext(ctx) - - // Skip if PV recovery is not configured - if !documentdb.IsPVRecoveryConfigured() { - return ctrl.Result{}, nil - } - - pvName := documentdb.GetPVNameForRecovery() - tempPVCName := util.TempPVCNameForPVRecovery(documentdb.Name) - - // Check if CNPG cluster exists - cnpgCluster := &cnpgv1.Cluster{} - cnpgErr := r.Get(ctx, types.NamespacedName{Name: cnpgClusterName, Namespace: namespace}, cnpgCluster) - - if cnpgErr == nil { - // CNPG exists - check if healthy and cleanup temp PVC - if cnpgCluster.Status.Phase == cnpgClusterHealthyPhase { - tempPVC := &corev1.PersistentVolumeClaim{} - if err := r.Get(ctx, types.NamespacedName{Name: tempPVCName, Namespace: namespace}, tempPVC); err == nil { - logger.Info("Deleting temp PVC after successful recovery", "pvc", tempPVCName) - if err := r.Delete(ctx, tempPVC); err != nil { - return ctrl.Result{}, fmt.Errorf("failed to delete temp PVC %s: %w", tempPVCName, err) - } - } - } - return ctrl.Result{}, nil - } - - if !errors.IsNotFound(cnpgErr) { - return ctrl.Result{}, fmt.Errorf("failed to get CNPG cluster: %w", cnpgErr) - } - - // CNPG doesn't exist - prepare temp PVC for recovery - - // Check if temp PVC already exists - tempPVC := &corev1.PersistentVolumeClaim{} - tempPVCErr := r.Get(ctx, types.NamespacedName{Name: tempPVCName, Namespace: namespace}, tempPVC) - if tempPVCErr == nil { - // Temp PVC exists, check if bound - if tempPVC.Status.Phase != corev1.ClaimBound { - logger.Info("Waiting for temp PVC to bind to PV", "pvc", tempPVCName, "phase", tempPVC.Status.Phase) - return ctrl.Result{RequeueAfter: RequeueAfterShort}, nil - } - // PVC is bound, ready to proceed with CNPG creation - return ctrl.Result{}, nil - } - - if !errors.IsNotFound(tempPVCErr) { - return ctrl.Result{}, fmt.Errorf("failed to get temp PVC %s: %w", tempPVCName, tempPVCErr) - } - - // Verify PV exists and is available - pv := &corev1.PersistentVolume{} - if err := r.Get(ctx, types.NamespacedName{Name: pvName}, pv); err != nil { - if errors.IsNotFound(err) { - return ctrl.Result{}, fmt.Errorf("PV %s not found for recovery", pvName) - } - return ctrl.Result{}, fmt.Errorf("failed to get PV %s: %w", pvName, err) - } - - if !util.IsPVAvailableForRecovery(pv) { - return ctrl.Result{}, fmt.Errorf("PV %s must be Available or Released for recovery, current phase: %s.", pvName, pv.Status.Phase) - } - - // Clear claimRef if PV is Released - if util.NeedsToClearClaimRef(pv) { - logger.Info("Clearing claimRef on Released PV", "pv", pvName) - pv.Spec.ClaimRef = nil - if err := r.Update(ctx, pv); err != nil { - return ctrl.Result{}, fmt.Errorf("failed to clear claimRef on PV %s: %w", pvName, err) - } - return ctrl.Result{RequeueAfter: RequeueAfterShort}, nil - } - - // Create temp PVC - newPVC := util.BuildTempPVCForPVRecovery(documentdb.Name, namespace, pv) - if err := controllerutil.SetControllerReference(documentdb, newPVC, r.Scheme); err != nil { - return ctrl.Result{}, fmt.Errorf("failed to set owner reference on temp PVC: %w", err) - } - - if err := r.Create(ctx, newPVC); err != nil { - return ctrl.Result{}, fmt.Errorf("failed to create temp PVC %s: %w", tempPVCName, err) - } - - logger.Info("Created temp PVC for PV recovery", "pvc", tempPVCName, "pv", pvName) - return ctrl.Result{RequeueAfter: RequeueAfterShort}, nil + return r.engine().ReconcilePVRecovery(ctx, documentdb, namespace, cnpgClusterName) } -// parseExtensionVersionsFromOutput parses the output of pg_available_extensions query -// Returns defaultVersion, installedVersion, and a boolean indicating if parsing was successful -// Expected output format: -// -// default_version | installed_version -// -----------------+------------------- -// 0.110-0 | 0.110-0 -// -// TODO: This parsing is fragile as it relies on psql tabular output format which can vary -// with locale or PostgreSQL version. Consider one of these improvements: -// - Option 1: Add -t -A flags to SQLExecutor for unaligned, tuples-only output ("0.110-0|0.110-0") -// - Option 2: Use json_build_object() in SQL and json.Unmarshal() in Go for robust parsing +// parseExtensionVersionsFromOutput parses the output of the pg_available_extensions query. func parseExtensionVersionsFromOutput(output string) (defaultVersion, installedVersion string, ok bool) { - lines := strings.Split(strings.TrimSpace(output), "\n") - if len(lines) < 3 { - return "", "", false - } - - // Parse the data row (3rd line, index 2) - dataLine := strings.TrimSpace(lines[2]) - parts := strings.Split(dataLine, "|") - if len(parts) != 2 { - return "", "", false - } - - defaultVersion = strings.TrimSpace(parts[0]) - installedVersion = strings.TrimSpace(parts[1]) - return defaultVersion, installedVersion, true + return engine.ParseExtensionVersions(output) } -// handleExtensionUpgrade handles the ALTER EXTENSION lifecycle after images have been synced -// by SyncCnpgCluster. It: -// 1. Updates DocumentDB status with the current images from the CNPG cluster -// 2. Checks if ALTER EXTENSION UPDATE is needed and runs it (with two-phase upgrade support) -// 3. Updates the DocumentDB schema version status +// handleExtensionUpgrade handles the ALTER EXTENSION lifecycle; see +// engine.ClusterReconciler.HandleExtensionUpgrade. func (r *DocumentDBReconciler) handleExtensionUpgrade(ctx context.Context, currentCluster *cnpgv1.Cluster, documentdb *dbpreview.DocumentDB) error { - logger := log.FromContext(ctx) - - // Refetch documentdb to avoid potential race conditions with status updates - if err := r.Get(ctx, types.NamespacedName{Name: documentdb.Name, Namespace: documentdb.Namespace}, documentdb); err != nil { - return fmt.Errorf("failed to refetch DocumentDB resource: %w", err) - } - - // Update image status fields to reflect what's currently applied - if err := r.updateImageStatus(ctx, documentdb, currentCluster); err != nil { - logger.Error(err, "Failed to update image status") - } - - // CNPG will trigger a rolling restart when images change. Wait for pods to become healthy - // before running ALTER EXTENSION. - if !slices.Contains(currentCluster.Status.InstancesStatus[cnpgv1.PodHealthy], currentCluster.Status.CurrentPrimary) { - logger.Info("Current primary pod is not healthy; skipping DocumentDB extension upgrade") - return nil - } - - // Check if ALTER EXTENSION UPDATE is needed - extension := documentDBExtensionName() - output, err := r.SQLExecutor(ctx, currentCluster, cnpg.ExtensionVersionSQL(extension)) - if err != nil { - return fmt.Errorf("failed to check %s extension versions: %w", extension, err) - } - - defaultVersion, installedVersion, ok := parseExtensionVersionsFromOutput(output) - if !ok { - logger.Info("DocumentDB extension not found or not installed yet", "output", output) - return nil - } - - if installedVersion == "" { - logger.Info("DocumentDB extension is not installed yet") - return nil - } - - // Update DocumentDB schema version in status (even if no upgrade needed) - // Convert from pg_available_extensions format ("0.110-0") to semver ("0.110.0") - installedSemver := util.ExtensionVersionToSemver(installedVersion) - if documentdb.Status.SchemaVersion != installedSemver { - // Re-fetch to get latest resourceVersion before status update - if err := r.Get(ctx, types.NamespacedName{Name: documentdb.Name, Namespace: documentdb.Namespace}, documentdb); err != nil { - return fmt.Errorf("failed to refetch DocumentDB before schema version update: %w", err) - } - documentdb.Status.SchemaVersion = installedSemver - if err := r.Status().Update(ctx, documentdb); err != nil { - logger.Error(err, "Failed to update DocumentDB status with schema version") - return fmt.Errorf("failed to update DocumentDB status with schema version: %w", err) - } - } - - // If versions match, no upgrade needed - if defaultVersion == installedVersion { - logger.V(1).Info("DocumentDB extension is up to date", "version", installedVersion) - return nil - } - - // Rollback detection — check if the new binary is older than the installed schema. - // Note: Image rollback below installed schema version is also blocked by the validating - // webhook at admission time. This is a defense-in-depth guard. - cmp, err := util.CompareExtensionVersions(defaultVersion, installedVersion) - if err != nil { - logger.Error(err, "Failed to compare extension versions, skipping ALTER EXTENSION as a safety measure", - "defaultVersion", defaultVersion, - "installedVersion", installedVersion) - return nil - } - - if cmp < 0 { - // ALTER EXTENSION UPDATE would attempt an unsupported downgrade. Skip it and warn the user. - msg := fmt.Sprintf( - "Extension rollback detected: binary offers version %s but schema is at %s. "+ - "ALTER EXTENSION UPDATE skipped — DocumentDB does not provide downgrade scripts. "+ - "The cluster will run with the older binary against the newer schema, which may cause issues. "+ - "To resolve, update the extension image to a version that matches or exceeds %s.", - defaultVersion, installedVersion, installedVersion) - logger.Info(msg) - if r.Recorder != nil { - r.Recorder.Event(documentdb, corev1.EventTypeWarning, "ExtensionRollback", msg) - } - return nil - } - - // Determine schema target based on spec.schemaVersion (two-phase upgrade logic) - schemaTarget, updateSQL := r.determineSchemaTarget(ctx, documentdb, defaultVersion, installedVersion) - if schemaTarget == "" { - // Two-phase mode or validation failure — do not run ALTER EXTENSION - return nil - } - - // Run ALTER EXTENSION to upgrade - logger.Info("Upgrading DocumentDB extension", - "fromVersion", installedVersion, - "toVersion", schemaTarget) - - if _, err := r.SQLExecutor(ctx, currentCluster, updateSQL); err != nil { - return fmt.Errorf("failed to run ALTER EXTENSION %s UPDATE: %w", extension, err) - } - - logger.Info("Successfully upgraded DocumentDB extension", - "fromVersion", installedVersion, - "toVersion", schemaTarget) - - // Update DocumentDB schema version in status after upgrade - if err := r.Get(ctx, types.NamespacedName{Name: documentdb.Name, Namespace: documentdb.Namespace}, documentdb); err != nil { - return fmt.Errorf("failed to refetch DocumentDB after schema upgrade: %w", err) - } - documentdb.Status.SchemaVersion = util.ExtensionVersionToSemver(schemaTarget) - if err := r.Status().Update(ctx, documentdb); err != nil { - logger.Error(err, "Failed to update DocumentDB status after schema upgrade") - return fmt.Errorf("failed to update DocumentDB status after schema upgrade: %w", err) - } - - return nil -} - -// determineSchemaTarget decides the target schema version based on spec.schemaVersion. -// Returns the target version (in pg_available_extensions format) and the SQL to execute. -// Returns ("", "") if no schema update should run (two-phase mode or validation failure). -func (r *DocumentDBReconciler) determineSchemaTarget( - ctx context.Context, - documentdb *dbpreview.DocumentDB, - binaryVersion string, - installedVersion string, -) (string, string) { - logger := log.FromContext(ctx) - specSchemaVersion := documentdb.Spec.SchemaVersion - - switch { - case specSchemaVersion == "": - // Two-phase mode: schema stays at current version until user explicitly sets schemaVersion. - // Use V(1) to avoid log spam — this fires every reconcile while upgrade is pending. - logger.V(1).Info("Schema update available but not requested (two-phase mode). "+ - "Set spec.schemaVersion to finalize the upgrade.", - "binaryVersion", binaryVersion, - "installedVersion", installedVersion) - // Only emit the event when the binary is actually ahead of the installed schema - // (avoids firing on new clusters where status.schemaVersion is empty). - // Kubernetes deduplicates events with the same reason+message, but skipping - // entirely is cleaner. - if r.Recorder != nil && binaryVersion != installedVersion { - msg := fmt.Sprintf( - "Schema update available: binary version is %s, schema is at %s. "+ - "Set spec.schemaVersion to %q or \"auto\" to finalize the upgrade.", - util.ExtensionVersionToSemver(binaryVersion), - util.ExtensionVersionToSemver(installedVersion), - util.ExtensionVersionToSemver(binaryVersion)) - r.Recorder.Event(documentdb, corev1.EventTypeNormal, "SchemaUpdateAvailable", msg) - } - return "", "" - - case specSchemaVersion == "auto": - // Auto-finalize: update schema to match binary version - return binaryVersion, cnpg.ExtensionUpdateSQL(documentDBExtensionName(), "") - - default: - // Explicit version: update to that specific version. - targetPgVersion := util.SemverToExtensionVersion(specSchemaVersion) - - // Guard: target must not exceed binary version. - // Note: schemaVersion <= binary version is enforced by the validating webhook at - // admission time. This is a defense-in-depth guard. - targetCmp, err := util.CompareExtensionVersions(targetPgVersion, binaryVersion) - if err != nil { - logger.Error(err, "Failed to compare target schema version with binary version", - "targetVersion", specSchemaVersion, - "binaryVersion", binaryVersion) - return "", "" - } - if targetCmp > 0 { - logger.Info("Skipping schema update: schemaVersion exceeds binary version (should have been rejected by webhook)", - "schemaVersion", specSchemaVersion, - "binaryVersion", util.ExtensionVersionToSemver(binaryVersion)) - return "", "" - } - - // Check: target must be > installed version (otherwise no-op) - installedCmp, err := util.CompareExtensionVersions(targetPgVersion, installedVersion) - if err != nil { - logger.Error(err, "Failed to compare target schema version with installed version", - "targetVersion", specSchemaVersion, - "installedVersion", installedVersion) - return "", "" - } - if installedCmp <= 0 { - logger.V(1).Info("Schema already at or beyond requested version", - "requestedVersion", specSchemaVersion, - "installedVersion", installedVersion) - return "", "" - } - - // Safety: targetPgVersion is derived from spec.schemaVersion via SemverToExtensionVersion, - // which produces a "Major.Minor-Patch" string (e.g., "0.112-0"). The input is validated - // by the CRD schema regex pattern and the validating webhook's version parsing, so it - // contains only digits, dots, and hyphens — no SQL injection risk. - return targetPgVersion, cnpg.ExtensionUpdateSQL(documentDBExtensionName(), targetPgVersion) - } + return r.engine().HandleExtensionUpgrade(ctx, currentCluster, documentdb) } -// updateImageStatus reads the current extension and gateway images from the CNPG cluster -// and persists them into the DocumentDB status fields. This is a no-op if both fields -// are already up to date. +// updateImageStatus persists the current extension and gateway images into the DocumentDB status. func (r *DocumentDBReconciler) updateImageStatus(ctx context.Context, documentdb *dbpreview.DocumentDB, cluster *cnpgv1.Cluster) error { - currentExtImage := cnpg.ExtensionImage(cluster, documentDBExtensionName()) - currentGwImage := cnpg.GatewayImage(cluster) - - // Only update if something changed - if documentdb.Status.DocumentDBImage == currentExtImage && documentdb.Status.GatewayImage == currentGwImage { - return nil - } - - // Re-fetch to get latest resourceVersion before status update - if err := r.Get(ctx, types.NamespacedName{Name: documentdb.Name, Namespace: documentdb.Namespace}, documentdb); err != nil { - return fmt.Errorf("failed to refetch DocumentDB before image status update: %w", err) - } - documentdb.Status.DocumentDBImage = currentExtImage - documentdb.Status.GatewayImage = currentGwImage - if err := r.Status().Update(ctx, documentdb); err != nil { - return fmt.Errorf("failed to update DocumentDB image status: %w", err) - } - return nil + return r.engine().UpdateImageStatus(ctx, documentdb, cluster) } // reconcileOtelConfigMap ensures the OTel Collector base config ConfigMap exists and is up-to-date. func (r *DocumentDBReconciler) reconcileOtelConfigMap(ctx context.Context, documentdb *dbpreview.DocumentDB, namespace string) error { - logger := log.FromContext(ctx) - cmName := otelcfg.ConfigMapName(documentdb.Name) - - cm := &corev1.ConfigMap{} - cm.Name = cmName - cm.Namespace = namespace - - result, err := controllerutil.CreateOrUpdate(ctx, r.Client, cm, func() error { - // Set owner reference so the ConfigMap is garbage-collected with the DocumentDB CR. - if err := controllerutil.SetControllerReference(documentdb, cm, r.Scheme); err != nil { - return fmt.Errorf("failed to set owner reference: %w", err) - } - - configData, err := otelcfg.GenerateConfigMapData(documentdb.Name, namespace, documentdb.Spec.Monitoring) - if err != nil { - return fmt.Errorf("failed to generate OTel config: %w", err) - } - cm.Data = configData - return nil - }) - if err != nil { - return fmt.Errorf("failed to reconcile OTel ConfigMap %s: %w", cmName, err) - } - if result != controllerutil.OperationResultNone { - logger.Info("OTel ConfigMap reconciled", "name", cmName, "operation", result) - } - return nil + return r.engine().ReconcileOtelConfigMap(ctx, documentdb, namespace) } // deleteOtelConfigMap removes the OTel ConfigMap when monitoring is no longer configured. func (r *DocumentDBReconciler) deleteOtelConfigMap(ctx context.Context, clusterName, namespace string) error { - logger := log.FromContext(ctx) - cmName := otelcfg.ConfigMapName(clusterName) - - cm := &corev1.ConfigMap{} - cm.Name = cmName - cm.Namespace = namespace - - err := r.Client.Delete(ctx, cm) - if err != nil { - if errors.IsNotFound(err) { - return nil - } - return fmt.Errorf("failed to delete OTel ConfigMap %s: %w", cmName, err) - } - logger.Info("OTel ConfigMap deleted", "name", cmName) - return nil -} - -// documentDBExtensionName is the PostgreSQL extension whose lifecycle this -// reconciler manages, taken from the DocumentDB product profile. -func documentDBExtensionName() string { - return product.DocumentDBProfile().Postgres.Extension.Name + return r.engine().DeleteOtelConfigMap(ctx, clusterName, namespace) } diff --git a/operator/src/internal/engine/cluster_reconciler.go b/operator/src/internal/engine/cluster_reconciler.go new file mode 100644 index 00000000..01e401bb --- /dev/null +++ b/operator/src/internal/engine/cluster_reconciler.go @@ -0,0 +1,574 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +// Package engine contains the product-neutral reconcilers. A product supplies a +// product.ClusterAdapter for its custom resource and gets the full single-region +// lifecycle: gateway Service, RBAC, CNPG cluster render and sync, PV retention +// and recovery, OTel collector config, extension upgrade, status, TLS +// certificates, persistent volumes, and backups. Multi-region replication is an +// optional extension point (ReplicationHooks). +package engine + +import ( + "context" + "fmt" + "slices" + "strings" + "sync" + "time" + + cnpgv1 "github.com/cloudnative-pg/cloudnative-pg/api/v1" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/tools/record" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/builder" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/log" + + cnpg "github.com/documentdb/documentdb-operator/internal/cnpg" + otelcfg "github.com/documentdb/documentdb-operator/internal/otel" + "github.com/documentdb/documentdb-operator/internal/product" + util "github.com/documentdb/documentdb-operator/internal/utils" +) + +const ( + RequeueAfterShort = 10 * time.Second + RequeueAfterLong = 30 * time.Second + + // CNPGClusterHealthyPhase is the CNPG cluster status phase indicating a healthy cluster. + // This value is from CNPG's internal status representation. + CNPGClusterHealthyPhase = "Cluster in healthy state" +) + +// SQLExecutor executes a SQL command against a CNPG cluster's primary and +// returns psql's tabular output. +type SQLExecutor func(ctx context.Context, cluster *cnpgv1.Cluster, sqlCommand string) (string, error) + +// ReplicationHooks is the optional multi-region extension point of the cluster +// reconciler. Without hooks the reconciler runs every cluster as a single-region +// primary. +type ReplicationHooks[T client.Object] interface { + // Context resolves the replication role of obj in its topology. + Context(ctx context.Context, obj T) (*util.ReplicationContext, error) + // AddToClusterSpec adds replication settings to the desired CNPG cluster; + // called only when the context is replicating. + AddToClusterSpec(ctx context.Context, obj T, rc *util.ReplicationContext, desired *cnpgv1.Cluster) error + // SyncChanges returns extra CNPG patch operations for replication changes, + // or a positive requeue delay to wait before syncing. + SyncChanges(ctx context.Context, current, desired *cnpgv1.Cluster, obj T, rc *util.ReplicationContext) ([]cnpg.JSONPatch, error, time.Duration) + // ReconcilePrimary handles failover/promotion after the cluster is synced. + // A non-nil result stops reconciliation and is returned. + ReconcilePrimary(ctx context.Context, obj T, rc *util.ReplicationContext, current *cnpgv1.Cluster) *ctrl.Result + // ReconcileNetworking remediates cross-cluster networking after status is + // updated. A non-nil result stops reconciliation and is returned. + ReconcileNetworking(ctx context.Context, obj T, rc *util.ReplicationContext) *ctrl.Result +} + +// ClusterReconciler is the product-neutral reconciler of a product's cluster +// custom resource. +type ClusterReconciler[T client.Object] struct { + client.Client + Scheme *runtime.Scheme + Recorder record.EventRecorder + // SQLExecutor runs SQL on the CNPG primary; see NewPodSQLExecutor. + SQLExecutor SQLExecutor + // Adapter maps the product custom resource onto the neutral model. + Adapter product.ClusterAdapter[T] + // Replication is optional; nil means single-region. + Replication ReplicationHooks[T] + // Mutex, when set, serializes reconciliations. + Mutex *sync.Mutex +} + +func (r *ClusterReconciler[T]) profile() product.ProductProfile { return r.Adapter.Profile() } + +func (r *ClusterReconciler[T]) replicationContext(ctx context.Context, obj T) (*util.ReplicationContext, error) { + if r.Replication != nil { + return r.Replication.Context(ctx, obj) + } + spec := r.Adapter.Spec(obj) + return util.NewSingleRegionContext(obj.GetName(), spec.Environment, spec.StorageClass), nil +} + +// GatewayPort is the gateway port exposed by the Service and in the +// connection string: the profile port, or the GATEWAY_PORT env default. +func GatewayPort(p product.ProductProfile) int32 { + if p.Gateway.Port != 0 { + return p.Gateway.Port + } + return util.GetPortFor(util.GATEWAY_PORT) +} + +// Reconcile drives the custom resource toward its desired state. +func (r *ClusterReconciler[T]) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + if r.Mutex != nil { + r.Mutex.Lock() + defer r.Mutex.Unlock() + } + + logger := log.FromContext(ctx) + p := r.profile() + + obj := r.Adapter.New() + err := r.Get(ctx, req.NamespacedName, obj) + if err != nil { + if errors.IsNotFound(err) { + logger.Info(p.Name + " resource not found. Cleaning up associated resources.") + if err := r.Cleanup(ctx, req); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{}, nil + } + logger.Error(err, "Failed to get "+p.Name+" resource") + return ctrl.Result{}, err + } + + // Handle finalizer lifecycle (add on create, remove on delete) + if done, result, err := r.ReconcileFinalizer(ctx, obj); done || err != nil { + return result, err + } + + replicationContext, err := r.replicationContext(ctx, obj) + if err != nil { + logger.Error(err, "Failed to determine replication context") + return ctrl.Result{}, err + } + + if replicationContext.IsNotPresent() { + logger.Info(p.Name + " instance is not part of the replication setup; skipping reconciliation and deleting any present resources") + if err := r.Cleanup(ctx, req); err != nil { + return ctrl.Result{}, err + } + owner := metav1.ObjectMeta{Name: obj.GetName(), Namespace: obj.GetNamespace(), UID: obj.GetUID()} + if err := util.DeleteOwnedResources(ctx, r.Client, owner); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{}, nil + } + + spec := r.Adapter.Spec(obj) + var serviceIP string + + // Only create/manage the service if a service type is configured + if spec.ServiceType != "" { + serviceType := corev1.ServiceTypeClusterIP + if spec.ServiceType == "LoadBalancer" { + serviceType = corev1.ServiceTypeLoadBalancer // Public LoadBalancer service + } + + apiVersion, kind := obj.GetObjectKind().GroupVersionKind().ToAPIVersionAndKind() + svc := util.BuildGatewayService(util.GatewayServiceOptions{ + Name: p.Naming.ServiceName(obj.GetName()), + Namespace: req.Namespace, + ClusterName: obj.GetName(), + OwnerAPIVersion: apiVersion, + OwnerKind: kind, + OwnerUID: obj.GetUID(), + Port: GatewayPort(p), + EndpointEnabled: replicationContext.EndpointEnabled(), + Environment: replicationContext.Environment, + ServiceType: serviceType, + }) + + foundService, err := util.UpsertService(ctx, r.Client, svc) + if err != nil { + logger.Error(err, "Failed to create "+p.Name+" Service; Requeuing.") + return ctrl.Result{RequeueAfter: RequeueAfterShort}, nil + } + + serviceIP, err = util.EnsureServiceIP(ctx, foundService) + if err != nil { + logger.Info(p.Name + " Service IP not assigned, pausing until update posted.") + return ctrl.Result{}, nil + } + } + + // Ensure App ServiceAccount, Role and RoleBindings are created + if err := r.EnsureRBAC(ctx, obj, req.Namespace); err != nil { + logger.Info("Failed to create ServiceAccount, Role and RoleBinding; Requeuing.") + return ctrl.Result{RequeueAfter: RequeueAfterShort}, nil + } + + // create the CNPG Cluster + intent := r.Adapter.Intent(obj) + + currentCnpgCluster := &cnpgv1.Cluster{} + desiredCnpgCluster := cnpg.GetCnpgClusterSpecFromIntent(req, intent, obj.GetName(), replicationContext.StorageClass, replicationContext.IsPrimary(), logger) + + if r.Replication != nil && replicationContext.IsReplicating() { + err = r.Replication.AddToClusterSpec(ctx, obj, replicationContext, desiredCnpgCluster) + if err != nil { + logger.Error(err, "Failed to add physical replication features cnpg Cluster spec") + return ctrl.Result{RequeueAfter: RequeueAfterShort}, nil + } + } + + // Handle PV recovery lifecycle (create temp PVC before CNPG, cleanup after healthy) + if result, err := r.ReconcilePVRecovery(ctx, obj, req.Namespace, desiredCnpgCluster.Name); err != nil { + logger.Error(err, "Failed to reconcile PV recovery") + return result, err + } else if result.Requeue || result.RequeueAfter > 0 { + return result, nil + } + + // Reconcile OTel resources when monitoring is enabled. + // The sidecar itself is added/removed via CNPG plugin parameters; + // the operator triggers a rolling restart (via restart annotation) + // and CNPG manages the pod rollout. + monitoringEnabled := spec.MonitoringEnabled() + if monitoringEnabled { + if err := r.ReconcileOtelConfigMap(ctx, obj, req.Namespace); err != nil { + logger.Error(err, "Failed to reconcile OTel ConfigMap") + return ctrl.Result{RequeueAfter: RequeueAfterShort}, nil + } + } + + if err := r.Get(ctx, types.NamespacedName{Name: desiredCnpgCluster.Name, Namespace: req.Namespace}, currentCnpgCluster); err != nil { + if errors.IsNotFound(err) { + if err := r.Create(ctx, desiredCnpgCluster); err != nil { + logger.Error(err, "Failed to create CNPG Cluster") + return ctrl.Result{RequeueAfter: RequeueAfterShort}, nil + } + logger.Info("CNPG Cluster created successfully", "Cluster.Name", desiredCnpgCluster.Name, "Namespace", desiredCnpgCluster.Namespace) + return ctrl.Result{RequeueAfter: RequeueAfterLong}, nil + } + logger.Error(err, "Failed to get CNPG Cluster") + return ctrl.Result{RequeueAfter: RequeueAfterShort}, nil + } + + var replicationOps []cnpg.JSONPatch + if r.Replication != nil { + // Build replication patch ops (may perform side effects such as token reads and service creation). + var requeueTime time.Duration + replicationOps, err, requeueTime = r.Replication.SyncChanges(ctx, currentCnpgCluster, desiredCnpgCluster, obj, replicationContext) + if err != nil { + logger.Error(err, "Failed to build replication patch ops") + } + if requeueTime > 0 { + return ctrl.Result{RequeueAfter: requeueTime}, nil + } + } + + // Sync all CNPG Cluster changes in one atomic patch (images + plugins + replication) + if err := cnpg.SyncCnpgCluster(ctx, r.Client, currentCnpgCluster, desiredCnpgCluster, replicationOps); err != nil { + logger.Error(err, "Failed to sync CNPG Cluster spec") + return ctrl.Result{RequeueAfter: RequeueAfterShort}, nil + } + + // Delete the OTel ConfigMap only after the CNPG Cluster spec no longer + // references it. If cluster sync fails, retaining it keeps the existing + // sidecar configuration functional and allows a subsequent reconcile to retry. + if !monitoringEnabled { + if err := r.DeleteOtelConfigMap(ctx, obj.GetName(), req.Namespace); err != nil { + logger.Error(err, "Failed to clean up OTel ConfigMap") + return ctrl.Result{RequeueAfter: RequeueAfterShort}, nil + } + } + + if slices.Contains(currentCnpgCluster.Status.InstancesStatus[cnpgv1.PodHealthy], currentCnpgCluster.Status.CurrentPrimary) && replicationContext.IsPrimary() { + if role := intent.Product.Postgres.ReplicationAdminRole; role != "" { + // Check if permissions have already been granted + output, err := r.SQLExecutor(ctx, currentCnpgCluster, cnpg.ReplicationRoleCheckSQL(role)) + if err != nil { + logger.Error(err, "Failed to check if permissions already granted") + return ctrl.Result{RequeueAfter: RequeueAfterLong}, nil + } + + if !strings.Contains(output, "(1 row)") { + if _, err := r.SQLExecutor(ctx, currentCnpgCluster, cnpg.ReplicationRoleGrantSQL(role)); err != nil { + logger.Error(err, "Failed to grant permissions to streaming_replica") + return ctrl.Result{RequeueAfter: RequeueAfterShort}, nil + } + } + } + } + + if r.Replication != nil { + if result := r.Replication.ReconcilePrimary(ctx, obj, replicationContext, currentCnpgCluster); result != nil { + return *result, nil + } + } + + // Update status with CNPG Cluster phase and connection string + if err := r.Get(ctx, types.NamespacedName{Name: desiredCnpgCluster.Name, Namespace: req.Namespace}, currentCnpgCluster); err == nil { + status := r.Adapter.Status(obj) + statusChanged := false + + // Update phase status from CNPG Cluster + if currentCnpgCluster.Status.Phase != "" && status.Phase != currentCnpgCluster.Status.Phase { + status.Phase = currentCnpgCluster.Status.Phase + statusChanged = true + } + + // Update connection string if primary and service IP available + if replicationContext.IsPrimary() && serviceIP != "" { + trustTLS := status.TLS != nil && status.TLS.Ready + newConnStr := r.Adapter.ConnectionString(obj, serviceIP, GatewayPort(p), trustTLS) + if status.ConnectionString != newConnStr { + status.ConnectionString = newConnStr + statusChanged = true + } + } + + if statusChanged { + r.Adapter.SetStatus(obj, status) + if err := r.Status().Update(ctx, obj); err != nil { + logger.Error(err, "Failed to update "+p.Name+" status") + } + } + } + + if r.Replication != nil { + if result := r.Replication.ReconcileNetworking(ctx, obj, replicationContext); result != nil { + return *result, nil + } + } + + // Check if the extension needs ALTER EXTENSION UPDATE + if err := r.HandleExtensionUpgrade(ctx, currentCnpgCluster, obj); err != nil { + logger.Error(err, "Failed to handle "+p.Name+" extension upgrade") + return ctrl.Result{RequeueAfter: RequeueAfterShort}, nil + } + + // Don't requeue again unless there is a change + return ctrl.Result{}, nil +} + +// Cleanup removes the per-cluster ServiceAccount, Role and RoleBinding when the +// custom resource is gone. +func (r *ClusterReconciler[T]) Cleanup(ctx context.Context, req ctrl.Request) error { + log := log.FromContext(ctx) + + // Cleanup ServiceAccount, Role and RoleBinding + if err := util.DeleteRoleBinding(ctx, r.Client, req.Name, req.Namespace); err != nil { + log.Error(err, "Failed to delete RoleBinding during cleanup", "RoleBindingName", req.Name) + // Continue with other cleanup even if this fails + } + + if err := util.DeleteServiceAccount(ctx, r.Client, req.Name, req.Namespace); err != nil { + log.Error(err, "Failed to delete ServiceAccount during cleanup", "ServiceAccountName", req.Name) + // Continue with other cleanup even if this fails + } + + if err := util.DeleteRole(ctx, r.Client, req.Name, req.Namespace); err != nil { + log.Error(err, "Failed to delete Role during cleanup", "RoleName", req.Name) + // Continue with other cleanup even if this fails + } + + log.Info("Cleanup process completed", r.profile().Name, req.Name, "Namespace", req.Namespace) + return nil +} + +// ReconcileFinalizer handles the finalizer lifecycle: +// - If resource is being deleted: process deletion and remove finalizer +// - If finalizer is missing: add it +// - Otherwise: continue with normal reconciliation +// +// Returns (done, result, error) where done=true means reconciliation should stop. +func (r *ClusterReconciler[T]) ReconcileFinalizer(ctx context.Context, obj T) (bool, ctrl.Result, error) { + logger := log.FromContext(ctx) + finalizer := r.profile().Naming.Finalizer() + + // Handle deletion + if !obj.GetDeletionTimestamp().IsZero() { + if !controllerutil.ContainsFinalizer(obj, finalizer) { + // Finalizer already removed, nothing to do + return true, ctrl.Result{}, nil + } + + // Check if PVs will be retained and emit warning + if r.Adapter.Spec(obj).RetainsPVs() { + if err := r.EmitPVRetentionWarning(ctx, obj); err != nil { + // Log but don't block deletion + logger.Error(err, "Failed to emit PV retention warning, continuing with deletion") + } + } + + // Remove finalizer to allow deletion to proceed + controllerutil.RemoveFinalizer(obj, finalizer) + if err := r.Update(ctx, obj); err != nil { + logger.Error(err, "Failed to remove finalizer") + return true, ctrl.Result{}, err + } + + logger.Info("Removed finalizer, deletion will proceed") + return true, ctrl.Result{}, nil + } + + // Ensure finalizer is present for non-deleting resources + if !controllerutil.ContainsFinalizer(obj, finalizer) { + controllerutil.AddFinalizer(obj, finalizer) + if err := r.Update(ctx, obj); err != nil { + logger.Error(err, "Failed to add finalizer") + return true, ctrl.Result{}, err + } + logger.Info("Added finalizer to " + r.profile().Name) + return true, ctrl.Result{Requeue: true}, nil + } + + // Finalizer is present and resource is not being deleted, continue reconciliation + return false, ctrl.Result{}, nil +} + +// EmitPVRetentionWarning emits a warning event listing PVs that will be retained after deletion. +func (r *ClusterReconciler[T]) EmitPVRetentionWarning(ctx context.Context, obj T) error { + logger := log.FromContext(ctx) + + if r.Recorder == nil { + logger.Info("Event recorder not configured, skipping PV retention warning") + return nil + } + + pvNames, err := r.FindPVNames(ctx, obj) + if err != nil { + return fmt.Errorf("failed to find PVs: %w", err) + } + + if len(pvNames) == 0 { + logger.V(1).Info("No PVs found for " + r.profile().Name) + return nil + } + + // Emit actionable warning event + message := fmt.Sprintf( + "PersistentVolumes retained after cluster deletion (policy=Retain). "+ + "To delete when no longer needed: kubectl delete pv %s", + strings.Join(pvNames, " ")) + + r.Recorder.Event(obj, corev1.EventTypeWarning, "PVsRetained", message) + logger.Info("Emitted PV retention warning", "pvCount", len(pvNames), "pvNames", pvNames) + + return nil +} + +// FindPVNames finds all PV names associated with a cluster through the product +// cluster/namespace labels the PV reconciler stamps on PVs. This works in both +// single and multi-cluster scenarios where CNPG cluster names may differ from +// the custom resource name. +func (r *ClusterReconciler[T]) FindPVNames(ctx context.Context, obj T) ([]string, error) { + naming := r.profile().Naming + pvList := &corev1.PersistentVolumeList{} + if err := r.List(ctx, pvList, + client.MatchingLabels{ + naming.ClusterLabel(): obj.GetName(), + naming.NamespaceLabel(): obj.GetNamespace(), + }, + ); err != nil { + return nil, err + } + + pvNames := make([]string, 0, len(pvList.Items)) + for _, pv := range pvList.Items { + pvNames = append(pvNames, pv.Name) + } + + return pvNames, nil +} + +// EnsureRBAC creates the per-cluster Role, ServiceAccount and RoleBinding. +func (r *ClusterReconciler[T]) EnsureRBAC(ctx context.Context, obj T, namespace string) error { + log := log.FromContext(ctx) + kind := r.profile().Name + name := obj.GetName() + + rules := []rbacv1.PolicyRule{ + { + APIGroups: []string{""}, + Resources: []string{"pods", "services", "endpoints"}, + Verbs: []string{"get", "list", "watch", "create", "update", "patch", "delete"}, + }, + } + + if err := util.CreateRole(ctx, r.Client, name, namespace, rules); err != nil { + log.Error(err, "Failed to create Role for "+kind, kind+".Name", name, "Namespace", namespace) + return err + } + + if err := util.CreateServiceAccount(ctx, r.Client, name, namespace); err != nil { + log.Error(err, "Failed to create ServiceAccount for "+kind, kind+".Name", name, "Namespace", namespace) + return err + } + + if err := util.CreateRoleBinding(ctx, r.Client, name, namespace); err != nil { + log.Error(err, "Failed to create RoleBinding for "+kind, kind+".Name", name, "Namespace", namespace) + return err + } + + return nil +} + +// ReconcileOtelConfigMap ensures the OTel Collector base config ConfigMap exists and is up-to-date. +func (r *ClusterReconciler[T]) ReconcileOtelConfigMap(ctx context.Context, obj T, namespace string) error { + logger := log.FromContext(ctx) + cmName := otelcfg.ConfigMapName(obj.GetName()) + + cm := &corev1.ConfigMap{} + cm.Name = cmName + cm.Namespace = namespace + + result, err := controllerutil.CreateOrUpdate(ctx, r.Client, cm, func() error { + // Set owner reference so the ConfigMap is garbage-collected with the custom resource. + if err := controllerutil.SetControllerReference(obj, cm, r.Scheme); err != nil { + return fmt.Errorf("failed to set owner reference: %w", err) + } + + configData, err := otelcfg.GenerateConfigMapDataWithOptions(cnpg.OTelOptions(r.profile()), obj.GetName(), namespace, r.Adapter.Spec(obj).Monitoring) + if err != nil { + return fmt.Errorf("failed to generate OTel config: %w", err) + } + cm.Data = configData + return nil + }) + if err != nil { + return fmt.Errorf("failed to reconcile OTel ConfigMap %s: %w", cmName, err) + } + if result != controllerutil.OperationResultNone { + logger.Info("OTel ConfigMap reconciled", "name", cmName, "operation", result) + } + return nil +} + +// DeleteOtelConfigMap removes the OTel ConfigMap when monitoring is no longer configured. +func (r *ClusterReconciler[T]) DeleteOtelConfigMap(ctx context.Context, clusterName, namespace string) error { + logger := log.FromContext(ctx) + cmName := otelcfg.ConfigMapName(clusterName) + + cm := &corev1.ConfigMap{} + cm.Name = cmName + cm.Namespace = namespace + + err := r.Delete(ctx, cm) + if err != nil { + if errors.IsNotFound(err) { + return nil + } + return fmt.Errorf("failed to delete OTel ConfigMap %s: %w", cmName, err) + } + logger.Info("OTel ConfigMap deleted", "name", cmName) + return nil +} + +// ControllerBuilder returns the controller builder watching the custom resource, +// its gateway Service, and its CNPG Cluster. Products may add more watches +// before calling Complete. +func (r *ClusterReconciler[T]) ControllerBuilder(mgr ctrl.Manager, name string) *builder.Builder { + return ctrl.NewControllerManagedBy(mgr). + For(r.Adapter.New()). + Owns(&corev1.Service{}, builder.WithPredicates(ServicePrefixPredicate(r.profile().Naming.ServicePrefix))). + Owns(&cnpgv1.Cluster{}, builder.WithPredicates(ClusterInstanceStatusChangedPredicate())). + Named(name) +} + +// SetupWithManager registers the reconciler as controller name. +func (r *ClusterReconciler[T]) SetupWithManager(mgr ctrl.Manager, name string) error { + if r.SQLExecutor == nil { + return fmt.Errorf("SQLExecutor must be configured: required for SQL execution against the CNPG primary") + } + return r.ControllerBuilder(mgr, name).Complete(r) +} diff --git a/operator/src/internal/engine/extension.go b/operator/src/internal/engine/extension.go new file mode 100644 index 00000000..86cef740 --- /dev/null +++ b/operator/src/internal/engine/extension.go @@ -0,0 +1,297 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package engine + +import ( + "context" + "fmt" + "slices" + "strings" + + cnpgv1 "github.com/cloudnative-pg/cloudnative-pg/api/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/log" + + cnpg "github.com/documentdb/documentdb-operator/internal/cnpg" + util "github.com/documentdb/documentdb-operator/internal/utils" +) + +// ParseExtensionVersions parses the output of the pg_available_extensions query. +// Returns defaultVersion, installedVersion, and a boolean indicating if parsing was successful. +// Expected output format: +// +// default_version | installed_version +// -----------------+------------------- +// 0.110-0 | 0.110-0 +// +// TODO: This parsing is fragile as it relies on psql tabular output format which can vary +// with locale or PostgreSQL version. Consider one of these improvements: +// - Option 1: Add -t -A flags to SQLExecutor for unaligned, tuples-only output ("0.110-0|0.110-0") +// - Option 2: Use json_build_object() in SQL and json.Unmarshal() in Go for robust parsing +func ParseExtensionVersions(output string) (defaultVersion, installedVersion string, ok bool) { + lines := strings.Split(strings.TrimSpace(output), "\n") + if len(lines) < 3 { + return "", "", false + } + + // Parse the data row (3rd line, index 2) + dataLine := strings.TrimSpace(lines[2]) + parts := strings.Split(dataLine, "|") + if len(parts) != 2 { + return "", "", false + } + + defaultVersion = strings.TrimSpace(parts[0]) + installedVersion = strings.TrimSpace(parts[1]) + return defaultVersion, installedVersion, true +} + +func (r *ClusterReconciler[T]) extensionName() string { + return r.profile().Postgres.Extension.Name +} + +func (r *ClusterReconciler[T]) refetch(ctx context.Context, obj T) error { + return r.Get(ctx, types.NamespacedName{Name: obj.GetName(), Namespace: obj.GetNamespace()}, obj) +} + +// HandleExtensionUpgrade handles the ALTER EXTENSION lifecycle after images have been synced +// by SyncCnpgCluster. It: +// 1. Updates status with the current images from the CNPG cluster +// 2. Checks if ALTER EXTENSION UPDATE is needed and runs it (with two-phase upgrade support) +// 3. Updates the schema version status +func (r *ClusterReconciler[T]) HandleExtensionUpgrade(ctx context.Context, currentCluster *cnpgv1.Cluster, obj T) error { + logger := log.FromContext(ctx) + name := r.profile().Name + + // Refetch to avoid potential race conditions with status updates + if err := r.refetch(ctx, obj); err != nil { + return fmt.Errorf("failed to refetch %s resource: %w", name, err) + } + + // Update image status fields to reflect what's currently applied + if err := r.UpdateImageStatus(ctx, obj, currentCluster); err != nil { + logger.Error(err, "Failed to update image status") + } + + // CNPG will trigger a rolling restart when images change. Wait for pods to become healthy + // before running ALTER EXTENSION. + if !slices.Contains(currentCluster.Status.InstancesStatus[cnpgv1.PodHealthy], currentCluster.Status.CurrentPrimary) { + logger.Info("Current primary pod is not healthy; skipping " + name + " extension upgrade") + return nil + } + + // Check if ALTER EXTENSION UPDATE is needed + extension := r.extensionName() + output, err := r.SQLExecutor(ctx, currentCluster, cnpg.ExtensionVersionSQL(extension)) + if err != nil { + return fmt.Errorf("failed to check %s extension versions: %w", extension, err) + } + + defaultVersion, installedVersion, ok := ParseExtensionVersions(output) + if !ok { + logger.Info(name+" extension not found or not installed yet", "output", output) + return nil + } + + if installedVersion == "" { + logger.Info(name + " extension is not installed yet") + return nil + } + + // Update schema version in status (even if no upgrade needed) + // Convert from pg_available_extensions format ("0.110-0") to semver ("0.110.0") + installedSemver := util.ExtensionVersionToSemver(installedVersion) + if r.Adapter.Status(obj).SchemaVersion != installedSemver { + // Re-fetch to get latest resourceVersion before status update + if err := r.refetch(ctx, obj); err != nil { + return fmt.Errorf("failed to refetch %s before schema version update: %w", name, err) + } + r.setSchemaVersion(obj, installedSemver) + if err := r.Status().Update(ctx, obj); err != nil { + logger.Error(err, "Failed to update "+name+" status with schema version") + return fmt.Errorf("failed to update %s status with schema version: %w", name, err) + } + } + + // If versions match, no upgrade needed + if defaultVersion == installedVersion { + logger.V(1).Info(name+" extension is up to date", "version", installedVersion) + return nil + } + + // Rollback detection — check if the new binary is older than the installed schema. + // Note: Image rollback below installed schema version is also blocked by the validating + // webhook at admission time. This is a defense-in-depth guard. + cmp, err := util.CompareExtensionVersions(defaultVersion, installedVersion) + if err != nil { + logger.Error(err, "Failed to compare extension versions, skipping ALTER EXTENSION as a safety measure", + "defaultVersion", defaultVersion, + "installedVersion", installedVersion) + return nil + } + + if cmp < 0 { + // ALTER EXTENSION UPDATE would attempt an unsupported downgrade. Skip it and warn the user. + msg := fmt.Sprintf( + "Extension rollback detected: binary offers version %s but schema is at %s. "+ + "ALTER EXTENSION UPDATE skipped — %s does not provide downgrade scripts. "+ + "The cluster will run with the older binary against the newer schema, which may cause issues. "+ + "To resolve, update the extension image to a version that matches or exceeds %s.", + defaultVersion, installedVersion, name, installedVersion) + logger.Info(msg) + if r.Recorder != nil { + r.Recorder.Event(obj, corev1.EventTypeWarning, "ExtensionRollback", msg) + } + return nil + } + + // Determine schema target based on spec.schemaVersion (two-phase upgrade logic) + schemaTarget, updateSQL := r.DetermineSchemaTarget(ctx, obj, defaultVersion, installedVersion) + if schemaTarget == "" { + // Two-phase mode or validation failure — do not run ALTER EXTENSION + return nil + } + + // Run ALTER EXTENSION to upgrade + logger.Info("Upgrading "+name+" extension", + "fromVersion", installedVersion, + "toVersion", schemaTarget) + + if _, err := r.SQLExecutor(ctx, currentCluster, updateSQL); err != nil { + return fmt.Errorf("failed to run ALTER EXTENSION %s UPDATE: %w", extension, err) + } + + logger.Info("Successfully upgraded "+name+" extension", + "fromVersion", installedVersion, + "toVersion", schemaTarget) + + // Update schema version in status after upgrade + if err := r.refetch(ctx, obj); err != nil { + return fmt.Errorf("failed to refetch %s after schema upgrade: %w", name, err) + } + r.setSchemaVersion(obj, util.ExtensionVersionToSemver(schemaTarget)) + if err := r.Status().Update(ctx, obj); err != nil { + logger.Error(err, "Failed to update "+name+" status after schema upgrade") + return fmt.Errorf("failed to update %s status after schema upgrade: %w", name, err) + } + + return nil +} + +func (r *ClusterReconciler[T]) setSchemaVersion(obj T, version string) { + status := r.Adapter.Status(obj) + status.SchemaVersion = version + r.Adapter.SetStatus(obj, status) +} + +// DetermineSchemaTarget decides the target schema version based on the spec schema version. +// Returns the target version (in pg_available_extensions format) and the SQL to execute. +// Returns ("", "") if no schema update should run (two-phase mode or validation failure). +func (r *ClusterReconciler[T]) DetermineSchemaTarget( + ctx context.Context, + obj T, + binaryVersion string, + installedVersion string, +) (string, string) { + logger := log.FromContext(ctx) + specSchemaVersion := r.Adapter.Spec(obj).SchemaVersion + + switch { + case specSchemaVersion == "": + // Two-phase mode: schema stays at current version until user explicitly sets schemaVersion. + // Use V(1) to avoid log spam — this fires every reconcile while upgrade is pending. + logger.V(1).Info("Schema update available but not requested (two-phase mode). "+ + "Set spec.schemaVersion to finalize the upgrade.", + "binaryVersion", binaryVersion, + "installedVersion", installedVersion) + // Only emit the event when the binary is actually ahead of the installed schema + // (avoids firing on new clusters where status.schemaVersion is empty). + // Kubernetes deduplicates events with the same reason+message, but skipping + // entirely is cleaner. + if r.Recorder != nil && binaryVersion != installedVersion { + msg := fmt.Sprintf( + "Schema update available: binary version is %s, schema is at %s. "+ + "Set spec.schemaVersion to %q or \"auto\" to finalize the upgrade.", + util.ExtensionVersionToSemver(binaryVersion), + util.ExtensionVersionToSemver(installedVersion), + util.ExtensionVersionToSemver(binaryVersion)) + r.Recorder.Event(obj, corev1.EventTypeNormal, "SchemaUpdateAvailable", msg) + } + return "", "" + + case specSchemaVersion == "auto": + // Auto-finalize: update schema to match binary version + return binaryVersion, cnpg.ExtensionUpdateSQL(r.extensionName(), "") + + default: + // Explicit version: update to that specific version. + targetPgVersion := util.SemverToExtensionVersion(specSchemaVersion) + + // Guard: target must not exceed binary version. + // Note: schemaVersion <= binary version is enforced by the validating webhook at + // admission time. This is a defense-in-depth guard. + targetCmp, err := util.CompareExtensionVersions(targetPgVersion, binaryVersion) + if err != nil { + logger.Error(err, "Failed to compare target schema version with binary version", + "targetVersion", specSchemaVersion, + "binaryVersion", binaryVersion) + return "", "" + } + if targetCmp > 0 { + logger.Info("Skipping schema update: schemaVersion exceeds binary version (should have been rejected by webhook)", + "schemaVersion", specSchemaVersion, + "binaryVersion", util.ExtensionVersionToSemver(binaryVersion)) + return "", "" + } + + // Check: target must be > installed version (otherwise no-op) + installedCmp, err := util.CompareExtensionVersions(targetPgVersion, installedVersion) + if err != nil { + logger.Error(err, "Failed to compare target schema version with installed version", + "targetVersion", specSchemaVersion, + "installedVersion", installedVersion) + return "", "" + } + if installedCmp <= 0 { + logger.V(1).Info("Schema already at or beyond requested version", + "requestedVersion", specSchemaVersion, + "installedVersion", installedVersion) + return "", "" + } + + // Safety: targetPgVersion is derived from spec.schemaVersion via SemverToExtensionVersion, + // which produces a "Major.Minor-Patch" string (e.g., "0.112-0"). The input is validated + // by the CRD schema regex pattern and the validating webhook's version parsing, so it + // contains only digits, dots, and hyphens — no SQL injection risk. + return targetPgVersion, cnpg.ExtensionUpdateSQL(r.extensionName(), targetPgVersion) + } +} + +// UpdateImageStatus reads the current extension and gateway images from the CNPG cluster +// and persists them into the status. This is a no-op if both fields are already up to date. +func (r *ClusterReconciler[T]) UpdateImageStatus(ctx context.Context, obj T, cluster *cnpgv1.Cluster) error { + name := r.profile().Name + currentExtImage := cnpg.ExtensionImage(cluster, r.extensionName()) + currentGwImage := cnpg.GatewayImage(cluster) + + // Only update if something changed + status := r.Adapter.Status(obj) + if status.ExtensionImage == currentExtImage && status.GatewayImage == currentGwImage { + return nil + } + + // Re-fetch to get latest resourceVersion before status update + if err := r.refetch(ctx, obj); err != nil { + return fmt.Errorf("failed to refetch %s before image status update: %w", name, err) + } + status = r.Adapter.Status(obj) + status.ExtensionImage = currentExtImage + status.GatewayImage = currentGwImage + r.Adapter.SetStatus(obj, status) + if err := r.Status().Update(ctx, obj); err != nil { + return fmt.Errorf("failed to update %s image status: %w", name, err) + } + return nil +} diff --git a/operator/src/internal/engine/predicates.go b/operator/src/internal/engine/predicates.go new file mode 100644 index 00000000..fdf8ab1d --- /dev/null +++ b/operator/src/internal/engine/predicates.go @@ -0,0 +1,53 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package engine + +import ( + "slices" + "strings" + + cnpgv1 "github.com/cloudnative-pg/cloudnative-pg/api/v1" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/predicate" +) + +// ClusterInstanceStatusChangedPredicate triggers on CNPG cluster healthy-instance +// or phase changes. If you ever have another state from the cluster that you +// want to trigger on, add it here. +func ClusterInstanceStatusChangedPredicate() predicate.Predicate { + return predicate.Funcs{ + UpdateFunc: func(e event.UpdateEvent) bool { + oldCluster, ok := e.ObjectOld.(*cnpgv1.Cluster) + if !ok { + return true + } + newCluster, ok := e.ObjectNew.(*cnpgv1.Cluster) + if !ok { + return true + } + // Trigger on healthy instances change OR phase change + return !slices.Equal(oldCluster.Status.InstancesStatus[cnpgv1.PodHealthy], newCluster.Status.InstancesStatus[cnpgv1.PodHealthy]) || + oldCluster.Status.Phase != newCluster.Status.Phase + }, + } +} + +// ServicePrefixPredicate only triggers reconciliation for Services whose name +// starts with the product's gateway Service prefix. +func ServicePrefixPredicate(prefix string) predicate.Predicate { + return predicate.Funcs{ + CreateFunc: func(e event.CreateEvent) bool { + return strings.HasPrefix(e.Object.GetName(), prefix) + }, + UpdateFunc: func(e event.UpdateEvent) bool { + return strings.HasPrefix(e.ObjectNew.GetName(), prefix) + }, + DeleteFunc: func(e event.DeleteEvent) bool { + return strings.HasPrefix(e.Object.GetName(), prefix) + }, + GenericFunc: func(e event.GenericEvent) bool { + return strings.HasPrefix(e.Object.GetName(), prefix) + }, + } +} diff --git a/operator/src/internal/engine/pv_recovery.go b/operator/src/internal/engine/pv_recovery.go new file mode 100644 index 00000000..600be7c2 --- /dev/null +++ b/operator/src/internal/engine/pv_recovery.go @@ -0,0 +1,121 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package engine + +import ( + "context" + "fmt" + + cnpgv1 "github.com/cloudnative-pg/cloudnative-pg/api/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/log" + + util "github.com/documentdb/documentdb-operator/internal/utils" +) + +// ReconcilePVRecovery handles recovery from a retained PersistentVolume. +// +// CNPG only supports recovery from PVC (via VolumeSnapshots.Storage with Kind: PersistentVolumeClaim), +// not directly from PV. To bridge this gap, we create a temporary PVC that binds to the retained PV +// via spec.volumeName. CNPG then clones the data from this temp PVC to new cluster PVCs. +// After recovery completes (cluster healthy), we delete the temp PVC to release the source PV +// back to the user for manual cleanup or reuse. +// +// Flow: +// - If no PV recovery configured, return immediately +// - If CNPG exists and healthy, delete temp PVC (recovery complete) +// - If CNPG doesn't exist, validate PV and create temp PVC bound to it +func (r *ClusterReconciler[T]) ReconcilePVRecovery(ctx context.Context, obj T, namespace, cnpgClusterName string) (ctrl.Result, error) { + logger := log.FromContext(ctx) + + // Skip if PV recovery is not configured + pvName := r.Adapter.Spec(obj).RecoveryPersistentVolume + if pvName == "" { + return ctrl.Result{}, nil + } + + tempPVCName := util.TempPVCNameForPVRecovery(obj.GetName()) + + // Check if CNPG cluster exists + cnpgCluster := &cnpgv1.Cluster{} + cnpgErr := r.Get(ctx, types.NamespacedName{Name: cnpgClusterName, Namespace: namespace}, cnpgCluster) + + if cnpgErr == nil { + // CNPG exists - check if healthy and cleanup temp PVC + if cnpgCluster.Status.Phase == CNPGClusterHealthyPhase { + tempPVC := &corev1.PersistentVolumeClaim{} + if err := r.Get(ctx, types.NamespacedName{Name: tempPVCName, Namespace: namespace}, tempPVC); err == nil { + logger.Info("Deleting temp PVC after successful recovery", "pvc", tempPVCName) + if err := r.Delete(ctx, tempPVC); err != nil { + return ctrl.Result{}, fmt.Errorf("failed to delete temp PVC %s: %w", tempPVCName, err) + } + } + } + return ctrl.Result{}, nil + } + + if !errors.IsNotFound(cnpgErr) { + return ctrl.Result{}, fmt.Errorf("failed to get CNPG cluster: %w", cnpgErr) + } + + // CNPG doesn't exist - prepare temp PVC for recovery + + // Check if temp PVC already exists + tempPVC := &corev1.PersistentVolumeClaim{} + tempPVCErr := r.Get(ctx, types.NamespacedName{Name: tempPVCName, Namespace: namespace}, tempPVC) + if tempPVCErr == nil { + // Temp PVC exists, check if bound + if tempPVC.Status.Phase != corev1.ClaimBound { + logger.Info("Waiting for temp PVC to bind to PV", "pvc", tempPVCName, "phase", tempPVC.Status.Phase) + return ctrl.Result{RequeueAfter: RequeueAfterShort}, nil + } + // PVC is bound, ready to proceed with CNPG creation + return ctrl.Result{}, nil + } + + if !errors.IsNotFound(tempPVCErr) { + return ctrl.Result{}, fmt.Errorf("failed to get temp PVC %s: %w", tempPVCName, tempPVCErr) + } + + // Verify PV exists and is available + pv := &corev1.PersistentVolume{} + if err := r.Get(ctx, types.NamespacedName{Name: pvName}, pv); err != nil { + if errors.IsNotFound(err) { + return ctrl.Result{}, fmt.Errorf("PV %s not found for recovery", pvName) + } + return ctrl.Result{}, fmt.Errorf("failed to get PV %s: %w", pvName, err) + } + + if !util.IsPVAvailableForRecovery(pv) { + return ctrl.Result{}, fmt.Errorf("PV %s must be Available or Released for recovery, current phase: %s.", pvName, pv.Status.Phase) + } + + // Clear claimRef if PV is Released + if util.NeedsToClearClaimRef(pv) { + logger.Info("Clearing claimRef on Released PV", "pv", pvName) + pv.Spec.ClaimRef = nil + if err := r.Update(ctx, pv); err != nil { + return ctrl.Result{}, fmt.Errorf("failed to clear claimRef on PV %s: %w", pvName, err) + } + return ctrl.Result{RequeueAfter: RequeueAfterShort}, nil + } + + // Create temp PVC + naming := r.profile().Naming + newPVC := util.BuildTempPVCForPVRecoveryWithLabels(obj.GetName(), namespace, pv, naming.RecoveryTempLabel(), naming.ClusterLabel()) + if err := controllerutil.SetControllerReference(obj, newPVC, r.Scheme); err != nil { + return ctrl.Result{}, fmt.Errorf("failed to set owner reference on temp PVC: %w", err) + } + + if err := r.Create(ctx, newPVC); err != nil { + return ctrl.Result{}, fmt.Errorf("failed to create temp PVC %s: %w", tempPVCName, err) + } + + logger.Info("Created temp PVC for PV recovery", "pvc", tempPVCName, "pv", pvName) + return ctrl.Result{RequeueAfter: RequeueAfterShort}, nil +} diff --git a/operator/src/internal/engine/sql.go b/operator/src/internal/engine/sql.go new file mode 100644 index 00000000..f0547e6b --- /dev/null +++ b/operator/src/internal/engine/sql.go @@ -0,0 +1,80 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package engine + +import ( + "bytes" + "context" + "fmt" + "strings" + + cnpgv1 "github.com/cloudnative-pg/cloudnative-pg/api/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/remotecommand" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/log" +) + +// NewPodSQLExecutor returns an SQLExecutor that runs psql in the postgres +// container of the CNPG primary pod over a SPDY exec stream. +func NewPodSQLExecutor(c client.Client, clientset kubernetes.Interface, config *rest.Config) SQLExecutor { + return func(ctx context.Context, cluster *cnpgv1.Cluster, sqlCommand string) (string, error) { + logger := log.FromContext(ctx) + + var targetPod corev1.Pod + if err := c.Get(ctx, types.NamespacedName{Name: cluster.Status.CurrentPrimary, Namespace: cluster.Namespace}, &targetPod); err != nil { + return "", fmt.Errorf("failed to get primary pod: %w", err) + } + + // Execute psql command in the postgres container + cmd := []string{ + "psql", + "-U", "postgres", + "-d", "postgres", + "-c", sqlCommand, + } + + req := clientset.CoreV1().RESTClient().Post(). + Resource("pods"). + Name(targetPod.Name). + Namespace(cluster.Namespace). + SubResource("exec"). + VersionedParams(&corev1.PodExecOptions{ + Container: "postgres", + Command: cmd, + Stdin: false, + Stdout: true, + Stderr: true, + TTY: false, + }, scheme.ParameterCodec) + + exec, err := remotecommand.NewSPDYExecutor(config, "POST", req.URL()) + if err != nil { + return "", fmt.Errorf("failed to create executor: %w", err) + } + + var stdout, stderr bytes.Buffer + err = exec.StreamWithContext(ctx, remotecommand.StreamOptions{ + Stdout: &stdout, + Stderr: &stderr, + }) + + if err != nil { + logger.Error(err, "Failed to execute SQL command", + "stdout", stdout.String(), + "stderr", stderr.String()) + return "", fmt.Errorf("failed to execute command: %w (stderr: %s)", err, stderr.String()) + } + + if stderr.Len() > 0 && !strings.Contains(stderr.String(), "GRANT") { + logger.Info("SQL command executed with warnings", "stderr", stderr.String()) + } + + return stdout.String(), nil + } +} diff --git a/operator/src/internal/utils/pv_recovery.go b/operator/src/internal/utils/pv_recovery.go index 257bec15..de81fbee 100644 --- a/operator/src/internal/utils/pv_recovery.go +++ b/operator/src/internal/utils/pv_recovery.go @@ -36,6 +36,12 @@ func TempPVCNameForPVRecovery(documentdbName string) string { // The PVC uses the PV's storage class, access modes, and capacity to ensure successful binding. // This temp PVC is used as a data source for CNPG to clone data during recovery. func BuildTempPVCForPVRecovery(documentdbName, namespace string, pv *corev1.PersistentVolume) *corev1.PersistentVolumeClaim { + return BuildTempPVCForPVRecoveryWithLabels(documentdbName, namespace, pv, LabelRecoveryTemp, LabelCluster) +} + +// BuildTempPVCForPVRecoveryWithLabels is BuildTempPVCForPVRecovery with the +// product's recovery-temp and cluster label keys. +func BuildTempPVCForPVRecoveryWithLabels(documentdbName, namespace string, pv *corev1.PersistentVolume, recoveryTempLabel, clusterLabel string) *corev1.PersistentVolumeClaim { // Use the PV's storage class directly. All PVs in DocumentDB are dynamically provisioned // by a StorageClass, so we can safely assume the storage class is always set. // Note: If PVs without a storage class need to be supported in the future, @@ -57,8 +63,8 @@ func BuildTempPVCForPVRecovery(documentdbName, namespace string, pv *corev1.Pers Name: TempPVCNameForPVRecovery(documentdbName), Namespace: namespace, Labels: map[string]string{ - LabelRecoveryTemp: "true", - LabelCluster: documentdbName, + recoveryTempLabel: "true", + clusterLabel: documentdbName, }, }, Spec: corev1.PersistentVolumeClaimSpec{ diff --git a/operator/src/internal/utils/replication_context.go b/operator/src/internal/utils/replication_context.go index da76dfd8..d750479b 100644 --- a/operator/src/internal/utils/replication_context.go +++ b/operator/src/internal/utils/replication_context.go @@ -45,16 +45,23 @@ const ( NotPresent ) -func GetReplicationContext(ctx context.Context, client client.Client, documentdb dbpreview.DocumentDB) (*ReplicationContext, error) { - singleClusterReplicationContext := ReplicationContext{ +// NewSingleRegionContext returns the replication context of a cluster that is +// not part of a multi-region topology: it is primary, serves its endpoint, and +// its CNPG cluster is named after the custom resource. +func NewSingleRegionContext(name, environment, storageClass string) *ReplicationContext { + return &ReplicationContext{ state: NoReplication, CrossCloudNetworkingStrategy: None, - Environment: documentdb.Spec.Environment, - StorageClass: documentdb.Spec.Resource.Storage.StorageClass, - CNPGClusterName: documentdb.Name, + Environment: environment, + StorageClass: storageClass, + CNPGClusterName: name, } +} + +func GetReplicationContext(ctx context.Context, client client.Client, documentdb dbpreview.DocumentDB) (*ReplicationContext, error) { + singleClusterReplicationContext := NewSingleRegionContext(documentdb.Name, documentdb.Spec.Environment, documentdb.Spec.Resource.Storage.StorageClass) if documentdb.Spec.ClusterReplication == nil { - return &singleClusterReplicationContext, nil + return singleClusterReplicationContext, nil } self, others, replicationState, err := getTopology(ctx, client, documentdb) @@ -77,7 +84,7 @@ func GetReplicationContext(ctx context.Context, client client.Client, documentdb // If no remote clusters, then just proceed with a regular cluster if len(others) == 0 { - return &singleClusterReplicationContext, nil + return singleClusterReplicationContext, nil } primaryCluster := generateCNPGClusterName(documentdb.Name, documentdb.Spec.ClusterReplication.Primary) From 0a6064150c87b416c051c168cfc9a23e1a29f3ab Mon Sep 17 00:00:00 2001 From: Rayhan Hossain Date: Mon, 19 Oct 2026 18:33:03 -0700 Subject: [PATCH 15/27] refactor: extract generic CertificateReconciler engine Move gateway TLS provisioning (SelfSigned, Provided, CertManager) into engine.CertificateReconciler[T]. DNS names use the product Service prefix and status is written through the adapter. The DocumentDB CertificateReconciler delegates; behavior is unchanged. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 33b3cb3c-1b25-4ffb-bff1-ebf0f3d04058 Signed-off-by: Rayhan Hossain --- .../controller/certificate_controller.go | 346 +--------------- operator/src/internal/engine/certificate.go | 381 ++++++++++++++++++ 2 files changed, 388 insertions(+), 339 deletions(-) create mode 100644 operator/src/internal/engine/certificate.go diff --git a/operator/src/internal/controller/certificate_controller.go b/operator/src/internal/controller/certificate_controller.go index 3335f986..9fa91014 100644 --- a/operator/src/internal/controller/certificate_controller.go +++ b/operator/src/internal/controller/certificate_controller.go @@ -5,23 +5,15 @@ package controller import ( "context" - "time" cmapi "github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1" - cmmeta "github.com/cert-manager/cert-manager/pkg/apis/meta/v1" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/types" - "k8s.io/client-go/util/retry" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" - "sigs.k8s.io/controller-runtime/pkg/log" dbpreview "github.com/documentdb/documentdb-operator/api/preview" - util "github.com/documentdb/documentdb-operator/internal/utils" + "github.com/documentdb/documentdb-operator/internal/engine" + "github.com/documentdb/documentdb-operator/internal/product" ) // CertificateReconciler manages certificate lifecycle for DocumentDB components. @@ -38,339 +30,15 @@ type CertificateReconciler struct { // +kubebuilder:rbac:groups="",resources=secrets,verbs=get;list;watch func (r *CertificateReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { - logger := log.FromContext(ctx) - - ddb := &dbpreview.DocumentDB{} - if err := r.Get(ctx, req.NamespacedName, ddb); err != nil { - if errors.IsNotFound(err) { - return ctrl.Result{}, nil - } - return ctrl.Result{}, err - } - - res, err := r.reconcileCertificates(ctx, ddb) - if err != nil { - logger.Error(err, "failed to reconcile certificate resources") - } - return res, err -} - -func (r *CertificateReconciler) reconcileCertificates(ctx context.Context, ddb *dbpreview.DocumentDB) (ctrl.Result, error) { - // TLS is always enabled. When tls or tls.gateway is unset, default to SelfSigned - // so the operator provisions a managed cert (issue #356 - no plaintext path). - var gatewayCfg *dbpreview.GatewayTLS - if ddb.Spec.TLS != nil { - gatewayCfg = ddb.Spec.TLS.Gateway - } - if gatewayCfg == nil { - gatewayCfg = &dbpreview.GatewayTLS{Mode: "SelfSigned"} - } - - // Empty mode defaults to SelfSigned for security - TLS is always enabled - mode := gatewayCfg.Mode - if mode == "" { - mode = "SelfSigned" - } - - if ddb.Status.TLS == nil { - if err := r.updateTLSStatus(ctx, ddb, func(status *dbpreview.TLSStatus) { - status.Ready = false - }); err != nil { - return ctrl.Result{}, err - } - } - - switch mode { - case "SelfSigned": - return r.ensureSelfSignedCert(ctx, ddb) - case "Provided": - return r.ensureProvidedSecret(ctx, ddb) - case "CertManager": - return r.ensureCertManagerManagedCert(ctx, ddb) - default: - // Unknown/legacy mode (e.g. "Disabled" from a pre-#357 resource still in etcd - // — apiserver does not re-validate stored objects against the trimmed enum). - // Fail-safe to SelfSigned so the gateway never serves plaintext. - log.FromContext(ctx).Info( - "unsupported TLS gateway mode; defaulting to SelfSigned to preserve no-plaintext invariant", - "requestedMode", mode, - "hint", "Update spec.tls.gateway.mode to SelfSigned, CertManager, or Provided.", - ) - return r.ensureSelfSignedCert(ctx, ddb) - } + return r.engine().Reconcile(ctx, req) } -func (r *CertificateReconciler) ensureProvidedSecret(ctx context.Context, ddb *dbpreview.DocumentDB) (ctrl.Result, error) { - gatewayCfg := ddb.Spec.TLS.Gateway - if gatewayCfg == nil || gatewayCfg.Provided == nil || gatewayCfg.Provided.SecretName == "" { - if err := r.updateTLSStatus(ctx, ddb, func(status *dbpreview.TLSStatus) { - status.Message = "Provided TLS secret name missing" - status.Ready = false - }); err != nil { - return ctrl.Result{}, err - } - return ctrl.Result{}, nil - } - - secret := &corev1.Secret{} - if err := r.Get(ctx, types.NamespacedName{Name: gatewayCfg.Provided.SecretName, Namespace: ddb.Namespace}, secret); err != nil { - if errors.IsNotFound(err) { - if err := r.updateTLSStatus(ctx, ddb, func(status *dbpreview.TLSStatus) { - status.Ready = false - status.SecretName = gatewayCfg.Provided.SecretName - status.Message = "Waiting for provided TLS secret" - }); err != nil { - return ctrl.Result{}, err - } - return ctrl.Result{RequeueAfter: RequeueAfterShort}, nil - } - return ctrl.Result{}, err - } - - if _, crtOk := secret.Data["tls.crt"]; !crtOk { - if err := r.updateTLSStatus(ctx, ddb, func(status *dbpreview.TLSStatus) { - status.Ready = false - status.Message = "Provided secret missing tls.crt" - status.SecretName = gatewayCfg.Provided.SecretName - }); err != nil { - return ctrl.Result{}, err - } - return ctrl.Result{RequeueAfter: RequeueAfterShort}, nil - } - if _, keyOk := secret.Data["tls.key"]; !keyOk { - if err := r.updateTLSStatus(ctx, ddb, func(status *dbpreview.TLSStatus) { - status.Ready = false - status.Message = "Provided secret missing tls.key" - status.SecretName = gatewayCfg.Provided.SecretName - }); err != nil { - return ctrl.Result{}, err - } - return ctrl.Result{RequeueAfter: RequeueAfterShort}, nil - } - - if err := r.updateTLSStatus(ctx, ddb, func(status *dbpreview.TLSStatus) { - status.Ready = true - status.SecretName = gatewayCfg.Provided.SecretName - status.Message = "Using provided TLS secret" - }); err != nil { - return ctrl.Result{}, err - } - return ctrl.Result{}, nil +func (r *CertificateReconciler) engine() *engine.CertificateReconciler[*dbpreview.DocumentDB] { + return &engine.CertificateReconciler[*dbpreview.DocumentDB]{Client: r.Client, Scheme: r.Scheme, Adapter: product.DocumentDBAdapter{}} } -func (r *CertificateReconciler) ensureCertManagerManagedCert(ctx context.Context, ddb *dbpreview.DocumentDB) (ctrl.Result, error) { - gatewayCfg := ddb.Spec.TLS.Gateway - if gatewayCfg == nil || gatewayCfg.CertManager == nil { - if err := r.updateTLSStatus(ctx, ddb, func(status *dbpreview.TLSStatus) { - status.Ready = false - status.Message = "CertManager configuration missing" - }); err != nil { - return ctrl.Result{}, err - } - return ctrl.Result{}, nil - } - - cmCfg := gatewayCfg.CertManager - - issuerRef := cmmeta.ObjectReference{Name: cmCfg.IssuerRef.Name} - if cmCfg.IssuerRef.Kind != "" { - issuerRef.Kind = cmCfg.IssuerRef.Kind - } else { - issuerRef.Kind = "Issuer" - } - if cmCfg.IssuerRef.Group != "" { - issuerRef.Group = cmCfg.IssuerRef.Group - } else { - issuerRef.Group = "cert-manager.io" - } - - secretName := cmCfg.SecretName - if secretName == "" { - secretName = ddb.Name + "-gateway-cert-tls" - } - - serviceBase := util.DOCUMENTDB_SERVICE_PREFIX + ddb.Name - baseDNS := []string{serviceBase, serviceBase + "." + ddb.Namespace, serviceBase + "." + ddb.Namespace + ".svc"} - dnsSet := map[string]struct{}{} - finalDNS := []string{} - for _, n := range cmCfg.DNSNames { - if _, ok := dnsSet[n]; !ok && n != "" { - dnsSet[n] = struct{}{} - finalDNS = append(finalDNS, n) - } - } - for _, n := range baseDNS { - if _, ok := dnsSet[n]; !ok { - dnsSet[n] = struct{}{} - finalDNS = append(finalDNS, n) - } - } - - certName := ddb.Name + "-gateway-cert" - cert := &cmapi.Certificate{} - if err := r.Get(ctx, types.NamespacedName{Name: certName, Namespace: ddb.Namespace}, cert); err != nil { - if !errors.IsNotFound(err) { - return ctrl.Result{}, err - } - - cert = &cmapi.Certificate{ - ObjectMeta: metav1.ObjectMeta{Name: certName, Namespace: ddb.Namespace}, - Spec: cmapi.CertificateSpec{ - SecretName: secretName, - DNSNames: finalDNS, - IssuerRef: issuerRef, - Duration: &metav1.Duration{Duration: 90 * 24 * time.Hour}, - RenewBefore: &metav1.Duration{Duration: 15 * 24 * time.Hour}, - Usages: []cmapi.KeyUsage{cmapi.UsageServerAuth}, - }, - } - if err := controllerutil.SetControllerReference(ddb, cert, r.Scheme); err != nil { - return ctrl.Result{}, err - } - if err := r.Create(ctx, cert); err != nil { - return ctrl.Result{}, err - } - if err := r.updateTLSStatus(ctx, ddb, func(status *dbpreview.TLSStatus) { - status.Ready = false - status.SecretName = secretName - status.Message = "Creating cert-manager certificate" - }); err != nil { - return ctrl.Result{}, err - } - return ctrl.Result{RequeueAfter: RequeueAfterShort}, nil - } - - for _, cond := range cert.Status.Conditions { - if cond.Type == cmapi.CertificateConditionReady && cond.Status == cmmeta.ConditionTrue { - if !ddb.Status.TLS.Ready { - if err := r.updateTLSStatus(ctx, ddb, func(status *dbpreview.TLSStatus) { - status.Ready = true - status.SecretName = cert.Spec.SecretName - status.Message = "Gateway TLS certificate ready (cert-manager)" - }); err != nil { - return ctrl.Result{}, err - } - } - return ctrl.Result{}, nil - } - } - - if err := r.updateTLSStatus(ctx, ddb, func(status *dbpreview.TLSStatus) { - status.Ready = false - status.SecretName = cert.Spec.SecretName - status.Message = "Waiting for cert-manager certificate to become ready" - }); err != nil { - return ctrl.Result{}, err - } - return ctrl.Result{RequeueAfter: RequeueAfterShort}, nil -} - -func (r *CertificateReconciler) ensureSelfSignedCert(ctx context.Context, ddb *dbpreview.DocumentDB) (ctrl.Result, error) { - namespace := ddb.Namespace - issuerName := ddb.Name + "-gateway-selfsigned" - certName := ddb.Name + "-gateway-cert" - secretName := certName + "-tls" - - issuer := &cmapi.Issuer{} - if err := r.Get(ctx, types.NamespacedName{Name: issuerName, Namespace: namespace}, issuer); err != nil { - if !errors.IsNotFound(err) { - return ctrl.Result{}, err - } - - issuer = &cmapi.Issuer{ - ObjectMeta: metav1.ObjectMeta{Name: issuerName, Namespace: namespace}, - Spec: cmapi.IssuerSpec{IssuerConfig: cmapi.IssuerConfig{SelfSigned: &cmapi.SelfSignedIssuer{}}}, - } - if err := controllerutil.SetControllerReference(ddb, issuer, r.Scheme); err != nil { - return ctrl.Result{}, err - } - if err := r.Create(ctx, issuer); err != nil { - return ctrl.Result{}, err - } - } - - serviceBase := util.DOCUMENTDB_SERVICE_PREFIX + ddb.Name - dnsNames := []string{ - serviceBase, - serviceBase + "." + namespace, - serviceBase + "." + namespace + ".svc", - } - - cert := &cmapi.Certificate{} - if err := r.Get(ctx, types.NamespacedName{Name: certName, Namespace: namespace}, cert); err != nil { - if !errors.IsNotFound(err) { - return ctrl.Result{}, err - } - - cert = &cmapi.Certificate{ - ObjectMeta: metav1.ObjectMeta{Name: certName, Namespace: namespace}, - Spec: cmapi.CertificateSpec{ - SecretName: secretName, - Duration: &metav1.Duration{Duration: 90 * 24 * time.Hour}, - RenewBefore: &metav1.Duration{Duration: 15 * 24 * time.Hour}, - DNSNames: dnsNames, - IssuerRef: cmmeta.ObjectReference{Name: issuerName, Kind: "Issuer", Group: "cert-manager.io"}, - Usages: []cmapi.KeyUsage{cmapi.UsageServerAuth}, - }, - } - if err := controllerutil.SetControllerReference(ddb, cert, r.Scheme); err != nil { - return ctrl.Result{}, err - } - if err := r.Create(ctx, cert); err != nil { - return ctrl.Result{}, err - } - if err := r.updateTLSStatus(ctx, ddb, func(status *dbpreview.TLSStatus) { - status.Ready = false - status.SecretName = secretName - status.Message = "Creating self-signed certificate" - }); err != nil { - return ctrl.Result{}, err - } - return ctrl.Result{RequeueAfter: RequeueAfterShort}, nil - } - - for _, cond := range cert.Status.Conditions { - if cond.Type == cmapi.CertificateConditionReady && cond.Status == cmmeta.ConditionTrue { - if !ddb.Status.TLS.Ready { - if err := r.updateTLSStatus(ctx, ddb, func(status *dbpreview.TLSStatus) { - status.Ready = true - status.SecretName = cert.Spec.SecretName - status.Message = "Gateway TLS certificate ready" - }); err != nil { - return ctrl.Result{}, err - } - } - return ctrl.Result{}, nil - } - } - - if err := r.updateTLSStatus(ctx, ddb, func(status *dbpreview.TLSStatus) { - status.Ready = false - status.SecretName = cert.Spec.SecretName - status.Message = "Waiting for gateway TLS certificate to become ready" - }); err != nil { - return ctrl.Result{}, err - } - return ctrl.Result{RequeueAfter: RequeueAfterShort}, nil -} - -func (r *CertificateReconciler) updateTLSStatus(ctx context.Context, ddb *dbpreview.DocumentDB, mutate func(*dbpreview.TLSStatus)) error { - key := types.NamespacedName{Name: ddb.Name, Namespace: ddb.Namespace} - return retry.RetryOnConflict(retry.DefaultRetry, func() error { - current := &dbpreview.DocumentDB{} - if err := r.Get(ctx, key, current); err != nil { - return err - } - if current.Status.TLS == nil { - current.Status.TLS = &dbpreview.TLSStatus{} - } - mutate(current.Status.TLS) - if err := r.Status().Update(ctx, current); err != nil { - return err - } - ddb.Status = current.Status - return nil - }) +func (r *CertificateReconciler) reconcileCertificates(ctx context.Context, ddb *dbpreview.DocumentDB) (ctrl.Result, error) { + return r.engine().ReconcileCertificates(ctx, ddb) } func (r *CertificateReconciler) SetupWithManager(mgr ctrl.Manager) error { diff --git a/operator/src/internal/engine/certificate.go b/operator/src/internal/engine/certificate.go new file mode 100644 index 00000000..270b9374 --- /dev/null +++ b/operator/src/internal/engine/certificate.go @@ -0,0 +1,381 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package engine + +import ( + "context" + "time" + + cmapi "github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1" + cmmeta "github.com/cert-manager/cert-manager/pkg/apis/meta/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/util/retry" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/log" + + dbpreview "github.com/documentdb/documentdb-operator/api/preview" + "github.com/documentdb/documentdb-operator/internal/product" +) + +// CertificateReconciler provisions the gateway TLS certificate of a product +// cluster: a self-signed cert-manager Issuer and Certificate (default), a +// user-provided Secret, or a cert-manager Certificate from a user Issuer. +type CertificateReconciler[T client.Object] struct { + client.Client + Scheme *runtime.Scheme + Adapter product.ClusterAdapter[T] +} + +// Reconcile reconciles the gateway TLS assets of the custom resource. +func (r *CertificateReconciler[T]) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + logger := log.FromContext(ctx) + + obj := r.Adapter.New() + if err := r.Get(ctx, req.NamespacedName, obj); err != nil { + if errors.IsNotFound(err) { + return ctrl.Result{}, nil + } + return ctrl.Result{}, err + } + + res, err := r.ReconcileCertificates(ctx, obj) + if err != nil { + logger.Error(err, "failed to reconcile certificate resources") + } + return res, err +} + +// ReconcileCertificates reconciles gateway TLS for obj by mode. +func (r *CertificateReconciler[T]) ReconcileCertificates(ctx context.Context, obj T) (ctrl.Result, error) { + // TLS is always enabled. When tls or tls.gateway is unset, default to SelfSigned + // so the operator provisions a managed cert (issue #356 - no plaintext path). + gatewayCfg := r.Adapter.Spec(obj).GatewayTLS + if gatewayCfg == nil { + gatewayCfg = &dbpreview.GatewayTLS{Mode: "SelfSigned"} + } + + // Empty mode defaults to SelfSigned for security - TLS is always enabled + mode := gatewayCfg.Mode + if mode == "" { + mode = "SelfSigned" + } + + if r.Adapter.Status(obj).TLS == nil { + if err := r.updateTLSStatus(ctx, obj, func(status *dbpreview.TLSStatus) { + status.Ready = false + }); err != nil { + return ctrl.Result{}, err + } + } + + switch mode { + case "SelfSigned": + return r.ensureSelfSignedCert(ctx, obj) + case "Provided": + return r.ensureProvidedSecret(ctx, obj) + case "CertManager": + return r.ensureCertManagerManagedCert(ctx, obj) + default: + // Unknown/legacy mode (e.g. "Disabled" from a pre-#357 resource still in etcd + // — apiserver does not re-validate stored objects against the trimmed enum). + // Fail-safe to SelfSigned so the gateway never serves plaintext. + log.FromContext(ctx).Info( + "unsupported TLS gateway mode; defaulting to SelfSigned to preserve no-plaintext invariant", + "requestedMode", mode, + "hint", "Update spec.tls.gateway.mode to SelfSigned, CertManager, or Provided.", + ) + return r.ensureSelfSignedCert(ctx, obj) + } +} + +func (r *CertificateReconciler[T]) ensureProvidedSecret(ctx context.Context, obj T) (ctrl.Result, error) { + gatewayCfg := r.Adapter.Spec(obj).GatewayTLS + if gatewayCfg == nil || gatewayCfg.Provided == nil || gatewayCfg.Provided.SecretName == "" { + if err := r.updateTLSStatus(ctx, obj, func(status *dbpreview.TLSStatus) { + status.Message = "Provided TLS secret name missing" + status.Ready = false + }); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{}, nil + } + + secret := &corev1.Secret{} + if err := r.Get(ctx, types.NamespacedName{Name: gatewayCfg.Provided.SecretName, Namespace: obj.GetNamespace()}, secret); err != nil { + if errors.IsNotFound(err) { + if err := r.updateTLSStatus(ctx, obj, func(status *dbpreview.TLSStatus) { + status.Ready = false + status.SecretName = gatewayCfg.Provided.SecretName + status.Message = "Waiting for provided TLS secret" + }); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{RequeueAfter: RequeueAfterShort}, nil + } + return ctrl.Result{}, err + } + + if _, crtOk := secret.Data["tls.crt"]; !crtOk { + if err := r.updateTLSStatus(ctx, obj, func(status *dbpreview.TLSStatus) { + status.Ready = false + status.Message = "Provided secret missing tls.crt" + status.SecretName = gatewayCfg.Provided.SecretName + }); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{RequeueAfter: RequeueAfterShort}, nil + } + if _, keyOk := secret.Data["tls.key"]; !keyOk { + if err := r.updateTLSStatus(ctx, obj, func(status *dbpreview.TLSStatus) { + status.Ready = false + status.Message = "Provided secret missing tls.key" + status.SecretName = gatewayCfg.Provided.SecretName + }); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{RequeueAfter: RequeueAfterShort}, nil + } + + if err := r.updateTLSStatus(ctx, obj, func(status *dbpreview.TLSStatus) { + status.Ready = true + status.SecretName = gatewayCfg.Provided.SecretName + status.Message = "Using provided TLS secret" + }); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{}, nil +} + +func (r *CertificateReconciler[T]) ensureCertManagerManagedCert(ctx context.Context, obj T) (ctrl.Result, error) { + gatewayCfg := r.Adapter.Spec(obj).GatewayTLS + if gatewayCfg == nil || gatewayCfg.CertManager == nil { + if err := r.updateTLSStatus(ctx, obj, func(status *dbpreview.TLSStatus) { + status.Ready = false + status.Message = "CertManager configuration missing" + }); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{}, nil + } + + cmCfg := gatewayCfg.CertManager + + issuerRef := cmmeta.ObjectReference{Name: cmCfg.IssuerRef.Name} + if cmCfg.IssuerRef.Kind != "" { + issuerRef.Kind = cmCfg.IssuerRef.Kind + } else { + issuerRef.Kind = "Issuer" + } + if cmCfg.IssuerRef.Group != "" { + issuerRef.Group = cmCfg.IssuerRef.Group + } else { + issuerRef.Group = "cert-manager.io" + } + + secretName := cmCfg.SecretName + if secretName == "" { + secretName = obj.GetName() + "-gateway-cert-tls" + } + + serviceBase := r.Adapter.Profile().Naming.ServicePrefix + obj.GetName() + baseDNS := []string{serviceBase, serviceBase + "." + obj.GetNamespace(), serviceBase + "." + obj.GetNamespace() + ".svc"} + dnsSet := map[string]struct{}{} + finalDNS := []string{} + for _, n := range cmCfg.DNSNames { + if _, ok := dnsSet[n]; !ok && n != "" { + dnsSet[n] = struct{}{} + finalDNS = append(finalDNS, n) + } + } + for _, n := range baseDNS { + if _, ok := dnsSet[n]; !ok { + dnsSet[n] = struct{}{} + finalDNS = append(finalDNS, n) + } + } + + certName := obj.GetName() + "-gateway-cert" + cert := &cmapi.Certificate{} + if err := r.Get(ctx, types.NamespacedName{Name: certName, Namespace: obj.GetNamespace()}, cert); err != nil { + if !errors.IsNotFound(err) { + return ctrl.Result{}, err + } + + cert = &cmapi.Certificate{ + ObjectMeta: metav1.ObjectMeta{Name: certName, Namespace: obj.GetNamespace()}, + Spec: cmapi.CertificateSpec{ + SecretName: secretName, + DNSNames: finalDNS, + IssuerRef: issuerRef, + Duration: &metav1.Duration{Duration: 90 * 24 * time.Hour}, + RenewBefore: &metav1.Duration{Duration: 15 * 24 * time.Hour}, + Usages: []cmapi.KeyUsage{cmapi.UsageServerAuth}, + }, + } + if err := controllerutil.SetControllerReference(obj, cert, r.Scheme); err != nil { + return ctrl.Result{}, err + } + if err := r.Create(ctx, cert); err != nil { + return ctrl.Result{}, err + } + if err := r.updateTLSStatus(ctx, obj, func(status *dbpreview.TLSStatus) { + status.Ready = false + status.SecretName = secretName + status.Message = "Creating cert-manager certificate" + }); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{RequeueAfter: RequeueAfterShort}, nil + } + + for _, cond := range cert.Status.Conditions { + if cond.Type == cmapi.CertificateConditionReady && cond.Status == cmmeta.ConditionTrue { + if !r.Adapter.Status(obj).TLS.Ready { + if err := r.updateTLSStatus(ctx, obj, func(status *dbpreview.TLSStatus) { + status.Ready = true + status.SecretName = cert.Spec.SecretName + status.Message = "Gateway TLS certificate ready (cert-manager)" + }); err != nil { + return ctrl.Result{}, err + } + } + return ctrl.Result{}, nil + } + } + + if err := r.updateTLSStatus(ctx, obj, func(status *dbpreview.TLSStatus) { + status.Ready = false + status.SecretName = cert.Spec.SecretName + status.Message = "Waiting for cert-manager certificate to become ready" + }); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{RequeueAfter: RequeueAfterShort}, nil +} + +func (r *CertificateReconciler[T]) ensureSelfSignedCert(ctx context.Context, obj T) (ctrl.Result, error) { + namespace := obj.GetNamespace() + issuerName := obj.GetName() + "-gateway-selfsigned" + certName := obj.GetName() + "-gateway-cert" + secretName := certName + "-tls" + + issuer := &cmapi.Issuer{} + if err := r.Get(ctx, types.NamespacedName{Name: issuerName, Namespace: namespace}, issuer); err != nil { + if !errors.IsNotFound(err) { + return ctrl.Result{}, err + } + + issuer = &cmapi.Issuer{ + ObjectMeta: metav1.ObjectMeta{Name: issuerName, Namespace: namespace}, + Spec: cmapi.IssuerSpec{IssuerConfig: cmapi.IssuerConfig{SelfSigned: &cmapi.SelfSignedIssuer{}}}, + } + if err := controllerutil.SetControllerReference(obj, issuer, r.Scheme); err != nil { + return ctrl.Result{}, err + } + if err := r.Create(ctx, issuer); err != nil { + return ctrl.Result{}, err + } + } + + serviceBase := r.Adapter.Profile().Naming.ServicePrefix + obj.GetName() + dnsNames := []string{ + serviceBase, + serviceBase + "." + namespace, + serviceBase + "." + namespace + ".svc", + } + + cert := &cmapi.Certificate{} + if err := r.Get(ctx, types.NamespacedName{Name: certName, Namespace: namespace}, cert); err != nil { + if !errors.IsNotFound(err) { + return ctrl.Result{}, err + } + + cert = &cmapi.Certificate{ + ObjectMeta: metav1.ObjectMeta{Name: certName, Namespace: namespace}, + Spec: cmapi.CertificateSpec{ + SecretName: secretName, + Duration: &metav1.Duration{Duration: 90 * 24 * time.Hour}, + RenewBefore: &metav1.Duration{Duration: 15 * 24 * time.Hour}, + DNSNames: dnsNames, + IssuerRef: cmmeta.ObjectReference{Name: issuerName, Kind: "Issuer", Group: "cert-manager.io"}, + Usages: []cmapi.KeyUsage{cmapi.UsageServerAuth}, + }, + } + if err := controllerutil.SetControllerReference(obj, cert, r.Scheme); err != nil { + return ctrl.Result{}, err + } + if err := r.Create(ctx, cert); err != nil { + return ctrl.Result{}, err + } + if err := r.updateTLSStatus(ctx, obj, func(status *dbpreview.TLSStatus) { + status.Ready = false + status.SecretName = secretName + status.Message = "Creating self-signed certificate" + }); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{RequeueAfter: RequeueAfterShort}, nil + } + + for _, cond := range cert.Status.Conditions { + if cond.Type == cmapi.CertificateConditionReady && cond.Status == cmmeta.ConditionTrue { + if !r.Adapter.Status(obj).TLS.Ready { + if err := r.updateTLSStatus(ctx, obj, func(status *dbpreview.TLSStatus) { + status.Ready = true + status.SecretName = cert.Spec.SecretName + status.Message = "Gateway TLS certificate ready" + }); err != nil { + return ctrl.Result{}, err + } + } + return ctrl.Result{}, nil + } + } + + if err := r.updateTLSStatus(ctx, obj, func(status *dbpreview.TLSStatus) { + status.Ready = false + status.SecretName = cert.Spec.SecretName + status.Message = "Waiting for gateway TLS certificate to become ready" + }); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{RequeueAfter: RequeueAfterShort}, nil +} + +func (r *CertificateReconciler[T]) updateTLSStatus(ctx context.Context, obj T, mutate func(*dbpreview.TLSStatus)) error { + key := types.NamespacedName{Name: obj.GetName(), Namespace: obj.GetNamespace()} + return retry.RetryOnConflict(retry.DefaultRetry, func() error { + current := r.Adapter.New() + if err := r.Get(ctx, key, current); err != nil { + return err + } + status := r.Adapter.Status(current) + if status.TLS == nil { + status.TLS = &dbpreview.TLSStatus{} + } + mutate(status.TLS) + r.Adapter.SetStatus(current, status) + if err := r.Status().Update(ctx, current); err != nil { + return err + } + r.Adapter.SetStatus(obj, r.Adapter.Status(current)) + return nil + }) +} + +// SetupWithManager registers the reconciler as controller name. +func (r *CertificateReconciler[T]) SetupWithManager(mgr ctrl.Manager, name string) error { + return ctrl.NewControllerManagedBy(mgr). + For(r.Adapter.New()). + Owns(&cmapi.Certificate{}). + Owns(&cmapi.Issuer{}). + Named(name). + Complete(r) +} From eaf4409ae588394ed1405452e47bb630b45aa7b7 Mon Sep 17 00:00:00 2001 From: Rayhan Hossain Date: Mon, 19 Oct 2026 18:35:05 -0700 Subject: [PATCH 16/27] refactor: extract generic PersistentVolumeReconciler engine Move PV labeling, schema-version stamping, reclaim policy and security mount options into engine.PersistentVolumeReconciler[T]. The owner walk PV -> PVC -> CNPG Cluster -> custom resource matches the new NamingProfile.Kind. The DocumentDB PV controller delegates; behavior is unchanged. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 33b3cb3c-1b25-4ffb-bff1-ebf0f3d04058 Signed-off-by: Rayhan Hossain --- .../src/internal/controller/pv_controller.go | 434 +-------------- operator/src/internal/engine/pv.go | 497 ++++++++++++++++++ operator/src/internal/product/cluster.go | 3 + operator/src/internal/product/cluster_test.go | 1 + operator/src/internal/product/documentdb.go | 1 + 5 files changed, 527 insertions(+), 409 deletions(-) create mode 100644 operator/src/internal/engine/pv.go diff --git a/operator/src/internal/controller/pv_controller.go b/operator/src/internal/controller/pv_controller.go index 3abb0669..908fe027 100644 --- a/operator/src/internal/controller/pv_controller.go +++ b/operator/src/internal/controller/pv_controller.go @@ -5,73 +5,31 @@ package controller import ( "context" - "slices" - "strings" cnpgv1 "github.com/cloudnative-pg/cloudnative-pg/api/v1" corev1 "k8s.io/api/core/v1" - storagev1 "k8s.io/api/storage/v1" - "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/event" - "sigs.k8s.io/controller-runtime/pkg/handler" - "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/predicate" "sigs.k8s.io/controller-runtime/pkg/reconcile" dbpreview "github.com/documentdb/documentdb-operator/api/preview" - util "github.com/documentdb/documentdb-operator/internal/utils" + "github.com/documentdb/documentdb-operator/internal/engine" + "github.com/documentdb/documentdb-operator/internal/product" ) -const ( - // cnpgAPIVersionPrefix is the prefix for CNPG API versions - cnpgAPIVersionPrefix = "postgresql.cnpg.io" - - // ownerRefKindCluster is the Kind for CNPG Cluster owner references - ownerRefKindCluster = "Cluster" - - // ownerRefKindDocumentDB is the Kind for DocumentDB owner references - ownerRefKindDocumentDB = "DocumentDB" - - // reclaimPolicyRetain is the string value for Retain policy in DocumentDB spec - reclaimPolicyRetain = "Retain" - - // reclaimPolicyDelete is the string value for Delete policy in DocumentDB spec - reclaimPolicyDelete = "Delete" -) - -// securityMountOptions defines the mount options applied to PVs for security hardening: -// - nodev: Prevents device files from being interpreted on the filesystem -// - nosuid: Prevents setuid/setgid bits from taking effect -// - noexec: Prevents execution of binaries on the filesystem -// -// NOTE: These mount options are compatible with most CSI drivers and storage providers. -// However, some storage classes may not support these options, which could cause PV -// binding issues or pod startup failures. If you encounter issues with PV binding after -// deploying the operator, verify your storage provider supports these mount options. -// Common compatible providers: Azure Disk, AWS EBS, GCE PD, most NFS implementations. -var securityMountOptions = []string{"nodev", "noexec", "nosuid"} - -// unsupportedMountOptionsProvisioners lists storage provisioners that do not support -// mount options. These are typically local/dev environment provisioners used in -// kind, minikube, or similar local Kubernetes clusters. -// When a PV uses one of these provisioners, security mount options will be skipped -// to avoid PV binding failures. -var unsupportedMountOptionsProvisioners = []string{ - "rancher.io/local-path", // kind default local-path-provisioner - "k8s.io/minikube-hostpath", // minikube default hostpath provisioner -} - // PersistentVolumeReconciler reconciles PersistentVolume objects -// to set their ReclaimPolicy and mount options based on the associated DocumentDB configuration +// to set their ReclaimPolicy and mount options based on the associated DocumentDB configuration. +// The logic lives in engine.PersistentVolumeReconciler. type PersistentVolumeReconciler struct { client.Client } +func (r *PersistentVolumeReconciler) engine() *engine.PersistentVolumeReconciler[*dbpreview.DocumentDB] { + return &engine.PersistentVolumeReconciler[*dbpreview.DocumentDB]{Client: r.Client, Adapter: product.DocumentDBAdapter{}} +} + // Package-level RBAC also covers PVC lifecycle operations performed by the // DocumentDB reconciler. // +kubebuilder:rbac:groups="",resources=persistentvolumes,verbs=get;list;watch;update;patch @@ -79,418 +37,76 @@ type PersistentVolumeReconciler struct { // +kubebuilder:rbac:groups=storage.k8s.io,resources=storageclasses,verbs=get;list;watch func (r *PersistentVolumeReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { - logger := log.FromContext(ctx) - - // Fetch the PersistentVolume - pv := &corev1.PersistentVolume{} - if err := r.Get(ctx, req.NamespacedName, pv); err != nil { - if errors.IsNotFound(err) { - return ctrl.Result{}, nil - } - logger.Error(err, "Failed to get PersistentVolume") - return ctrl.Result{}, err - } - - // Find the associated DocumentDB through the ownership chain: - // PV -> PVC -> CNPG Cluster -> DocumentDB - documentdb, err := r.findDocumentDBForPV(ctx, pv) - if err != nil { - logger.Error(err, "Failed to find DocumentDB for PV") - return ctrl.Result{}, err - } - - if documentdb == nil { - logger.V(1).Info("PV is not associated with a DocumentDB cluster, skipping", "pv", pv.Name) - return ctrl.Result{}, nil - } - - // Apply desired configuration to PV - needsUpdate := r.applyDesiredPVConfiguration(ctx, pv, documentdb) - - if needsUpdate { - if err := r.Update(ctx, pv); err != nil { - logger.Error(err, "Failed to update PV") - return ctrl.Result{}, err - } - - logger.Info("Successfully updated PV", - "pv", pv.Name, - "reclaimPolicy", pv.Spec.PersistentVolumeReclaimPolicy, - "mountOptions", pv.Spec.MountOptions) - } - - return ctrl.Result{}, nil + return r.engine().Reconcile(ctx, req) } // applyDesiredPVConfiguration applies the desired reclaim policy, mount options, and labels to a PV. // Returns true if any changes were made. func (r *PersistentVolumeReconciler) applyDesiredPVConfiguration(ctx context.Context, pv *corev1.PersistentVolume, documentdb *dbpreview.DocumentDB) bool { - logger := log.FromContext(ctx) - needsUpdate := false - - // Ensure PV is labeled with the DocumentDB cluster name. - // This label is stable across single and multi-cluster scenarios (where the - // CNPG cluster name may differ from the DocumentDB name due to hashing). - if pv.Labels == nil { - pv.Labels = make(map[string]string) - } - if pv.Labels[util.LabelCluster] != documentdb.Name { - logger.Info("PV cluster label needs update", - "pv", pv.Name, - "currentLabel", pv.Labels[util.LabelCluster], - "desiredLabel", documentdb.Name, - "documentdb", documentdb.Name) - pv.Labels[util.LabelCluster] = documentdb.Name - needsUpdate = true - } - if pv.Labels[util.LabelNamespace] != documentdb.Namespace { - logger.Info("PV namespace label needs update", - "pv", pv.Name, - "currentLabel", pv.Labels[util.LabelNamespace], - "desiredLabel", documentdb.Namespace, - "documentdb", documentdb.Name) - pv.Labels[util.LabelNamespace] = documentdb.Namespace - needsUpdate = true - } - - // Stamp the installed schema version so a future PV restore can validate - // binary/schema compatibility at admission time. Only set it once the cluster - // has reported a schema version; never clear a previously stamped value (a - // transient empty status must not erase the last-known-good version, which is - // what a retained PV would be restored from). - if schemaVersion := documentdb.Status.SchemaVersion; schemaVersion != "" && - pv.Annotations[util.AnnotationSchemaVersion] != schemaVersion { - if pv.Annotations == nil { - pv.Annotations = make(map[string]string) - } - logger.Info("PV schema-version annotation needs update", - "pv", pv.Name, - "currentValue", pv.Annotations[util.AnnotationSchemaVersion], - "desiredValue", schemaVersion, - "documentdb", documentdb.Name) - pv.Annotations[util.AnnotationSchemaVersion] = schemaVersion - needsUpdate = true - } - - // Check if reclaim policy needs update - desiredPolicy := r.getDesiredReclaimPolicy(documentdb) - if pv.Spec.PersistentVolumeReclaimPolicy != desiredPolicy { - logger.Info("PV reclaim policy needs update", - "pv", pv.Name, - "currentPolicy", pv.Spec.PersistentVolumeReclaimPolicy, - "desiredPolicy", desiredPolicy, - "documentdb", documentdb.Name) - pv.Spec.PersistentVolumeReclaimPolicy = desiredPolicy - needsUpdate = true - } - - // Check if the storage provisioner supports mount options - // Skip mount options for local/dev provisioners (kind, minikube, etc.) - if r.provisionerSupportsMountOptions(ctx, pv) { - // Check if mount options need update - if !containsAllMountOptions(pv.Spec.MountOptions, securityMountOptions) { - logger.Info("PV mount options need update", - "pv", pv.Name, - "currentMountOptions", pv.Spec.MountOptions, - "desiredMountOptions", securityMountOptions) - pv.Spec.MountOptions = mergeMountOptions(pv.Spec.MountOptions, securityMountOptions) - needsUpdate = true - } - } else { - logger.V(1).Info("Skipping mount options for PV - provisioner does not support them", - "pv", pv.Name, - "storageClassName", pv.Spec.StorageClassName) - } - - return needsUpdate + return r.engine().ApplyDesiredPVConfiguration(ctx, pv, documentdb) } // provisionerSupportsMountOptions checks if the PV's storage class provisioner supports mount options. -// Returns false for known local/dev provisioners (kind, minikube, etc.) that don't support mount options. -// Returns true for production provisioners (Azure Disk, AWS EBS, etc.) or if the provisioner cannot be determined. func (r *PersistentVolumeReconciler) provisionerSupportsMountOptions(ctx context.Context, pv *corev1.PersistentVolume) bool { - logger := log.FromContext(ctx) - - // If no storage class is specified, assume mount options are supported (safer default for production) - if pv.Spec.StorageClassName == "" { - return true - } - - // Fetch the StorageClass to get the provisioner - storageClass := &storagev1.StorageClass{} - if err := r.Get(ctx, types.NamespacedName{Name: pv.Spec.StorageClassName}, storageClass); err != nil { - if errors.IsNotFound(err) { - logger.V(1).Info("StorageClass not found, assuming mount options are supported", - "storageClassName", pv.Spec.StorageClassName) - return true - } - logger.Error(err, "Failed to get StorageClass, assuming mount options are supported", - "storageClassName", pv.Spec.StorageClassName) - return true - } - - // Check if the provisioner is in the unsupported list - for _, unsupportedProvisioner := range unsupportedMountOptionsProvisioners { - if storageClass.Provisioner == unsupportedProvisioner { - logger.V(1).Info("Provisioner does not support mount options", - "provisioner", storageClass.Provisioner, - "storageClassName", pv.Spec.StorageClassName) - return false - } - } - - return true + return r.engine().ProvisionerSupportsMountOptions(ctx, pv) } // containsAllMountOptions checks if all desired mount options are present in current options func containsAllMountOptions(current, desired []string) bool { - for _, opt := range desired { - if !slices.Contains(current, opt) { - return false - } - } - return true + return engine.ContainsAllMountOptions(current, desired) } // mergeMountOptions merges desired mount options into current, avoiding duplicates. func mergeMountOptions(current, desired []string) []string { - optSet := make(map[string]struct{}, len(current)+len(desired)) - for _, opt := range current { - optSet[opt] = struct{}{} - } - for _, opt := range desired { - optSet[opt] = struct{}{} - } - - result := make([]string, 0, len(optSet)) - for opt := range optSet { - result = append(result, opt) - } - return result + return engine.MergeMountOptions(current, desired) } // findDocumentDBForPV traverses the ownership chain to find the DocumentDB // associated with a PersistentVolume: // PV.claimRef -> PVC -> (ownerRef) CNPG Cluster -> (ownerRef) DocumentDB func (r *PersistentVolumeReconciler) findDocumentDBForPV(ctx context.Context, pv *corev1.PersistentVolume) (*dbpreview.DocumentDB, error) { - logger := log.FromContext(ctx) - - // Step 1: Get the PVC from PV's claimRef - if pv.Spec.ClaimRef == nil { - return nil, nil - } - - pvc := &corev1.PersistentVolumeClaim{} - pvcKey := types.NamespacedName{ - Name: pv.Spec.ClaimRef.Name, - Namespace: pv.Spec.ClaimRef.Namespace, - } - if err := r.Get(ctx, pvcKey, pvc); err != nil { - if errors.IsNotFound(err) { - logger.V(1).Info("PVC not found for PV", "pvc", pvcKey, "pv", pv.Name) - return nil, nil - } - return nil, err - } - - // Step 2: Find CNPG Cluster that owns the PVC - cnpgCluster := r.findCNPGClusterOwner(ctx, pvc) - if cnpgCluster == nil { - logger.V(1).Info("No CNPG Cluster owner found for PVC", "pvc", pvc.Name) - return nil, nil - } - - // Step 3: Find DocumentDB that owns the CNPG Cluster - documentdb := r.findDocumentDBOwner(ctx, cnpgCluster) - if documentdb == nil { - logger.V(1).Info("No DocumentDB owner found for CNPG Cluster", "cluster", cnpgCluster.Name) - return nil, nil - } - - logger.V(1).Info("Found DocumentDB for PV", - "pv", pv.Name, - "pvc", pvc.Name, - "cluster", cnpgCluster.Name, - "documentdb", documentdb.Name) - - return documentdb, nil + documentdb, _, err := r.engine().FindOwnerForPV(ctx, pv) + return documentdb, err } // findCNPGClusterOwner finds the CNPG Cluster that owns the given PVC func (r *PersistentVolumeReconciler) findCNPGClusterOwner(ctx context.Context, pvc *corev1.PersistentVolumeClaim) *cnpgv1.Cluster { - logger := log.FromContext(ctx) - - for _, ownerRef := range pvc.OwnerReferences { - if !isCNPGClusterOwnerRef(ownerRef) { - continue - } - - cluster := &cnpgv1.Cluster{} - if err := r.Get(ctx, types.NamespacedName{ - Name: ownerRef.Name, - Namespace: pvc.Namespace, - }, cluster); err != nil { - if !errors.IsNotFound(err) { - logger.Error(err, "Failed to get CNPG Cluster", "name", ownerRef.Name) - } - continue - } - return cluster - } - - return nil + return r.engine().FindCNPGClusterOwner(ctx, pvc) } // findDocumentDBOwner finds the DocumentDB that owns the given CNPG Cluster func (r *PersistentVolumeReconciler) findDocumentDBOwner(ctx context.Context, cluster *cnpgv1.Cluster) *dbpreview.DocumentDB { - logger := log.FromContext(ctx) - - for _, ownerRef := range cluster.OwnerReferences { - if ownerRef.Kind != ownerRefKindDocumentDB { - continue - } - - documentdb := &dbpreview.DocumentDB{} - if err := r.Get(ctx, types.NamespacedName{ - Name: ownerRef.Name, - Namespace: cluster.Namespace, - }, documentdb); err != nil { - if !errors.IsNotFound(err) { - logger.Error(err, "Failed to get DocumentDB", "name", ownerRef.Name) - } - continue - } - return documentdb - } - - return nil + documentdb, _ := r.engine().FindClusterOwner(ctx, cluster) + return documentdb } // isCNPGClusterOwnerRef checks if an owner reference refers to a CNPG Cluster func isCNPGClusterOwnerRef(ownerRef metav1.OwnerReference) bool { - return ownerRef.Kind == ownerRefKindCluster && strings.Contains(ownerRef.APIVersion, cnpgAPIVersionPrefix) + return engine.IsCNPGClusterOwnerRef(ownerRef) } // getDesiredReclaimPolicy returns the reclaim policy based on DocumentDB configuration func (r *PersistentVolumeReconciler) getDesiredReclaimPolicy(documentdb *dbpreview.DocumentDB) corev1.PersistentVolumeReclaimPolicy { - switch documentdb.Spec.Resource.Storage.PersistentVolumeReclaimPolicy { - case reclaimPolicyRetain: - return corev1.PersistentVolumeReclaimRetain - case reclaimPolicyDelete: - return corev1.PersistentVolumeReclaimDelete - default: - // Default to Retain if not specified - safer for database workloads - return corev1.PersistentVolumeReclaimRetain - } + return engine.DesiredReclaimPolicy(documentdb.Spec.Resource.Storage.PersistentVolumeReclaimPolicy) } // pvPredicate filters PV events to only process bound PVs func pvPredicate() predicate.Predicate { - return predicate.Funcs{ - CreateFunc: func(e event.CreateEvent) bool { - pv, ok := e.Object.(*corev1.PersistentVolume) - if !ok { - return false - } - // Only process PVs that are bound and have a claimRef - return pv.Status.Phase == corev1.VolumeBound && pv.Spec.ClaimRef != nil - }, - UpdateFunc: func(e event.UpdateEvent) bool { - newPV, ok := e.ObjectNew.(*corev1.PersistentVolume) - if !ok { - return false - } - // Process when PV becomes bound or when claimRef changes - return newPV.Status.Phase == corev1.VolumeBound && newPV.Spec.ClaimRef != nil - }, - DeleteFunc: func(e event.DeleteEvent) bool { - // No need to reconcile deleted PVs - return false - }, - GenericFunc: func(e event.GenericEvent) bool { - pv, ok := e.Object.(*corev1.PersistentVolume) - if !ok { - return false - } - return pv.Status.Phase == corev1.VolumeBound && pv.Spec.ClaimRef != nil - }, - } + return engine.BoundPVPredicate() } // SetupWithManager sets up the controller with the Manager func (r *PersistentVolumeReconciler) SetupWithManager(mgr ctrl.Manager) error { - return ctrl.NewControllerManagedBy(mgr). - // Apply pvPredicate only to PersistentVolume events, not globally - For(&corev1.PersistentVolume{}, builder.WithPredicates(pvPredicate())). - // Watch DocumentDB changes and trigger reconciliation of associated PVs - // when the reclaim policy or the installed schema version changes. - Watches( - &dbpreview.DocumentDB{}, - handler.EnqueueRequestsFromMapFunc(r.findPVsForDocumentDB), - builder.WithPredicates(documentDBPVRelevantPredicate()), - ). - Named("pv-controller"). - Complete(r) + return r.engine().SetupWithManager(mgr, "pv-controller") } // documentDBPVRelevantPredicate triggers PV reconciliation when a DocumentDB -// change affects PV-level state: the reclaim policy (mutates the PV spec) or the -// installed schema version (stamped onto the PV as an annotation for restore -// compatibility validation). +// change affects the reclaim policy or the installed schema version. func documentDBPVRelevantPredicate() predicate.Predicate { - return predicate.Funcs{ - UpdateFunc: func(e event.UpdateEvent) bool { - oldDB, ok := e.ObjectOld.(*dbpreview.DocumentDB) - if !ok { - return false - } - newDB, ok := e.ObjectNew.(*dbpreview.DocumentDB) - if !ok { - return false - } - return oldDB.Spec.Resource.Storage.PersistentVolumeReclaimPolicy != newDB.Spec.Resource.Storage.PersistentVolumeReclaimPolicy || - oldDB.Status.SchemaVersion != newDB.Status.SchemaVersion - }, - CreateFunc: func(e event.CreateEvent) bool { return false }, - DeleteFunc: func(e event.DeleteEvent) bool { return false }, - GenericFunc: func(e event.GenericEvent) bool { return false }, - } + return (&PersistentVolumeReconciler{}).engine().PVRelevantPredicate() } // findPVsForDocumentDB finds all PVs associated with a DocumentDB and returns reconcile requests for them. -// Uses the documentdb.io/cluster and documentdb.io/namespace labels on PVs, which is set by the PV controller. -// This works correctly in both single and multi-cluster scenarios where CNPG -// cluster names may differ from the DocumentDB name. func (r *PersistentVolumeReconciler) findPVsForDocumentDB(ctx context.Context, obj client.Object) []reconcile.Request { - logger := log.FromContext(ctx) - documentdb, ok := obj.(*dbpreview.DocumentDB) - if !ok { - return nil - } - - pvList := &corev1.PersistentVolumeList{} - if err := r.List(ctx, pvList, - client.MatchingLabels{ - util.LabelCluster: documentdb.Name, - util.LabelNamespace: documentdb.Namespace, - }, - ); err != nil { - logger.Error(err, "Failed to list PVs for DocumentDB") - return nil - } - - requests := make([]reconcile.Request, 0, len(pvList.Items)) - for _, pv := range pvList.Items { - requests = append(requests, reconcile.Request{ - NamespacedName: types.NamespacedName{ - Name: pv.Name, - }, - }) - } - - logger.Info("Found PVs to reconcile for DocumentDB update", - "documentdb", documentdb.Name, - "pvCount", len(requests)) - - return requests + return r.engine().MapClusterToPVs(ctx, obj) } diff --git a/operator/src/internal/engine/pv.go b/operator/src/internal/engine/pv.go new file mode 100644 index 00000000..688ce14b --- /dev/null +++ b/operator/src/internal/engine/pv.go @@ -0,0 +1,497 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package engine + +import ( + "context" + "slices" + "strings" + + cnpgv1 "github.com/cloudnative-pg/cloudnative-pg/api/v1" + corev1 "k8s.io/api/core/v1" + storagev1 "k8s.io/api/storage/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/builder" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/predicate" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + "github.com/documentdb/documentdb-operator/internal/product" +) + +const ( + // cnpgAPIVersionPrefix is the prefix for CNPG API versions + cnpgAPIVersionPrefix = "postgresql.cnpg.io" + + // ownerRefKindCluster is the Kind for CNPG Cluster owner references + ownerRefKindCluster = "Cluster" +) + +// SecurityMountOptions defines the mount options applied to PVs for security hardening: +// - nodev: Prevents device files from being interpreted on the filesystem +// - nosuid: Prevents setuid/setgid bits from taking effect +// - noexec: Prevents execution of binaries on the filesystem +// +// NOTE: These mount options are compatible with most CSI drivers and storage providers. +// However, some storage classes may not support these options, which could cause PV +// binding issues or pod startup failures. If you encounter issues with PV binding after +// deploying the operator, verify your storage provider supports these mount options. +// Common compatible providers: Azure Disk, AWS EBS, GCE PD, most NFS implementations. +var SecurityMountOptions = []string{"nodev", "noexec", "nosuid"} + +// unsupportedMountOptionsProvisioners lists storage provisioners that do not support +// mount options. These are typically local/dev environment provisioners used in +// kind, minikube, or similar local Kubernetes clusters. +// When a PV uses one of these provisioners, security mount options will be skipped +// to avoid PV binding failures. +var unsupportedMountOptionsProvisioners = []string{ + "rancher.io/local-path", // kind default local-path-provisioner + "k8s.io/minikube-hostpath", // minikube default hostpath provisioner +} + +// PersistentVolumeReconciler sets the reclaim policy, security mount options, +// ownership labels and schema-version annotation of PVs that back a product +// cluster. +type PersistentVolumeReconciler[T client.Object] struct { + client.Client + Adapter product.ClusterAdapter[T] +} + +func (r *PersistentVolumeReconciler[T]) naming() product.NamingProfile { + return r.Adapter.Profile().Naming +} + +// Reconcile applies the desired configuration to one PersistentVolume. +func (r *PersistentVolumeReconciler[T]) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + logger := log.FromContext(ctx) + kind := r.naming().Kind + + // Fetch the PersistentVolume + pv := &corev1.PersistentVolume{} + if err := r.Get(ctx, req.NamespacedName, pv); err != nil { + if errors.IsNotFound(err) { + return ctrl.Result{}, nil + } + logger.Error(err, "Failed to get PersistentVolume") + return ctrl.Result{}, err + } + + // Find the associated custom resource through the ownership chain: + // PV -> PVC -> CNPG Cluster -> custom resource + obj, found, err := r.FindOwnerForPV(ctx, pv) + if err != nil { + logger.Error(err, "Failed to find "+kind+" for PV") + return ctrl.Result{}, err + } + + if !found { + logger.V(1).Info("PV is not associated with a "+kind+" cluster, skipping", "pv", pv.Name) + return ctrl.Result{}, nil + } + + // Apply desired configuration to PV + needsUpdate := r.ApplyDesiredPVConfiguration(ctx, pv, obj) + + if needsUpdate { + if err := r.Update(ctx, pv); err != nil { + logger.Error(err, "Failed to update PV") + return ctrl.Result{}, err + } + + logger.Info("Successfully updated PV", + "pv", pv.Name, + "reclaimPolicy", pv.Spec.PersistentVolumeReclaimPolicy, + "mountOptions", pv.Spec.MountOptions) + } + + return ctrl.Result{}, nil +} + +// ApplyDesiredPVConfiguration applies the desired reclaim policy, mount options, and labels to a PV. +// Returns true if any changes were made. +func (r *PersistentVolumeReconciler[T]) ApplyDesiredPVConfiguration(ctx context.Context, pv *corev1.PersistentVolume, obj T) bool { + logger := log.FromContext(ctx) + naming := r.naming() + ownerKey := strings.ToLower(naming.Kind) + clusterLabel, namespaceLabel, schemaAnnotation := naming.ClusterLabel(), naming.NamespaceLabel(), naming.SchemaVersionAnnotation() + needsUpdate := false + + // Ensure PV is labeled with the custom resource name. + // This label is stable across single and multi-cluster scenarios (where the + // CNPG cluster name may differ from the custom resource name due to hashing). + if pv.Labels == nil { + pv.Labels = make(map[string]string) + } + if pv.Labels[clusterLabel] != obj.GetName() { + logger.Info("PV cluster label needs update", + "pv", pv.Name, + "currentLabel", pv.Labels[clusterLabel], + "desiredLabel", obj.GetName(), + ownerKey, obj.GetName()) + pv.Labels[clusterLabel] = obj.GetName() + needsUpdate = true + } + if pv.Labels[namespaceLabel] != obj.GetNamespace() { + logger.Info("PV namespace label needs update", + "pv", pv.Name, + "currentLabel", pv.Labels[namespaceLabel], + "desiredLabel", obj.GetNamespace(), + ownerKey, obj.GetName()) + pv.Labels[namespaceLabel] = obj.GetNamespace() + needsUpdate = true + } + + // Stamp the installed schema version so a future PV restore can validate + // binary/schema compatibility at admission time. Only set it once the cluster + // has reported a schema version; never clear a previously stamped value (a + // transient empty status must not erase the last-known-good version, which is + // what a retained PV would be restored from). + if schemaVersion := r.Adapter.Status(obj).SchemaVersion; schemaVersion != "" && + pv.Annotations[schemaAnnotation] != schemaVersion { + if pv.Annotations == nil { + pv.Annotations = make(map[string]string) + } + logger.Info("PV schema-version annotation needs update", + "pv", pv.Name, + "currentValue", pv.Annotations[schemaAnnotation], + "desiredValue", schemaVersion, + ownerKey, obj.GetName()) + pv.Annotations[schemaAnnotation] = schemaVersion + needsUpdate = true + } + + // Check if reclaim policy needs update + desiredPolicy := DesiredReclaimPolicy(r.Adapter.Spec(obj).PersistentVolumeReclaimPolicy) + if pv.Spec.PersistentVolumeReclaimPolicy != desiredPolicy { + logger.Info("PV reclaim policy needs update", + "pv", pv.Name, + "currentPolicy", pv.Spec.PersistentVolumeReclaimPolicy, + "desiredPolicy", desiredPolicy, + ownerKey, obj.GetName()) + pv.Spec.PersistentVolumeReclaimPolicy = desiredPolicy + needsUpdate = true + } + + // Check if the storage provisioner supports mount options + // Skip mount options for local/dev provisioners (kind, minikube, etc.) + if r.ProvisionerSupportsMountOptions(ctx, pv) { + // Check if mount options need update + if !ContainsAllMountOptions(pv.Spec.MountOptions, SecurityMountOptions) { + logger.Info("PV mount options need update", + "pv", pv.Name, + "currentMountOptions", pv.Spec.MountOptions, + "desiredMountOptions", SecurityMountOptions) + pv.Spec.MountOptions = MergeMountOptions(pv.Spec.MountOptions, SecurityMountOptions) + needsUpdate = true + } + } else { + logger.V(1).Info("Skipping mount options for PV - provisioner does not support them", + "pv", pv.Name, + "storageClassName", pv.Spec.StorageClassName) + } + + return needsUpdate +} + +// ProvisionerSupportsMountOptions checks if the PV's storage class provisioner supports mount options. +// Returns false for known local/dev provisioners (kind, minikube, etc.) that don't support mount options. +// Returns true for production provisioners (Azure Disk, AWS EBS, etc.) or if the provisioner cannot be determined. +func (r *PersistentVolumeReconciler[T]) ProvisionerSupportsMountOptions(ctx context.Context, pv *corev1.PersistentVolume) bool { + logger := log.FromContext(ctx) + + // If no storage class is specified, assume mount options are supported (safer default for production) + if pv.Spec.StorageClassName == "" { + return true + } + + // Fetch the StorageClass to get the provisioner + storageClass := &storagev1.StorageClass{} + if err := r.Get(ctx, types.NamespacedName{Name: pv.Spec.StorageClassName}, storageClass); err != nil { + if errors.IsNotFound(err) { + logger.V(1).Info("StorageClass not found, assuming mount options are supported", + "storageClassName", pv.Spec.StorageClassName) + return true + } + logger.Error(err, "Failed to get StorageClass, assuming mount options are supported", + "storageClassName", pv.Spec.StorageClassName) + return true + } + + // Check if the provisioner is in the unsupported list + for _, unsupportedProvisioner := range unsupportedMountOptionsProvisioners { + if storageClass.Provisioner == unsupportedProvisioner { + logger.V(1).Info("Provisioner does not support mount options", + "provisioner", storageClass.Provisioner, + "storageClassName", pv.Spec.StorageClassName) + return false + } + } + + return true +} + +// ContainsAllMountOptions checks if all desired mount options are present in current options +func ContainsAllMountOptions(current, desired []string) bool { + for _, opt := range desired { + if !slices.Contains(current, opt) { + return false + } + } + return true +} + +// MergeMountOptions merges desired mount options into current, avoiding duplicates. +func MergeMountOptions(current, desired []string) []string { + optSet := make(map[string]struct{}, len(current)+len(desired)) + for _, opt := range current { + optSet[opt] = struct{}{} + } + for _, opt := range desired { + optSet[opt] = struct{}{} + } + + result := make([]string, 0, len(optSet)) + for opt := range optSet { + result = append(result, opt) + } + return result +} + +// FindOwnerForPV traverses the ownership chain to find the custom resource +// associated with a PersistentVolume: +// PV.claimRef -> PVC -> (ownerRef) CNPG Cluster -> (ownerRef) custom resource +func (r *PersistentVolumeReconciler[T]) FindOwnerForPV(ctx context.Context, pv *corev1.PersistentVolume) (T, bool, error) { + logger := log.FromContext(ctx) + kind := r.naming().Kind + var none T + + // Step 1: Get the PVC from PV's claimRef + if pv.Spec.ClaimRef == nil { + return none, false, nil + } + + pvc := &corev1.PersistentVolumeClaim{} + pvcKey := types.NamespacedName{ + Name: pv.Spec.ClaimRef.Name, + Namespace: pv.Spec.ClaimRef.Namespace, + } + if err := r.Get(ctx, pvcKey, pvc); err != nil { + if errors.IsNotFound(err) { + logger.V(1).Info("PVC not found for PV", "pvc", pvcKey, "pv", pv.Name) + return none, false, nil + } + return none, false, err + } + + // Step 2: Find CNPG Cluster that owns the PVC + cnpgCluster := r.FindCNPGClusterOwner(ctx, pvc) + if cnpgCluster == nil { + logger.V(1).Info("No CNPG Cluster owner found for PVC", "pvc", pvc.Name) + return none, false, nil + } + + // Step 3: Find the custom resource that owns the CNPG Cluster + obj, found := r.FindClusterOwner(ctx, cnpgCluster) + if !found { + logger.V(1).Info("No "+kind+" owner found for CNPG Cluster", "cluster", cnpgCluster.Name) + return none, false, nil + } + + logger.V(1).Info("Found "+kind+" for PV", + "pv", pv.Name, + "pvc", pvc.Name, + "cluster", cnpgCluster.Name, + strings.ToLower(kind), obj.GetName()) + + return obj, true, nil +} + +// FindCNPGClusterOwner finds the CNPG Cluster that owns the given PVC +func (r *PersistentVolumeReconciler[T]) FindCNPGClusterOwner(ctx context.Context, pvc *corev1.PersistentVolumeClaim) *cnpgv1.Cluster { + logger := log.FromContext(ctx) + + for _, ownerRef := range pvc.OwnerReferences { + if !IsCNPGClusterOwnerRef(ownerRef) { + continue + } + + cluster := &cnpgv1.Cluster{} + if err := r.Get(ctx, types.NamespacedName{ + Name: ownerRef.Name, + Namespace: pvc.Namespace, + }, cluster); err != nil { + if !errors.IsNotFound(err) { + logger.Error(err, "Failed to get CNPG Cluster", "name", ownerRef.Name) + } + continue + } + return cluster + } + + return nil +} + +// FindClusterOwner finds the product custom resource that owns the given CNPG Cluster. +func (r *PersistentVolumeReconciler[T]) FindClusterOwner(ctx context.Context, cluster *cnpgv1.Cluster) (T, bool) { + logger := log.FromContext(ctx) + kind := r.naming().Kind + var none T + + for _, ownerRef := range cluster.OwnerReferences { + if ownerRef.Kind != kind { + continue + } + + obj := r.Adapter.New() + if err := r.Get(ctx, types.NamespacedName{ + Name: ownerRef.Name, + Namespace: cluster.Namespace, + }, obj); err != nil { + if !errors.IsNotFound(err) { + logger.Error(err, "Failed to get "+kind, "name", ownerRef.Name) + } + continue + } + return obj, true + } + + return none, false +} + +// IsCNPGClusterOwnerRef checks if an owner reference refers to a CNPG Cluster +func IsCNPGClusterOwnerRef(ownerRef metav1.OwnerReference) bool { + return ownerRef.Kind == ownerRefKindCluster && strings.Contains(ownerRef.APIVersion, cnpgAPIVersionPrefix) +} + +// DesiredReclaimPolicy maps the spec reclaim policy to the PV reclaim policy; +// anything but "Delete" is Retain, the safer default for database workloads. +func DesiredReclaimPolicy(policy string) corev1.PersistentVolumeReclaimPolicy { + switch policy { + case "Retain": + return corev1.PersistentVolumeReclaimRetain + case "Delete": + return corev1.PersistentVolumeReclaimDelete + default: + // Default to Retain if not specified - safer for database workloads + return corev1.PersistentVolumeReclaimRetain + } +} + +// BoundPVPredicate filters PV events to only process bound PVs +func BoundPVPredicate() predicate.Predicate { + return predicate.Funcs{ + CreateFunc: func(e event.CreateEvent) bool { + pv, ok := e.Object.(*corev1.PersistentVolume) + if !ok { + return false + } + // Only process PVs that are bound and have a claimRef + return pv.Status.Phase == corev1.VolumeBound && pv.Spec.ClaimRef != nil + }, + UpdateFunc: func(e event.UpdateEvent) bool { + newPV, ok := e.ObjectNew.(*corev1.PersistentVolume) + if !ok { + return false + } + // Process when PV becomes bound or when claimRef changes + return newPV.Status.Phase == corev1.VolumeBound && newPV.Spec.ClaimRef != nil + }, + DeleteFunc: func(e event.DeleteEvent) bool { + // No need to reconcile deleted PVs + return false + }, + GenericFunc: func(e event.GenericEvent) bool { + pv, ok := e.Object.(*corev1.PersistentVolume) + if !ok { + return false + } + return pv.Status.Phase == corev1.VolumeBound && pv.Spec.ClaimRef != nil + }, + } +} + +// PVRelevantPredicate triggers PV reconciliation when a custom resource change +// affects PV-level state: the reclaim policy (mutates the PV spec) or the +// installed schema version (stamped onto the PV as an annotation for restore +// compatibility validation). +func (r *PersistentVolumeReconciler[T]) PVRelevantPredicate() predicate.Predicate { + return predicate.Funcs{ + UpdateFunc: func(e event.UpdateEvent) bool { + oldObj, ok := e.ObjectOld.(T) + if !ok { + return false + } + newObj, ok := e.ObjectNew.(T) + if !ok { + return false + } + return r.Adapter.Spec(oldObj).PersistentVolumeReclaimPolicy != r.Adapter.Spec(newObj).PersistentVolumeReclaimPolicy || + r.Adapter.Status(oldObj).SchemaVersion != r.Adapter.Status(newObj).SchemaVersion + }, + CreateFunc: func(e event.CreateEvent) bool { return false }, + DeleteFunc: func(e event.DeleteEvent) bool { return false }, + GenericFunc: func(e event.GenericEvent) bool { return false }, + } +} + +// MapClusterToPVs returns reconcile requests for all PVs labeled with the +// custom resource. The cluster/namespace labels are set by this reconciler and +// work in both single and multi-cluster scenarios where CNPG cluster names may +// differ from the custom resource name. +func (r *PersistentVolumeReconciler[T]) MapClusterToPVs(ctx context.Context, o client.Object) []reconcile.Request { + logger := log.FromContext(ctx) + naming := r.naming() + obj, ok := o.(T) + if !ok { + return nil + } + + pvList := &corev1.PersistentVolumeList{} + if err := r.List(ctx, pvList, + client.MatchingLabels{ + naming.ClusterLabel(): obj.GetName(), + naming.NamespaceLabel(): obj.GetNamespace(), + }, + ); err != nil { + logger.Error(err, "Failed to list PVs for "+naming.Kind) + return nil + } + + requests := make([]reconcile.Request, 0, len(pvList.Items)) + for _, pv := range pvList.Items { + requests = append(requests, reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: pv.Name, + }, + }) + } + + logger.Info("Found PVs to reconcile for "+naming.Kind+" update", + strings.ToLower(naming.Kind), obj.GetName(), + "pvCount", len(requests)) + + return requests +} + +// SetupWithManager registers the reconciler as controller name. +func (r *PersistentVolumeReconciler[T]) SetupWithManager(mgr ctrl.Manager, name string) error { + return ctrl.NewControllerManagedBy(mgr). + // Apply BoundPVPredicate only to PersistentVolume events, not globally + For(&corev1.PersistentVolume{}, builder.WithPredicates(BoundPVPredicate())). + // Watch custom resource changes and trigger reconciliation of associated PVs + // when the reclaim policy or the installed schema version changes. + Watches( + r.Adapter.New(), + handler.EnqueueRequestsFromMapFunc(r.MapClusterToPVs), + builder.WithPredicates(r.PVRelevantPredicate()), + ). + Named(name). + Complete(r) +} diff --git a/operator/src/internal/product/cluster.go b/operator/src/internal/product/cluster.go index c68030a0..9d23cd72 100644 --- a/operator/src/internal/product/cluster.go +++ b/operator/src/internal/product/cluster.go @@ -24,6 +24,9 @@ type NamingProfile struct { MetricClusterAttribute string // OperatorName identifies the operator in generated artifacts. OperatorName string + // Kind is the custom resource kind; it identifies the product owner + // reference on CNPG clusters. + Kind string } // ClusterLabel is the PV/PVC label holding the owning custom resource name. diff --git a/operator/src/internal/product/cluster_test.go b/operator/src/internal/product/cluster_test.go index bf349143..789dd741 100644 --- a/operator/src/internal/product/cluster_test.go +++ b/operator/src/internal/product/cluster_test.go @@ -22,6 +22,7 @@ func TestDocumentDBNamingMatchesWireContract(t *testing.T) { n.SchemaVersionAnnotation(): util.AnnotationSchemaVersion, n.Finalizer(): "documentdb.io/pv-retention-finalizer", n.ServicePrefix: util.DOCUMENTDB_SERVICE_PREFIX, + n.Kind: "DocumentDB", } { if got != want { t.Errorf("got %q, want %q", got, want) diff --git a/operator/src/internal/product/documentdb.go b/operator/src/internal/product/documentdb.go index 8e262797..dbccbf8a 100644 --- a/operator/src/internal/product/documentdb.go +++ b/operator/src/internal/product/documentdb.go @@ -49,6 +49,7 @@ func DocumentDBProfile() ProductProfile { ServicePrefix: util.DOCUMENTDB_SERVICE_PREFIX, MetricClusterAttribute: "documentdb.cluster", OperatorName: "documentdb-operator", + Kind: "DocumentDB", }, } } From 1ef401cd4138efc61065e93abadd2520b745905c Mon Sep 17 00:00:00 2001 From: Rayhan Hossain Date: Mon, 19 Oct 2026 18:37:46 -0700 Subject: [PATCH 17/27] refactor: extract generic backup and scheduled-backup engines Move backup and scheduled-backup reconciliation into engine.BackupReconciler[T,B] and engine.ScheduledBackupReconciler[T,S,B]. Product Backup and ScheduledBackup CRs expose the shared preview spec and status types through small adapters. Pure backup logic becomes dbpreview package functions (NewCNPGBackup, UpdateBackupStatus, BackupExpirationTime, NextScheduleTime, ScheduledBackupName) and the existing methods delegate. The DocumentDB controllers delegate; behavior is unchanged. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 33b3cb3c-1b25-4ffb-bff1-ebf0f3d04058 Signed-off-by: Rayhan Hossain --- operator/src/api/preview/backup_funcs.go | 65 ++-- .../src/api/preview/scheduledbackup_funcs.go | 34 +- .../internal/controller/backup_controller.go | 247 ++------------ .../controller/scheduledbackup_controller.go | 141 +------- operator/src/internal/engine/backup.go | 301 ++++++++++++++++++ .../src/internal/engine/scheduledbackup.go | 220 +++++++++++++ .../src/internal/product/documentdb_backup.go | 51 +++ 7 files changed, 676 insertions(+), 383 deletions(-) create mode 100644 operator/src/internal/engine/backup.go create mode 100644 operator/src/internal/engine/scheduledbackup.go create mode 100644 operator/src/internal/product/documentdb_backup.go diff --git a/operator/src/api/preview/backup_funcs.go b/operator/src/api/preview/backup_funcs.go index 0c4d1fe9..a59372a8 100644 --- a/operator/src/api/preview/backup_funcs.go +++ b/operator/src/api/preview/backup_funcs.go @@ -9,15 +9,23 @@ import ( cnpgv1 "github.com/cloudnative-pg/cloudnative-pg/api/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" ) // CreateCNPGBackup creates a CNPG Backup resource based on the DocumentDB Backup spec. func (backup *Backup) CreateCNPGBackup(scheme *runtime.Scheme, clusterName string) (*cnpgv1.Backup, error) { + return NewCNPGBackup(scheme, backup, clusterName) +} + +// NewCNPGBackup creates a volume-snapshot CNPG Backup of clusterName with the +// same name and namespace as owner, controlled by owner so it is garbage +// collected with it. +func NewCNPGBackup(scheme *runtime.Scheme, owner client.Object, clusterName string) (*cnpgv1.Backup, error) { cnpgBackup := &cnpgv1.Backup{ ObjectMeta: metav1.ObjectMeta{ - Name: backup.Name, - Namespace: backup.Namespace, + Name: owner.GetName(), + Namespace: owner.GetNamespace(), }, Spec: cnpgv1.BackupSpec{ Method: cnpgv1.BackupMethodVolumeSnapshot, @@ -27,8 +35,8 @@ func (backup *Backup) CreateCNPGBackup(scheme *runtime.Scheme, clusterName strin }, } // Set owner reference for garbage collection - // This ensures that the CNPG Backup is deleted when the DocumentDB Backup is deleted. - if err := controllerutil.SetControllerReference(backup, cnpgBackup, scheme); err != nil { + // This ensures that the CNPG Backup is deleted when the owning Backup is deleted. + if err := controllerutil.SetControllerReference(owner, cnpgBackup, scheme); err != nil { return nil, err } return cnpgBackup, nil @@ -37,38 +45,44 @@ func (backup *Backup) CreateCNPGBackup(scheme *runtime.Scheme, clusterName strin // UpdateStatus reconciles the Backup status from the observed inputs and returns // whether any status field changed. func (backup *Backup) UpdateStatus(cnpgBackup *cnpgv1.Backup, backupConfiguration *BackupConfiguration, sourceSchemaVersion string) bool { + return UpdateBackupStatus(&backup.Spec, &backup.Status, backup.CreationTimestamp, cnpgBackup, backupConfiguration, sourceSchemaVersion) +} + +// UpdateBackupStatus reconciles status from the observed CNPG backup and +// returns whether any status field changed. +func UpdateBackupStatus(spec *BackupSpec, status *BackupStatus, created metav1.Time, cnpgBackup *cnpgv1.Backup, backupConfiguration *BackupConfiguration, sourceSchemaVersion string) bool { needsUpdate := false - if backup.Status.Phase != cnpgBackup.Status.Phase { - backup.Status.Phase = cnpgBackup.Status.Phase + if status.Phase != cnpgBackup.Status.Phase { + status.Phase = cnpgBackup.Status.Phase needsUpdate = true } - if !areTimesEqual(backup.Status.StartedAt, cnpgBackup.Status.StartedAt) { - backup.Status.StartedAt = cnpgBackup.Status.StartedAt + if !areTimesEqual(status.StartedAt, cnpgBackup.Status.StartedAt) { + status.StartedAt = cnpgBackup.Status.StartedAt needsUpdate = true } - if !areTimesEqual(backup.Status.StoppedAt, cnpgBackup.Status.StoppedAt) { - backup.Status.StoppedAt = cnpgBackup.Status.StoppedAt + if !areTimesEqual(status.StoppedAt, cnpgBackup.Status.StoppedAt) { + status.StoppedAt = cnpgBackup.Status.StoppedAt needsUpdate = true } - if backup.Status.Message != cnpgBackup.Status.Error { - backup.Status.Message = cnpgBackup.Status.Error + if status.Message != cnpgBackup.Status.Error { + status.Message = cnpgBackup.Status.Error needsUpdate = true } - expirationTime := backup.CalculateExpirationTime(backupConfiguration) - if !areTimesEqual(backup.Status.ExpiredAt, expirationTime) { - backup.Status.ExpiredAt = expirationTime + expirationTime := BackupExpirationTime(spec, status, created, backupConfiguration) + if !areTimesEqual(status.ExpiredAt, expirationTime) { + status.ExpiredAt = expirationTime needsUpdate = true } // Record the source cluster's schema version at backup time: captured once // when first available and left immutable afterward, so a later restore can // validate binary-vs-schema compatibility. - if backup.Status.SchemaVersion == "" && sourceSchemaVersion != "" { - backup.Status.SchemaVersion = sourceSchemaVersion + if status.SchemaVersion == "" && sourceSchemaVersion != "" { + status.SchemaVersion = sourceSchemaVersion needsUpdate = true } @@ -77,13 +91,20 @@ func (backup *Backup) UpdateStatus(cnpgBackup *cnpgv1.Backup, backupConfiguratio // CalculateExpirationTime calculates the expiration time of the backup based on retention policy. func (backup *Backup) CalculateExpirationTime(backupConfiguration *BackupConfiguration) *metav1.Time { - if !backup.Status.IsDone() { + return BackupExpirationTime(&backup.Spec, &backup.Status, backup.CreationTimestamp, backupConfiguration) +} + +// BackupExpirationTime calculates when a done backup expires from its +// retention days, the cluster backup configuration, or the 30-day default. +// It returns nil while the backup is still running. +func BackupExpirationTime(spec *BackupSpec, status *BackupStatus, created metav1.Time, backupConfiguration *BackupConfiguration) *metav1.Time { + if !status.IsDone() { return nil } retentionHours := 0 - if backup.Spec.RetentionDays != nil { - retentionHours = *backup.Spec.RetentionDays * 24 + if spec.RetentionDays != nil { + retentionHours = *spec.RetentionDays * 24 } else if backupConfiguration != nil { retentionHours = backupConfiguration.RetentionDays * 24 } else { @@ -93,9 +114,9 @@ func (backup *Backup) CalculateExpirationTime(backupConfiguration *BackupConfigu // Determine the start time for retention calculation // If backup completed, use StoppedAt; // If backup failed, StoppedAt is not set, use CreationTimestamp - retentionStart := backup.Status.StoppedAt + retentionStart := status.StoppedAt if retentionStart == nil { - retentionStart = &backup.CreationTimestamp + retentionStart = &created } expirationTime := retentionStart.Time.Add(time.Duration(retentionHours) * time.Hour) diff --git a/operator/src/api/preview/scheduledbackup_funcs.go b/operator/src/api/preview/scheduledbackup_funcs.go index 5a0e388a..4a6e84e5 100644 --- a/operator/src/api/preview/scheduledbackup_funcs.go +++ b/operator/src/api/preview/scheduledbackup_funcs.go @@ -14,15 +14,12 @@ import ( // CreateBackup generates a new Backup resource for this ScheduledBackup. // The backup name is generated with a timestamp suffix to ensure uniqueness. func (scheduledBackup *ScheduledBackup) CreateBackup(now time.Time) *Backup { - // Generate backup name with timestamp - backupName := fmt.Sprintf("%s-%s", scheduledBackup.Name, now.Format("20060102-150405")) - return &Backup{ ObjectMeta: metav1.ObjectMeta{ - Name: backupName, + Name: ScheduledBackupName(scheduledBackup.Name, now), Namespace: scheduledBackup.Namespace, Labels: map[string]string{ - "scheduledbackup": scheduledBackup.Name, + ScheduledBackupLabel: scheduledBackup.Name, }, }, Spec: BackupSpec{ @@ -32,15 +29,34 @@ func (scheduledBackup *ScheduledBackup) CreateBackup(now time.Time) *Backup { } } +// ScheduledBackupLabel labels a Backup with the ScheduledBackup that created it. +const ScheduledBackupLabel = "scheduledbackup" + +// ScheduledBackupName generates the name of the Backup a ScheduledBackup +// creates at now, with a timestamp suffix to ensure uniqueness. +func ScheduledBackupName(scheduledBackupName string, now time.Time) string { + return fmt.Sprintf("%s-%s", scheduledBackupName, now.Format("20060102-150405")) +} + // GetNextScheduleTime calculates the next scheduled time func (scheduledBackup *ScheduledBackup) GetNextScheduleTime(schedule cron.Schedule, lastBackup *Backup) time.Time { + var lastBackupCreated *metav1.Time + if lastBackup != nil { + lastBackupCreated = &lastBackup.CreationTimestamp + } + return NextScheduleTime(schedule, &scheduledBackup.Status, lastBackupCreated) +} + +// NextScheduleTime calculates the next scheduled time from the creation time +// of the last backup, the recorded next time, or now. +func NextScheduleTime(schedule cron.Schedule, status *ScheduledBackupStatus, lastBackupCreated *metav1.Time) time.Time { // If there is a last backup, calculate the next schedule time based on its creation time - if lastBackup != nil && lastBackup.CreationTimestamp.Time.After(time.Time{}) { - return schedule.Next(lastBackup.CreationTimestamp.Time) + if lastBackupCreated != nil && lastBackupCreated.Time.After(time.Time{}) { + return schedule.Next(lastBackupCreated.Time) } - if scheduledBackup.Status.NextScheduledTime != nil { - return scheduledBackup.Status.NextScheduledTime.Time + if status.NextScheduledTime != nil { + return status.NextScheduledTime.Time } return schedule.Next(time.Now()) diff --git a/operator/src/internal/controller/backup_controller.go b/operator/src/internal/controller/backup_controller.go index ebb2b279..43cc4542 100644 --- a/operator/src/internal/controller/backup_controller.go +++ b/operator/src/internal/controller/backup_controller.go @@ -5,269 +5,62 @@ package controller import ( "context" - "fmt" - "time" cnpgv1 "github.com/cloudnative-pg/cloudnative-pg/api/v1" - snapshotv1 "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/tools/record" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/log" dbpreview "github.com/documentdb/documentdb-operator/api/preview" + "github.com/documentdb/documentdb-operator/internal/engine" + "github.com/documentdb/documentdb-operator/internal/product" util "github.com/documentdb/documentdb-operator/internal/utils" ) -// BackupReconciler reconciles a Backup object +// BackupReconciler reconciles a Backup object. The logic lives in +// engine.BackupReconciler. type BackupReconciler struct { client.Client Scheme *runtime.Scheme Recorder record.EventRecorder } -// Reconcile handles the reconciliation loop for Backup resources. -func (r *BackupReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { - logger := log.FromContext(ctx) - - // Fetch the Backup resource - backup := &dbpreview.Backup{} - if err := r.Get(ctx, req.NamespacedName, backup); err != nil { - if apierrors.IsNotFound(err) { - logger.Info("Backup resource not found, might have been deleted") - return ctrl.Result{}, nil - } - logger.Error(err, "Failed to get Backup") - return ctrl.Result{}, err - } - - // Delete the Backup resource if it has expired - if backup.Status.IsExpired() { - r.Recorder.Event(backup, "Normal", "BackupExpired", "Backup has expired and will be deleted") - if err := r.Delete(ctx, backup); err != nil { - r.Recorder.Event(backup, "Warning", "BackupDeleteFailed", "Failed to delete expired Backup: "+err.Error()) - return ctrl.Result{}, err - } - r.Recorder.Event(backup, "Normal", "BackupDeleted", "Expired Backup has been deleted") - return ctrl.Result{}, nil - } - - // If the backup is already done and not expired, requeue to check expiration - if backup.Status.IsDone() && backup.Status.ExpiredAt != nil { - requeueAfter := time.Until(backup.Status.ExpiredAt.Time) - if requeueAfter < 0 { - requeueAfter = time.Minute - } - return ctrl.Result{RequeueAfter: requeueAfter}, nil - } - - // Fetch the associated DocumentDB cluster - cluster := &dbpreview.DocumentDB{} - clusterKey := client.ObjectKey{ - Name: backup.Spec.Cluster.Name, - Namespace: backup.Namespace, - } - if err := r.Get(ctx, clusterKey, cluster); err != nil { - return r.SetBackupPhaseFailed(ctx, backup, "Failed to get associated DocumentDB cluster: "+err.Error(), nil) - } - - // Ensure VolumeSnapshotClass exists - if err := r.ensureVolumeSnapshotClass(ctx, cluster.Spec.Environment); err != nil { - return r.SetBackupPhaseFailed(ctx, backup, "Failed to ensure VolumeSnapshotClass: "+err.Error(), cluster.Spec.Backup) - } - - // Get or create the CNPG Backup - cnpgBackup := &cnpgv1.Backup{} - cnpgBackupKey := client.ObjectKey{ - Name: backup.Name, - Namespace: backup.Namespace, - } - if err := r.Get(ctx, cnpgBackupKey, cnpgBackup); err != nil { - if apierrors.IsNotFound(err) { - - // Skip backup if the cluster is not primary - replicationContext, err := util.GetReplicationContext(ctx, r.Client, *cluster) - if err != nil { - logger.Error(err, "Failed to determine replication context") - return ctrl.Result{}, err - } - if !replicationContext.IsPrimary() { - return r.SetBackupPhaseSkipped(ctx, backup, "Backups can only be created from the primary cluster", cluster.Spec.Backup) - } - if !replicationContext.EndpointEnabled() { - logger.Info("Backup deferred: primary cluster endpoint not ready, waiting for promotion to complete") - return ctrl.Result{RequeueAfter: time.Minute * 1}, nil - } - - return r.createCNPGBackup(ctx, backup, cluster, replicationContext) - } - logger.Error(err, "Failed to get CNPG Backup") - return ctrl.Result{}, err +func (r *BackupReconciler) engine() *engine.BackupReconciler[*dbpreview.DocumentDB, *dbpreview.Backup] { + return &engine.BackupReconciler[*dbpreview.DocumentDB, *dbpreview.Backup]{ + Client: r.Client, + Scheme: r.Scheme, + Recorder: r.Recorder, + Cluster: product.DocumentDBAdapter{}, + Backups: product.DocumentDBBackupAdapter{}, + Replication: documentDBReplication{&DocumentDBReconciler{Client: r.Client}}, } - - return r.updateBackupStatus(ctx, backup, cnpgBackup, cluster.Spec.Backup, cluster.Status.SchemaVersion) } -// ensureVolumeSnapshotClass creates a VolumeSnapshotClass based on the cloud environment -func (r *BackupReconciler) ensureVolumeSnapshotClass(ctx context.Context, environment string) error { - logger := log.FromContext(ctx) - - // Check if any VolumeSnapshotClass exists - vscList := &snapshotv1.VolumeSnapshotClassList{} - if err := r.List(ctx, vscList); err != nil { - logger.Error(err, "Failed to list VolumeSnapshotClasses") - return err - } - - for _, vsc := range vscList.Items { - if val, ok := vsc.Annotations["snapshot.storage.kubernetes.io/is-default-class"]; ok && val == "true" { - return nil - } - } - - r.Recorder.Event(nil, "Normal", "VolumeSnapshotClass", "No default VolumeSnapshotClass found, creating one") - vsc := buildVolumeSnapshotClass(environment) - if vsc == nil { - err := fmt.Errorf("Please create a default VolumeSnapshotClass before creating backups") - logger.Error(err, "Failed to build VolumeSnapshotClass", "environment", environment) - return err - } - - if err := r.Create(ctx, vsc); err != nil { - logger.Error(err, "Failed to create VolumeSnapshotClass") - return err - } - - r.Recorder.Event(nil, "Normal", "VolumeSnapshotClass", "Successfully created VolumeSnapshotClass "+vsc.Name) - return nil -} - -// buildVolumeSnapshotClass builds a VolumeSnapshotClass based on cloud provider -func buildVolumeSnapshotClass(environment string) *snapshotv1.VolumeSnapshotClass { - deletionPolicy := snapshotv1.VolumeSnapshotContentDelete - - var driver string - var name string - - switch environment { - case "aks": - driver = "disk.csi.azure.com" - name = "azure-disk-snapclass" - default: - // TODO: add support for other cloud providers - return nil - } - - return &snapshotv1.VolumeSnapshotClass{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Annotations: map[string]string{ - "snapshot.storage.kubernetes.io/is-default-class": "true", - }, - }, - Driver: driver, - DeletionPolicy: deletionPolicy, - } +// Reconcile handles the reconciliation loop for Backup resources. +func (r *BackupReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + return r.engine().Reconcile(ctx, req) } // createCNPGBackup creates a new CNPG Backup resource func (r *BackupReconciler) createCNPGBackup(ctx context.Context, backup *dbpreview.Backup, cluster *dbpreview.DocumentDB, replicationContext *util.ReplicationContext) (ctrl.Result, error) { - cnpgClusterName := replicationContext.CNPGClusterName - - cnpgBackup, err := backup.CreateCNPGBackup(r.Scheme, cnpgClusterName) - if err != nil { - return r.SetBackupPhaseFailed(ctx, backup, "Failed to initialize backup: "+err.Error(), cluster.Spec.Backup) - } - - if err := r.Create(ctx, cnpgBackup); err != nil { - return r.SetBackupPhaseFailed(ctx, backup, "Failed to initialize backup: "+err.Error(), cluster.Spec.Backup) - } - - r.Recorder.Event(backup, "Normal", "BackupInitialized", "Successfully initialized backup") - - // Requeue to check status - return ctrl.Result{RequeueAfter: 5 * time.Second}, nil + return r.engine().CreateCNPGBackup(ctx, backup, cluster.Spec.Backup, replicationContext) } // updateBackupStatus reconciles and persists the Backup's status, returning the reconcile result. func (r *BackupReconciler) updateBackupStatus(ctx context.Context, backup *dbpreview.Backup, cnpgBackup *cnpgv1.Backup, backupConfiguration *dbpreview.BackupConfiguration, sourceSchemaVersion string) (ctrl.Result, error) { - original := backup.DeepCopy() - needsUpdate := backup.UpdateStatus(cnpgBackup, backupConfiguration, sourceSchemaVersion) - - if needsUpdate { - if err := r.Status().Patch(ctx, backup, client.MergeFrom(original)); err != nil { - logger := log.FromContext(ctx) - logger.Error(err, "Failed to patch Backup status") - return ctrl.Result{}, err - } - } - - if backup.Status.IsDone() && backup.Status.ExpiredAt != nil { - requeueAfter := time.Until(backup.Status.ExpiredAt.Time) - if requeueAfter < 0 { - requeueAfter = time.Minute - } - return ctrl.Result{RequeueAfter: requeueAfter}, nil - } - - // Backup is still in progress, requeue to check status again - return ctrl.Result{RequeueAfter: 10 * time.Second}, nil + return r.engine().UpdateBackupStatus(ctx, backup, cnpgBackup, backupConfiguration, sourceSchemaVersion) } func (r *BackupReconciler) SetBackupPhaseFailed(ctx context.Context, backup *dbpreview.Backup, errMessage string, backupConfiguration *dbpreview.BackupConfiguration) (ctrl.Result, error) { - original := backup.DeepCopy() - - backup.Status.Phase = cnpgv1.BackupPhaseFailed - backup.Status.Message = errMessage - backup.Status.ExpiredAt = backup.CalculateExpirationTime(backupConfiguration) - - if err := r.Status().Patch(ctx, backup, client.MergeFrom(original)); err != nil { - logger := log.FromContext(ctx) - logger.Error(err, "Failed to patch Backup status") - return ctrl.Result{}, err - } - - r.Recorder.Event(backup, "Warning", "BackupFailed", errMessage) - requeueAfter := time.Until(backup.Status.ExpiredAt.Time) - if requeueAfter < 0 { - requeueAfter = time.Minute - } - return ctrl.Result{RequeueAfter: requeueAfter}, nil + return r.engine().SetBackupPhaseFailed(ctx, backup, errMessage, backupConfiguration) } func (r *BackupReconciler) SetBackupPhaseSkipped(ctx context.Context, backup *dbpreview.Backup, message string, backupConfiguration *dbpreview.BackupConfiguration) (ctrl.Result, error) { - original := backup.DeepCopy() - - backup.Status.Phase = dbpreview.BackupPhaseSkipped - backup.Status.Message = message - backup.Status.ExpiredAt = backup.CalculateExpirationTime(backupConfiguration) - - if err := r.Status().Patch(ctx, backup, client.MergeFrom(original)); err != nil { - logger := log.FromContext(ctx) - logger.Error(err, "Failed to patch Backup status") - return ctrl.Result{}, err - } - - r.Recorder.Event(backup, "Warning", "BackupSkipped", message) - requeueAfter := time.Until(backup.Status.ExpiredAt.Time) - if requeueAfter < 0 { - requeueAfter = time.Minute - } - return ctrl.Result{RequeueAfter: requeueAfter}, nil + return r.engine().SetBackupPhaseSkipped(ctx, backup, message, backupConfiguration) } // SetupWithManager sets up the controller with the Manager. func (r *BackupReconciler) SetupWithManager(mgr ctrl.Manager) error { - // Register VolumeSnapshotClass with the scheme - if err := snapshotv1.AddToScheme(mgr.GetScheme()); err != nil { - return err - } - - return ctrl.NewControllerManagedBy(mgr). - For(&dbpreview.Backup{}). - Owns(&cnpgv1.Backup{}). - Complete(r) + return r.engine().SetupWithManager(mgr) } diff --git a/operator/src/internal/controller/scheduledbackup_controller.go b/operator/src/internal/controller/scheduledbackup_controller.go index 0086eedf..3b381327 100644 --- a/operator/src/internal/controller/scheduledbackup_controller.go +++ b/operator/src/internal/controller/scheduledbackup_controller.go @@ -5,151 +5,42 @@ package controller import ( "context" - "time" - "github.com/robfig/cron" - apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/tools/record" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" - "sigs.k8s.io/controller-runtime/pkg/log" dbpreview "github.com/documentdb/documentdb-operator/api/preview" + "github.com/documentdb/documentdb-operator/internal/engine" + "github.com/documentdb/documentdb-operator/internal/product" ) -// ScheduledBackupReconciler reconciles a ScheduledBackup object +// ScheduledBackupReconciler reconciles a ScheduledBackup object. The logic +// lives in engine.ScheduledBackupReconciler. type ScheduledBackupReconciler struct { client.Client Scheme *runtime.Scheme Recorder record.EventRecorder } -// Reconcile handles the reconciliation loop for ScheduledBackup resources. -func (r *ScheduledBackupReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { - logger := log.FromContext(ctx) - - // Fetch the ScheduledBackup resource - scheduledBackup := &dbpreview.ScheduledBackup{} - if err := r.Get(ctx, req.NamespacedName, scheduledBackup); err != nil { - if apierrors.IsNotFound(err) { - logger.Info("ScheduledBackup resource not found, might have been deleted") - return ctrl.Result{}, nil - } - logger.Error(err, "Failed to get ScheduledBackup") - return ctrl.Result{}, err - } - - // Ensure ScheduledBackup is owned by the referenced Cluster so it's garbage collected when the Cluster is deleted. - err := r.ensureOwnerReference(ctx, scheduledBackup) - if err != nil { - logger.Error(err, "Failed to ensure owner reference on ScheduledBackup") - return ctrl.Result{}, err - } - - // Parse cron schedule - schedule, err := cron.ParseStandard(scheduledBackup.Spec.Schedule) - if err != nil { - logger.Error(err, "Failed to parse schedule", "schedule", scheduledBackup.Spec.Schedule) - r.Recorder.Event(scheduledBackup, "Warning", "InvalidSchedule", "Failed to parse schedule: "+err.Error()) - return ctrl.Result{}, err - } - - // If there is an ongoing backup, wait for it to finish before starting a new one - backupList := &dbpreview.BackupList{} - if err := r.List(ctx, backupList, client.InNamespace(scheduledBackup.Namespace), client.MatchingFields{"spec.cluster": scheduledBackup.Spec.Cluster.Name}); err != nil { - logger.Error(err, "Failed to list backups") - return ctrl.Result{}, err - } - - if backupList.IsBackupRunning() { - // If a backup is currently running, requeue after a short delay - logger.Info("A backup is currently running, requeuing") - return ctrl.Result{RequeueAfter: time.Minute}, nil - } - - // If it's time to create a backup - nextScheduleTime := scheduledBackup.GetNextScheduleTime(schedule, backupList.GetLastBackup()) - now := time.Now() - if !now.Before(nextScheduleTime) { - backup := scheduledBackup.CreateBackup(now) - logger.Info("Creating new backup", "backupName", backup.Name) - if err := r.Create(ctx, backup); err != nil { - logger.Error(err, "Failed to create backup", "backupName", backup.Name) - r.Recorder.Event(scheduledBackup, "Warning", "BackupCreation", "Failed to create backup: "+err.Error()) - return ctrl.Result{}, err - } - - scheduledBackup.Status.LastScheduledTime = &metav1.Time{Time: now} - - // Calculate next run time - nextScheduleTime = schedule.Next(now) - } - - scheduledBackup.Status.NextScheduledTime = &metav1.Time{Time: nextScheduleTime} - if err := r.Status().Update(ctx, scheduledBackup); err != nil { - logger.Error(err, "Failed to update ScheduledBackup status with next scheduled time") - return ctrl.Result{}, err +func (r *ScheduledBackupReconciler) engine() *engine.ScheduledBackupReconciler[*dbpreview.DocumentDB, *dbpreview.ScheduledBackup, *dbpreview.Backup] { + return &engine.ScheduledBackupReconciler[*dbpreview.DocumentDB, *dbpreview.ScheduledBackup, *dbpreview.Backup]{ + Client: r.Client, + Scheme: r.Scheme, + Recorder: r.Recorder, + Cluster: product.DocumentDBAdapter{}, + Scheduled: product.DocumentDBScheduledBackupAdapter{}, + Backups: product.DocumentDBBackupAdapter{}, } - - logger.Info("Next backup scheduled", "time", nextScheduleTime) - r.Recorder.Event(scheduledBackup, "Normal", "BackupSchedule", "Next backup scheduled at: "+nextScheduleTime.String()) - - // Requeue at next schedule time - requeueAfter := time.Until(nextScheduleTime) - if requeueAfter < 0 { - requeueAfter = time.Minute - } - return ctrl.Result{RequeueAfter: requeueAfter}, nil } -func (r *ScheduledBackupReconciler) ensureOwnerReference(ctx context.Context, scheduledBackup *dbpreview.ScheduledBackup) error { - if len(scheduledBackup.OwnerReferences) > 0 { - // Owner reference already set - return nil - } - - // Fetch the associated DocumentDB cluster - cluster := &dbpreview.DocumentDB{} - clusterKey := client.ObjectKey{ - Name: scheduledBackup.Spec.Cluster.Name, - Namespace: scheduledBackup.Namespace, - } - if err := r.Get(ctx, clusterKey, cluster); err != nil { - r.Recorder.Event(scheduledBackup, "Warning", "ClusterNotFound", "Failed to find associated DocumentDB cluster: "+err.Error()) - return err - } - - // Set owner reference - if err := controllerutil.SetControllerReference(cluster, scheduledBackup, r.Scheme); err != nil { - logger := log.FromContext(ctx) - logger.Error(err, "Failed to set owner reference on ScheduledBackup") - return err - } - - // Update the ScheduledBackup with the new owner reference - if err := r.Update(ctx, scheduledBackup); err != nil { - logger := log.FromContext(ctx) - logger.Error(err, "Failed to update ScheduledBackup with owner reference") - return err - } - - return nil +// Reconcile handles the reconciliation loop for ScheduledBackup resources. +func (r *ScheduledBackupReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + return r.engine().Reconcile(ctx, req) } // SetupWithManager sets up the controller with the Manager. func (r *ScheduledBackupReconciler) SetupWithManager(mgr ctrl.Manager) error { - // Register field index for spec.cluster so we can query Backups by cluster name - if err := mgr.GetFieldIndexer().IndexField(context.Background(), &dbpreview.Backup{}, "spec.cluster", func(rawObj client.Object) []string { - backup := rawObj.(*dbpreview.Backup) - return []string{backup.Spec.Cluster.Name} - }); err != nil { - return err - } - - return ctrl.NewControllerManagedBy(mgr). - For(&dbpreview.ScheduledBackup{}). - Complete(r) + return r.engine().SetupWithManager(mgr) } diff --git a/operator/src/internal/engine/backup.go b/operator/src/internal/engine/backup.go new file mode 100644 index 00000000..1f0e47f6 --- /dev/null +++ b/operator/src/internal/engine/backup.go @@ -0,0 +1,301 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package engine + +import ( + "context" + "fmt" + "time" + + cnpgv1 "github.com/cloudnative-pg/cloudnative-pg/api/v1" + snapshotv1 "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/tools/record" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/log" + + dbpreview "github.com/documentdb/documentdb-operator/api/preview" + "github.com/documentdb/documentdb-operator/internal/product" + util "github.com/documentdb/documentdb-operator/internal/utils" +) + +// BackupAdapter maps a product Backup custom resource onto the shared backup +// spec and status types. Spec and Status return pointers into the object. +type BackupAdapter[B client.Object] interface { + New() B + NewList() client.ObjectList + // Items returns pointers to the items of a list built by NewList. + Items(list client.ObjectList) []B + Spec(B) *dbpreview.BackupSpec + Status(B) *dbpreview.BackupStatus +} + +// BackupReconciler takes on-demand volume-snapshot backups of a product +// cluster through CNPG and expires them by retention policy. +type BackupReconciler[T client.Object, B client.Object] struct { + client.Client + Scheme *runtime.Scheme + Recorder record.EventRecorder + Cluster product.ClusterAdapter[T] + Backups BackupAdapter[B] + // Replication resolves the replication role of the cluster; nil means + // single-region primary. Only Context is used. + Replication ReplicationHooks[T] +} + +func (r *BackupReconciler[T, B]) replicationContext(ctx context.Context, cluster T) (*util.ReplicationContext, error) { + if r.Replication != nil { + return r.Replication.Context(ctx, cluster) + } + spec := r.Cluster.Spec(cluster) + return util.NewSingleRegionContext(cluster.GetName(), spec.Environment, spec.StorageClass), nil +} + +// Reconcile handles the reconciliation loop for Backup resources. +func (r *BackupReconciler[T, B]) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + logger := log.FromContext(ctx) + + // Fetch the Backup resource + backup := r.Backups.New() + if err := r.Get(ctx, req.NamespacedName, backup); err != nil { + if apierrors.IsNotFound(err) { + logger.Info("Backup resource not found, might have been deleted") + return ctrl.Result{}, nil + } + logger.Error(err, "Failed to get Backup") + return ctrl.Result{}, err + } + status := r.Backups.Status(backup) + + // Delete the Backup resource if it has expired + if status.IsExpired() { + r.Recorder.Event(backup, "Normal", "BackupExpired", "Backup has expired and will be deleted") + if err := r.Delete(ctx, backup); err != nil { + r.Recorder.Event(backup, "Warning", "BackupDeleteFailed", "Failed to delete expired Backup: "+err.Error()) + return ctrl.Result{}, err + } + r.Recorder.Event(backup, "Normal", "BackupDeleted", "Expired Backup has been deleted") + return ctrl.Result{}, nil + } + + // If the backup is already done and not expired, requeue to check expiration + if status.IsDone() && status.ExpiredAt != nil { + requeueAfter := time.Until(status.ExpiredAt.Time) + if requeueAfter < 0 { + requeueAfter = time.Minute + } + return ctrl.Result{RequeueAfter: requeueAfter}, nil + } + + // Fetch the associated cluster + cluster := r.Cluster.New() + clusterKey := client.ObjectKey{ + Name: r.Backups.Spec(backup).Cluster.Name, + Namespace: backup.GetNamespace(), + } + if err := r.Get(ctx, clusterKey, cluster); err != nil { + return r.SetBackupPhaseFailed(ctx, backup, "Failed to get associated "+r.Cluster.Profile().Naming.Kind+" cluster: "+err.Error(), nil) + } + clusterSpec := r.Cluster.Spec(cluster) + + // Ensure VolumeSnapshotClass exists + if err := r.EnsureVolumeSnapshotClass(ctx, clusterSpec.Environment); err != nil { + return r.SetBackupPhaseFailed(ctx, backup, "Failed to ensure VolumeSnapshotClass: "+err.Error(), clusterSpec.Backup) + } + + // Get or create the CNPG Backup + cnpgBackup := &cnpgv1.Backup{} + cnpgBackupKey := client.ObjectKey{ + Name: backup.GetName(), + Namespace: backup.GetNamespace(), + } + if err := r.Get(ctx, cnpgBackupKey, cnpgBackup); err != nil { + if apierrors.IsNotFound(err) { + + // Skip backup if the cluster is not primary + replicationContext, err := r.replicationContext(ctx, cluster) + if err != nil { + logger.Error(err, "Failed to determine replication context") + return ctrl.Result{}, err + } + if !replicationContext.IsPrimary() { + return r.SetBackupPhaseSkipped(ctx, backup, "Backups can only be created from the primary cluster", clusterSpec.Backup) + } + if !replicationContext.EndpointEnabled() { + logger.Info("Backup deferred: primary cluster endpoint not ready, waiting for promotion to complete") + return ctrl.Result{RequeueAfter: time.Minute * 1}, nil + } + + return r.CreateCNPGBackup(ctx, backup, clusterSpec.Backup, replicationContext) + } + logger.Error(err, "Failed to get CNPG Backup") + return ctrl.Result{}, err + } + + return r.UpdateBackupStatus(ctx, backup, cnpgBackup, clusterSpec.Backup, r.Cluster.Status(cluster).SchemaVersion) +} + +// EnsureVolumeSnapshotClass creates a VolumeSnapshotClass based on the cloud environment +func (r *BackupReconciler[T, B]) EnsureVolumeSnapshotClass(ctx context.Context, environment string) error { + logger := log.FromContext(ctx) + + // Check if any VolumeSnapshotClass exists + vscList := &snapshotv1.VolumeSnapshotClassList{} + if err := r.List(ctx, vscList); err != nil { + logger.Error(err, "Failed to list VolumeSnapshotClasses") + return err + } + + for _, vsc := range vscList.Items { + if val, ok := vsc.Annotations["snapshot.storage.kubernetes.io/is-default-class"]; ok && val == "true" { + return nil + } + } + + r.Recorder.Event(nil, "Normal", "VolumeSnapshotClass", "No default VolumeSnapshotClass found, creating one") + vsc := BuildVolumeSnapshotClass(environment) + if vsc == nil { + err := fmt.Errorf("Please create a default VolumeSnapshotClass before creating backups") + logger.Error(err, "Failed to build VolumeSnapshotClass", "environment", environment) + return err + } + + if err := r.Create(ctx, vsc); err != nil { + logger.Error(err, "Failed to create VolumeSnapshotClass") + return err + } + + r.Recorder.Event(nil, "Normal", "VolumeSnapshotClass", "Successfully created VolumeSnapshotClass "+vsc.Name) + return nil +} + +// BuildVolumeSnapshotClass builds a VolumeSnapshotClass based on cloud provider +func BuildVolumeSnapshotClass(environment string) *snapshotv1.VolumeSnapshotClass { + deletionPolicy := snapshotv1.VolumeSnapshotContentDelete + + var driver string + var name string + + switch environment { + case "aks": + driver = "disk.csi.azure.com" + name = "azure-disk-snapclass" + default: + // TODO: add support for other cloud providers + return nil + } + + return &snapshotv1.VolumeSnapshotClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Annotations: map[string]string{ + "snapshot.storage.kubernetes.io/is-default-class": "true", + }, + }, + Driver: driver, + DeletionPolicy: deletionPolicy, + } +} + +// CreateCNPGBackup creates a new CNPG Backup resource +func (r *BackupReconciler[T, B]) CreateCNPGBackup(ctx context.Context, backup B, backupConfiguration *dbpreview.BackupConfiguration, replicationContext *util.ReplicationContext) (ctrl.Result, error) { + cnpgClusterName := replicationContext.CNPGClusterName + + cnpgBackup, err := dbpreview.NewCNPGBackup(r.Scheme, backup, cnpgClusterName) + if err != nil { + return r.SetBackupPhaseFailed(ctx, backup, "Failed to initialize backup: "+err.Error(), backupConfiguration) + } + + if err := r.Create(ctx, cnpgBackup); err != nil { + return r.SetBackupPhaseFailed(ctx, backup, "Failed to initialize backup: "+err.Error(), backupConfiguration) + } + + r.Recorder.Event(backup, "Normal", "BackupInitialized", "Successfully initialized backup") + + // Requeue to check status + return ctrl.Result{RequeueAfter: 5 * time.Second}, nil +} + +func (r *BackupReconciler[T, B]) expirationTime(backup B, backupConfiguration *dbpreview.BackupConfiguration) *metav1.Time { + return dbpreview.BackupExpirationTime(r.Backups.Spec(backup), r.Backups.Status(backup), backup.GetCreationTimestamp(), backupConfiguration) +} + +func (r *BackupReconciler[T, B]) patchStatus(ctx context.Context, backup B, original client.Object) error { + if err := r.Status().Patch(ctx, backup, client.MergeFrom(original)); err != nil { + logger := log.FromContext(ctx) + logger.Error(err, "Failed to patch Backup status") + return err + } + return nil +} + +// UpdateBackupStatus reconciles and persists the Backup's status, returning the reconcile result. +func (r *BackupReconciler[T, B]) UpdateBackupStatus(ctx context.Context, backup B, cnpgBackup *cnpgv1.Backup, backupConfiguration *dbpreview.BackupConfiguration, sourceSchemaVersion string) (ctrl.Result, error) { + original := backup.DeepCopyObject().(client.Object) + status := r.Backups.Status(backup) + needsUpdate := dbpreview.UpdateBackupStatus(r.Backups.Spec(backup), status, backup.GetCreationTimestamp(), cnpgBackup, backupConfiguration, sourceSchemaVersion) + + if needsUpdate { + if err := r.patchStatus(ctx, backup, original); err != nil { + return ctrl.Result{}, err + } + } + + if status.IsDone() && status.ExpiredAt != nil { + requeueAfter := time.Until(status.ExpiredAt.Time) + if requeueAfter < 0 { + requeueAfter = time.Minute + } + return ctrl.Result{RequeueAfter: requeueAfter}, nil + } + + // Backup is still in progress, requeue to check status again + return ctrl.Result{RequeueAfter: 10 * time.Second}, nil +} + +func (r *BackupReconciler[T, B]) setTerminalPhase(ctx context.Context, backup B, phase cnpgv1.BackupPhase, reason, message string, backupConfiguration *dbpreview.BackupConfiguration) (ctrl.Result, error) { + original := backup.DeepCopyObject().(client.Object) + + status := r.Backups.Status(backup) + status.Phase = phase + status.Message = message + status.ExpiredAt = r.expirationTime(backup, backupConfiguration) + + if err := r.patchStatus(ctx, backup, original); err != nil { + return ctrl.Result{}, err + } + + r.Recorder.Event(backup, "Warning", reason, message) + requeueAfter := time.Until(status.ExpiredAt.Time) + if requeueAfter < 0 { + requeueAfter = time.Minute + } + return ctrl.Result{RequeueAfter: requeueAfter}, nil +} + +// SetBackupPhaseFailed marks the backup failed and records errMessage. +func (r *BackupReconciler[T, B]) SetBackupPhaseFailed(ctx context.Context, backup B, errMessage string, backupConfiguration *dbpreview.BackupConfiguration) (ctrl.Result, error) { + return r.setTerminalPhase(ctx, backup, cnpgv1.BackupPhaseFailed, "BackupFailed", errMessage, backupConfiguration) +} + +// SetBackupPhaseSkipped marks the backup skipped and records message. +func (r *BackupReconciler[T, B]) SetBackupPhaseSkipped(ctx context.Context, backup B, message string, backupConfiguration *dbpreview.BackupConfiguration) (ctrl.Result, error) { + return r.setTerminalPhase(ctx, backup, dbpreview.BackupPhaseSkipped, "BackupSkipped", message, backupConfiguration) +} + +// SetupWithManager registers the reconciler with the manager. +func (r *BackupReconciler[T, B]) SetupWithManager(mgr ctrl.Manager) error { + // Register VolumeSnapshotClass with the scheme + if err := snapshotv1.AddToScheme(mgr.GetScheme()); err != nil { + return err + } + + return ctrl.NewControllerManagedBy(mgr). + For(r.Backups.New()). + Owns(&cnpgv1.Backup{}). + Complete(r) +} diff --git a/operator/src/internal/engine/scheduledbackup.go b/operator/src/internal/engine/scheduledbackup.go new file mode 100644 index 00000000..b647ebf9 --- /dev/null +++ b/operator/src/internal/engine/scheduledbackup.go @@ -0,0 +1,220 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package engine + +import ( + "context" + "time" + + "github.com/robfig/cron" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/tools/record" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/log" + + dbpreview "github.com/documentdb/documentdb-operator/api/preview" + "github.com/documentdb/documentdb-operator/internal/product" +) + +// BackupClusterField is the Backup field index holding the cluster name. +const BackupClusterField = "spec.cluster" + +// ScheduledBackupAdapter maps a product ScheduledBackup custom resource onto +// the shared scheduled-backup spec and status types. +type ScheduledBackupAdapter[S client.Object] interface { + New() S + Spec(S) *dbpreview.ScheduledBackupSpec + Status(S) *dbpreview.ScheduledBackupStatus +} + +// ScheduledBackupReconciler creates Backups of a product cluster on a cron schedule. +type ScheduledBackupReconciler[T client.Object, S client.Object, B client.Object] struct { + client.Client + Scheme *runtime.Scheme + Recorder record.EventRecorder + Cluster product.ClusterAdapter[T] + Scheduled ScheduledBackupAdapter[S] + Backups BackupAdapter[B] +} + +// Reconcile handles the reconciliation loop for ScheduledBackup resources. +func (r *ScheduledBackupReconciler[T, S, B]) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + logger := log.FromContext(ctx) + + // Fetch the ScheduledBackup resource + scheduledBackup := r.Scheduled.New() + if err := r.Get(ctx, req.NamespacedName, scheduledBackup); err != nil { + if apierrors.IsNotFound(err) { + logger.Info("ScheduledBackup resource not found, might have been deleted") + return ctrl.Result{}, nil + } + logger.Error(err, "Failed to get ScheduledBackup") + return ctrl.Result{}, err + } + spec := r.Scheduled.Spec(scheduledBackup) + + // Ensure ScheduledBackup is owned by the referenced Cluster so it's garbage collected when the Cluster is deleted. + err := r.EnsureOwnerReference(ctx, scheduledBackup) + if err != nil { + logger.Error(err, "Failed to ensure owner reference on ScheduledBackup") + return ctrl.Result{}, err + } + + // Parse cron schedule + schedule, err := cron.ParseStandard(spec.Schedule) + if err != nil { + logger.Error(err, "Failed to parse schedule", "schedule", spec.Schedule) + r.Recorder.Event(scheduledBackup, "Warning", "InvalidSchedule", "Failed to parse schedule: "+err.Error()) + return ctrl.Result{}, err + } + + // If there is an ongoing backup, wait for it to finish before starting a new one + backupList := r.Backups.NewList() + if err := r.List(ctx, backupList, client.InNamespace(scheduledBackup.GetNamespace()), client.MatchingFields{BackupClusterField: spec.Cluster.Name}); err != nil { + logger.Error(err, "Failed to list backups") + return ctrl.Result{}, err + } + backups := r.Backups.Items(backupList) + + if r.isBackupRunning(backups) { + // If a backup is currently running, requeue after a short delay + logger.Info("A backup is currently running, requeuing") + return ctrl.Result{RequeueAfter: time.Minute}, nil + } + + // If it's time to create a backup + var lastBackupCreated *metav1.Time + if last, ok := lastBackup(backups); ok { + created := last.GetCreationTimestamp() + lastBackupCreated = &created + } + status := r.Scheduled.Status(scheduledBackup) + nextScheduleTime := dbpreview.NextScheduleTime(schedule, status, lastBackupCreated) + now := time.Now() + if !now.Before(nextScheduleTime) { + backup := r.newBackup(scheduledBackup, now) + logger.Info("Creating new backup", "backupName", backup.GetName()) + if err := r.Create(ctx, backup); err != nil { + logger.Error(err, "Failed to create backup", "backupName", backup.GetName()) + r.Recorder.Event(scheduledBackup, "Warning", "BackupCreation", "Failed to create backup: "+err.Error()) + return ctrl.Result{}, err + } + + status.LastScheduledTime = &metav1.Time{Time: now} + + // Calculate next run time + nextScheduleTime = schedule.Next(now) + } + + status.NextScheduledTime = &metav1.Time{Time: nextScheduleTime} + if err := r.Status().Update(ctx, scheduledBackup); err != nil { + logger.Error(err, "Failed to update ScheduledBackup status with next scheduled time") + return ctrl.Result{}, err + } + + logger.Info("Next backup scheduled", "time", nextScheduleTime) + r.Recorder.Event(scheduledBackup, "Normal", "BackupSchedule", "Next backup scheduled at: "+nextScheduleTime.String()) + + // Requeue at next schedule time + requeueAfter := time.Until(nextScheduleTime) + if requeueAfter < 0 { + requeueAfter = time.Minute + } + return ctrl.Result{RequeueAfter: requeueAfter}, nil +} + +// newBackup builds the Backup a ScheduledBackup creates at now. +func (r *ScheduledBackupReconciler[T, S, B]) newBackup(scheduledBackup S, now time.Time) B { + spec := r.Scheduled.Spec(scheduledBackup) + backup := r.Backups.New() + backup.SetName(dbpreview.ScheduledBackupName(scheduledBackup.GetName(), now)) + backup.SetNamespace(scheduledBackup.GetNamespace()) + backup.SetLabels(map[string]string{ + dbpreview.ScheduledBackupLabel: scheduledBackup.GetName(), + }) + *r.Backups.Spec(backup) = dbpreview.BackupSpec{ + Cluster: spec.Cluster, + RetentionDays: spec.RetentionDays, + } + return backup +} + +func (r *ScheduledBackupReconciler[T, S, B]) isBackupRunning(backups []B) bool { + for _, backup := range backups { + if !r.Backups.Status(backup).IsDone() { + return true + } + } + return false +} + +// lastBackup returns the most recently created backup. +func lastBackup[B client.Object](backups []B) (B, bool) { + var last B + found := false + for _, backup := range backups { + if !found || backup.GetCreationTimestamp().After(last.GetCreationTimestamp().Time) { + last = backup + found = true + } + } + return last, found +} + +// EnsureOwnerReference makes the referenced cluster the controller of the ScheduledBackup. +func (r *ScheduledBackupReconciler[T, S, B]) EnsureOwnerReference(ctx context.Context, scheduledBackup S) error { + if len(scheduledBackup.GetOwnerReferences()) > 0 { + // Owner reference already set + return nil + } + + // Fetch the associated cluster + cluster := r.Cluster.New() + clusterKey := client.ObjectKey{ + Name: r.Scheduled.Spec(scheduledBackup).Cluster.Name, + Namespace: scheduledBackup.GetNamespace(), + } + if err := r.Get(ctx, clusterKey, cluster); err != nil { + r.Recorder.Event(scheduledBackup, "Warning", "ClusterNotFound", "Failed to find associated "+r.Cluster.Profile().Naming.Kind+" cluster: "+err.Error()) + return err + } + + // Set owner reference + if err := controllerutil.SetControllerReference(cluster, scheduledBackup, r.Scheme); err != nil { + logger := log.FromContext(ctx) + logger.Error(err, "Failed to set owner reference on ScheduledBackup") + return err + } + + // Update the ScheduledBackup with the new owner reference + if err := r.Update(ctx, scheduledBackup); err != nil { + logger := log.FromContext(ctx) + logger.Error(err, "Failed to update ScheduledBackup with owner reference") + return err + } + + return nil +} + +// SetupWithManager registers the reconciler with the manager. +func (r *ScheduledBackupReconciler[T, S, B]) SetupWithManager(mgr ctrl.Manager) error { + // Register field index for spec.cluster so we can query Backups by cluster name + if err := mgr.GetFieldIndexer().IndexField(context.Background(), r.Backups.New(), BackupClusterField, func(rawObj client.Object) []string { + backup, ok := rawObj.(B) + if !ok { + return nil + } + return []string{r.Backups.Spec(backup).Cluster.Name} + }); err != nil { + return err + } + + return ctrl.NewControllerManagedBy(mgr). + For(r.Scheduled.New()). + Complete(r) +} diff --git a/operator/src/internal/product/documentdb_backup.go b/operator/src/internal/product/documentdb_backup.go new file mode 100644 index 00000000..33d31504 --- /dev/null +++ b/operator/src/internal/product/documentdb_backup.go @@ -0,0 +1,51 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package product + +import ( + "sigs.k8s.io/controller-runtime/pkg/client" + + dbpreview "github.com/documentdb/documentdb-operator/api/preview" +) + +// DocumentDBBackupAdapter maps the documentdb.io Backup custom resource onto +// the shared backup spec and status (engine.BackupAdapter). +type DocumentDBBackupAdapter struct{} + +func (DocumentDBBackupAdapter) New() *dbpreview.Backup { return &dbpreview.Backup{} } +func (DocumentDBBackupAdapter) NewList() client.ObjectList { return &dbpreview.BackupList{} } +func (DocumentDBBackupAdapter) Spec(b *dbpreview.Backup) *dbpreview.BackupSpec { + return &b.Spec +} +func (DocumentDBBackupAdapter) Status(b *dbpreview.Backup) *dbpreview.BackupStatus { + return &b.Status +} + +// Items returns pointers to the Backups of a BackupList. +func (DocumentDBBackupAdapter) Items(list client.ObjectList) []*dbpreview.Backup { + backupList, ok := list.(*dbpreview.BackupList) + if !ok { + return nil + } + items := make([]*dbpreview.Backup, 0, len(backupList.Items)) + for i := range backupList.Items { + items = append(items, &backupList.Items[i]) + } + return items +} + +// DocumentDBScheduledBackupAdapter maps the documentdb.io ScheduledBackup +// custom resource onto the shared scheduled-backup spec and status +// (engine.ScheduledBackupAdapter). +type DocumentDBScheduledBackupAdapter struct{} + +func (DocumentDBScheduledBackupAdapter) New() *dbpreview.ScheduledBackup { + return &dbpreview.ScheduledBackup{} +} +func (DocumentDBScheduledBackupAdapter) Spec(s *dbpreview.ScheduledBackup) *dbpreview.ScheduledBackupSpec { + return &s.Spec +} +func (DocumentDBScheduledBackupAdapter) Status(s *dbpreview.ScheduledBackup) *dbpreview.ScheduledBackupStatus { + return &s.Status +} From 8b407f402da657a1541a3f356721f88d15aa6d7d Mon Sep 17 00:00:00 2001 From: Rayhan Hossain Date: Mon, 19 Oct 2026 18:39:46 -0700 Subject: [PATCH 18/27] refactor: extract product-neutral admission validation Move schema/binary version, image rollback, storage resize, restore schema compatibility and the ImageVolume preflight into internal/validation, parameterized by product Terms. DocumentDBValidator delegates; messages and behavior are unchanged. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 33b3cb3c-1b25-4ffb-bff1-ebf0f3d04058 Signed-off-by: Rayhan Hossain --- .../src/internal/validation/imagevolume.go | 188 +++++++++++ .../src/internal/validation/validation.go | 305 ++++++++++++++++++ .../internal/webhook/documentdb_webhook.go | 290 ++--------------- operator/src/internal/webhook/imagevolume.go | 173 +--------- 4 files changed, 539 insertions(+), 417 deletions(-) create mode 100644 operator/src/internal/validation/imagevolume.go create mode 100644 operator/src/internal/validation/validation.go diff --git a/operator/src/internal/validation/imagevolume.go b/operator/src/internal/validation/imagevolume.go new file mode 100644 index 00000000..b728398a --- /dev/null +++ b/operator/src/internal/validation/imagevolume.go @@ -0,0 +1,188 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package validation + +import ( + "context" + "fmt" + "strings" + "sync/atomic" + + "github.com/go-logr/logr" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// imageVolumeProbeImage is a syntactically valid image reference used only for +// the dry-run capability probe. It is never pulled or scheduled because every +// probe Pod is submitted with client.DryRunAll. +const imageVolumeProbeImage = "registry.k8s.io/pause:3.10" + +// imageVolumeProbeVolumeName is the volume/mount name shared by both probe Pods. +const imageVolumeProbeVolumeName = "probe" + +// ImageVolumeProbe is the cluster-capability preflight for the Kubernetes +// ImageVolume feature the operator relies on to mount the extension. +type ImageVolumeProbe struct { + Client client.Client + // Confirmed caches a positive detection of ImageVolume support. + // + // Only positive results are cached: once the cluster is known to support + // ImageVolume, subsequent creates skip the probe entirely (steady-state + // latency ~0). A negative or inconclusive result is never cached, so if an + // administrator enables the feature gate later the next create re-probes and + // succeeds without restarting the operator. + Confirmed *atomic.Bool + // Product names the operator and extension in the error message. + Product string + Log logr.Logger +} + +// Ensure blocks cluster creation when the cluster does +// not support the Kubernetes ImageVolume feature (feature gate off, or a +// runtime/version that lacks it). The operator mounts the DocumentDB extension +// into the PostgreSQL pod via a corev1.ImageVolumeSource, so without this +// feature the cluster can never become ready. +// +// The check is intentionally fail-open: it blocks only when it can positively +// prove the feature is unavailable. If the cluster environment prevents a +// conclusive answer (e.g. Pod Security Admission or a ResourceQuota rejects the +// baseline probe Pod), creation is allowed and the reconcile loop surfaces any +// real failure later. +func (v *ImageVolumeProbe) Ensure(ctx context.Context, namespace string) error { + if v.Confirmed.Load() { + return nil + } + + supported, conclusive := v.Probe(ctx, namespace) + if !conclusive { + // Could not determine support; do not block on an inconclusive probe. + return nil + } + if supported { + v.Confirmed.Store(true) + return nil + } + + return fmt.Errorf( + "the Kubernetes ImageVolume feature is not enabled on this cluster, but it is required by the %s "+ + "operator to mount the %s extension into PostgreSQL pods. ImageVolume is GA (on by default) in "+ + "Kubernetes 1.35+. On Kubernetes 1.33/1.34 you may enable the ImageVolume feature gate on both the "+ + "kube-apiserver and every kubelet; the container runtime must also support it (containerd >= 2.1 or "+ + "CRI-O >= 1.33), which this preflight cannot verify. "+ + "See https://kubernetes.io/docs/concepts/storage/volumes/#image", + v.Product, v.Product, + ) +} + +// Probe performs a differential server-side dry-run to detect +// ImageVolume support without relying on error-string matching. +// +// It submits two dry-run Pods that are identical except for a single volume: +// - baseline: an emptyDir volume (always a valid volume source), and +// - image: an ImageVolumeSource volume. +// +// Interpretation: +// - baseline rejected -> the environment blocks pod creation +// entirely (PSA, quota, other webhooks); inconclusive. +// - baseline ok, image rejected -> the image volume is the sole difference, +// so ImageVolume is unsupported. +// - baseline ok, image accepted but the ImageVolumeSource was pruned from the +// returned Pod -> the feature gate is off, so the API server silently drops +// the disabled field instead of rejecting it; ImageVolume is unsupported. +// - baseline ok, image accepted and the ImageVolumeSource survived -> the +// feature is supported. +// +// The pruning case is the important one: on Kubernetes 1.33/1.34 with the +// ImageVolume beta gate OFF, a dry-run create does NOT fail; the API server +// admits the Pod and simply strips the unknown/disabled volume source. Relying +// on a create error alone therefore false-positives as "supported". We close +// that gap by inspecting the returned object for field retention. +// +// The second return value reports whether the result is conclusive. +func (v *ImageVolumeProbe) Probe(ctx context.Context, namespace string) (supported bool, conclusive bool) { + if _, err := v.dryRunProbePod(ctx, namespace, baselineProbeVolume()); err != nil { + v.Log.Info("ImageVolume preflight inconclusive: baseline probe pod rejected", "error", err) + return false, false + } + + returned, err := v.dryRunProbePod(ctx, namespace, imageProbeVolume()) + if err != nil { + v.Log.Info("ImageVolume preflight: image-volume probe pod rejected while baseline passed; feature unavailable", "error", err) + return false, true + } + + if !hasImageVolume(returned) { + v.Log.Info("ImageVolume preflight: image-volume probe pod admitted but the ImageVolumeSource was pruned from the returned spec; feature gate is off, feature unavailable") + return false, true + } + + return true, true +} + +// hasImageVolume reports whether the server-returned Pod still carries the probe +// volume as an ImageVolumeSource. When the ImageVolume feature gate is off the +// API server prunes the field, so its absence means the feature is unavailable. +func hasImageVolume(pod *corev1.Pod) bool { + if pod == nil { + return false + } + for _, vol := range pod.Spec.Volumes { + if vol.Name == imageVolumeProbeVolumeName { + return vol.Image != nil + } + } + return false +} + +// dryRunProbePod submits a minimal Pod carrying the given probe volume as a +// server-side dry-run (never persisted) and returns the Pod as processed by the +// API server (with defaulting and field pruning applied) alongside its verdict. +func (v *ImageVolumeProbe) dryRunProbePod(ctx context.Context, namespace string, volume corev1.Volume) (*corev1.Pod, error) { + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: strings.ToLower(v.Product) + "-imagevolume-preflight-", + Namespace: namespace, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{{ + Name: "probe", + Image: imageVolumeProbeImage, + VolumeMounts: []corev1.VolumeMount{{ + Name: imageVolumeProbeVolumeName, + MountPath: "/probe", + }}, + }}, + Volumes: []corev1.Volume{volume}, + }, + } + err := v.Client.Create(ctx, pod, client.DryRunAll) + return pod, err +} + +// baselineProbeVolume returns an emptyDir volume, which every Kubernetes +// version accepts. It establishes whether the environment permits creating the +// probe Pod at all. +func baselineProbeVolume() corev1.Volume { + return corev1.Volume{ + Name: imageVolumeProbeVolumeName, + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + } +} + +// imageProbeVolume returns an ImageVolumeSource volume, the feature under test. +func imageProbeVolume() corev1.Volume { + return corev1.Volume{ + Name: imageVolumeProbeVolumeName, + VolumeSource: corev1.VolumeSource{ + Image: &corev1.ImageVolumeSource{ + Reference: imageVolumeProbeImage, + PullPolicy: corev1.PullIfNotPresent, + }, + }, + } +} diff --git a/operator/src/internal/validation/validation.go b/operator/src/internal/validation/validation.go new file mode 100644 index 00000000..3b107cc7 --- /dev/null +++ b/operator/src/internal/validation/validation.go @@ -0,0 +1,305 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +// Package validation holds the product-neutral admission rules shared by the +// product validating webhooks: schema/binary version guards, image rollback, +// storage resize, restore schema compatibility, and the ImageVolume preflight. +package validation + +import ( + "context" + "fmt" + "regexp" + "strconv" + "strings" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/apimachinery/pkg/util/validation/field" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" + + util "github.com/documentdb/documentdb-operator/internal/utils" +) + +// Terms names the product and its version fields in validation messages. +type Terms struct { + // Product is the product name, e.g. "DocumentDB". + Product string + // VersionField and ImageField are the spec paths that pin the extension + // binary version, e.g. "spec.documentDBVersion" and "spec.image.documentDB". + VersionField string + ImageField string + // SchemaWithoutBinaryFields names the fields to set when an explicit + // schemaVersion has no binary version to validate against. + SchemaWithoutBinaryFields string +} + +var rcImageTagPattern = regexp.MustCompile(`^[vV]?([0-9]+)\.([0-9]+)-[rR][cC][1-9][0-9]*(?:$|[-+])`) + +// ExtractSemver returns the leading "X.Y.Z" portion from a tag string, +// normalizing "X.Y-RC" tags to the extension version "X.Y.0". +// It returns "" when neither version format is recognized. +func ExtractSemver(tag string) string { + if matches := rcImageTagPattern.FindStringSubmatch(tag); matches != nil { + return matches[1] + "." + matches[2] + ".0" + } + // Match digits.digits.digits at start of string + parts := strings.SplitN(tag, ".", 3) + if len(parts) < 3 { + return "" + } + // Validate major and minor are numeric + if _, err := strconv.Atoi(parts[0]); err != nil { + return "" + } + if _, err := strconv.Atoi(parts[1]); err != nil { + return "" + } + // Third part may have a suffix (e.g., "0-amd64"), take only leading digits + thirdPart := parts[2] + i := 0 + for i < len(thirdPart) && thirdPart[i] >= '0' && thirdPart[i] <= '9' { + i++ + } + if i == 0 { + return "" + } + return parts[0] + "." + parts[1] + "." + thirdPart[:i] +} + +// ImageTagSemver returns the semver of an image reference's tag, or "" for +// digest references and unparseable tags. +func ImageTagSemver(ref string) string { + if ref == "" || strings.Contains(ref, "@sha256:") { + return "" + } + if tagIdx := strings.LastIndex(ref, ":"); tagIdx >= 0 { + return ExtractSemver(ref[tagIdx+1:]) + } + return "" +} + +// ResolveBinaryVersion extracts the binary version pinned by a spec. +// Priority: image tag > version field > "" (unknown). Digest-only references +// (e.g., "image@sha256:...") are not parseable as versions. +func ResolveBinaryVersion(imageRef, version string) string { + if semver := ImageTagSemver(imageRef); semver != "" { + return semver + } + if v := ExtractSemver(version); v != "" { + return v + } + return version +} + +// SchemaVersionNotExceedsBinary ensures an explicit spec.schemaVersion is <= the binary version. +func SchemaVersionNotExceedsBinary(schemaVersion, binaryVersion string, t Terms) field.ErrorList { + if schemaVersion == "" || schemaVersion == "auto" { + return nil + } + + path := field.NewPath("spec", "schemaVersion") + if binaryVersion == "" { + return field.ErrorList{field.Invalid( + path, + schemaVersion, + "cannot set an explicit schemaVersion without also setting "+t.SchemaWithoutBinaryFields+"; "+ + "the webhook needs a binary version to validate against", + )} + } + + schemaExtensionVersion := util.SemverToExtensionVersion(schemaVersion) + binaryExtensionVersion := util.SemverToExtensionVersion(binaryVersion) + + cmp, err := util.CompareExtensionVersions(schemaExtensionVersion, binaryExtensionVersion) + if err != nil { + return field.ErrorList{field.Invalid( + path, + schemaVersion, + fmt.Sprintf("cannot validate schemaVersion: version comparison failed: %v", err), + )} + } + if cmp > 0 { + return field.ErrorList{field.Invalid( + path, + schemaVersion, + fmt.Sprintf("schemaVersion %s exceeds the binary version %s; schema version must be <= binary version", + schemaVersion, binaryVersion), + )} + } + return nil +} + +// ImageRollback blocks a new binary version below the installed schema version. +// Once ALTER EXTENSION UPDATE has run, the schema is irreversible. Running an older +// binary against a newer schema is untested and may cause data corruption. +// Callers should only invoke it when an image-related field changes. +func ImageRollback(installedSchemaVersion, newBinaryVersion string) field.ErrorList { + if installedSchemaVersion == "" || newBinaryVersion == "" { + return nil + } + + newBinaryExtensionVersion := util.SemverToExtensionVersion(newBinaryVersion) + schemaExtensionVersion := util.SemverToExtensionVersion(installedSchemaVersion) + + cmp, err := util.CompareExtensionVersions(newBinaryExtensionVersion, schemaExtensionVersion) + if err != nil { + return field.ErrorList{field.Forbidden( + field.NewPath("spec"), + fmt.Sprintf("cannot validate image rollback: version comparison failed: %v", err), + )} + } + if cmp < 0 { + return field.ErrorList{field.Forbidden( + field.NewPath("spec"), + fmt.Sprintf( + "image rollback blocked: requested version %s is older than installed schema version %s. "+ + "ALTER EXTENSION has no downgrade path — running an older binary with a newer schema may cause data corruption. "+ + "To recover, restore from backup or update to a version >= %s.", + newBinaryVersion, installedSchemaVersion, installedSchemaVersion), + )} + } + return nil +} + +// StorageResize ensures PVC size can only grow, never shrink. +func StorageResize(pvcSizePath *field.Path, oldSize, newSize string) field.ErrorList { + if oldSize == newSize { + return nil + } + + var allErrs field.ErrorList + + oldQty, errOld := resource.ParseQuantity(oldSize) + if errOld != nil { + allErrs = append(allErrs, field.Invalid( + pvcSizePath, + oldSize, + fmt.Sprintf("existing pvcSize is not a valid resource quantity: %v", errOld), + )) + } + + newQty, errNew := resource.ParseQuantity(newSize) + if errNew != nil { + allErrs = append(allErrs, field.Invalid( + pvcSizePath, + newSize, + fmt.Sprintf("pvcSize must be a valid resource quantity: %v", errNew), + )) + } + + if len(allErrs) > 0 { + return allErrs + } + + if newQty.Cmp(oldQty) < 0 { + return field.ErrorList{field.Forbidden( + pvcSizePath, + fmt.Sprintf("storage size can only be increased; attempted shrink from %s to %s", oldSize, newSize), + )} + } + return nil +} + +// RestoreSource identifies where a restore draws its data — and thus its schema +// version — from, carrying the diagnostics and the spec field to flag on error. +type RestoreSource struct { + Kind string // SourceKindBackup or SourceKindPV + Name string + FieldPath *field.Path +} + +const ( + SourceKindBackup = "backup" + SourceKindPV = "PersistentVolume" +) + +// BackupSource and PVSource build the RestoreSource of a recovery stanza at base. +func BackupSource(base *field.Path, name string) RestoreSource { + return RestoreSource{Kind: SourceKindBackup, Name: name, FieldPath: base.Child("backup")} +} + +func PVSource(base *field.Path, name string) RestoreSource { + return RestoreSource{Kind: SourceKindPV, Name: name, FieldPath: base.Child("persistentVolume")} +} + +// SourceSchemaVersion reads the schema version recorded on a restore source: +// readBackup returns it for a backup, and PVs carry it in pvAnnotation. It +// only reports warnings for I/O failures (source not found or unreadable); a +// successfully read source with no schema version returns ("", nil), leaving +// the "unknown schema" policy to CompareBinaryToSchema. +func SourceSchemaVersion(ctx context.Context, c client.Reader, src RestoreSource, pvAnnotation string, readBackup func() (string, error)) (string, admission.Warnings) { + var schemaVersion string + var readErr error + + switch src.Kind { + case SourceKindBackup: + schemaVersion, readErr = readBackup() + case SourceKindPV: + pv := &corev1.PersistentVolume{} + if err := c.Get(ctx, client.ObjectKey{Name: src.Name}, pv); err != nil { + readErr = err + } else { + schemaVersion = pv.Annotations[pvAnnotation] + } + } + + if readErr != nil { + if apierrors.IsNotFound(readErr) { + return "", admission.Warnings{ + fmt.Sprintf("%s %q not found: schema-version compatibility cannot be verified", src.Kind, src.Name), + } + } + // Transient/API error: don't hard-block the restore, but warn. + return "", admission.Warnings{ + fmt.Sprintf("failed to read %s %q: schema-version compatibility cannot be verified: %v", src.Kind, src.Name, readErr), + } + } + return schemaVersion, nil +} + +// CompareBinaryToSchema is the single authority on the restore decision. It warns +// (and allows) when either the source schema version or the target version is +// unknown, and rejects only a known target version that is older than a known +// schema. +func CompareBinaryToSchema(binaryVersion, schemaVersion string, src RestoreSource, t Terms) (admission.Warnings, field.ErrorList) { + target := "the target " + t.Product + " version" + if schemaVersion == "" { + return admission.Warnings{ + fmt.Sprintf("%s %q has no recorded schema version: schema-version compatibility cannot be verified; "+ + "ensure %s (%s or %s) is >= the source's "+ + "schema version to avoid data corruption", src.Kind, src.Name, target, t.VersionField, t.ImageField), + }, nil + } + if binaryVersion == "" { + return admission.Warnings{ + fmt.Sprintf("cannot determine %s for restore from %s %q "+ + "(set %s or %s): compatibility with the %s's schema version %s cannot be verified", + target, src.Kind, src.Name, t.VersionField, t.ImageField, src.Kind, schemaVersion), + }, nil + } + + binaryExtensionVersion := util.SemverToExtensionVersion(binaryVersion) + schemaExtensionVersion := util.SemverToExtensionVersion(schemaVersion) + + cmp, err := util.CompareExtensionVersions(binaryExtensionVersion, schemaExtensionVersion) + if err != nil { + return admission.Warnings{ + fmt.Sprintf("cannot compare %s %s with the %s's schema version %s: %v; compatibility not verified", + target, binaryVersion, src.Kind, schemaVersion, err), + }, nil + } + if cmp < 0 { + return nil, field.ErrorList{field.Forbidden( + src.FieldPath, + fmt.Sprintf( + "restore blocked: %s %s is older than the %s's schema version %s. "+ + "Restoring onto an older %s version runs it against a newer, irreversible schema and may cause data corruption. "+ + "Set %s (or %s) to %s or newer.", + target, binaryVersion, src.Kind, schemaVersion, t.Product, t.VersionField, t.ImageField, schemaVersion), + )} + } + return nil, nil +} diff --git a/operator/src/internal/webhook/documentdb_webhook.go b/operator/src/internal/webhook/documentdb_webhook.go index 06c13bf8..db5a56ab 100644 --- a/operator/src/internal/webhook/documentdb_webhook.go +++ b/operator/src/internal/webhook/documentdb_webhook.go @@ -5,15 +5,9 @@ package webhook import ( "context" - "fmt" "reflect" - "regexp" - "strconv" - "strings" - corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/api/resource" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/validation/field" ctrl "sigs.k8s.io/controller-runtime" @@ -25,11 +19,18 @@ import ( "github.com/documentdb/documentdb-operator/internal/cnpg" "github.com/documentdb/documentdb-operator/internal/product" util "github.com/documentdb/documentdb-operator/internal/utils" + "github.com/documentdb/documentdb-operator/internal/validation" ) var documentdbLog = logf.Log.WithName("documentdb-webhook") -var rcImageTagPattern = regexp.MustCompile(`^[vV]?([0-9]+)\.([0-9]+)-[rR][cC][1-9][0-9]*(?:$|[-+])`) +// documentDBTerms names DocumentDB's version fields in validation messages. +var documentDBTerms = validation.Terms{ + Product: "DocumentDB", + VersionField: "spec.documentDBVersion", + ImageField: "spec.image.documentDB", + SchemaWithoutBinaryFields: "spec.documentDBVersion or spec.documentDBImage", +} // DocumentDBValidator validates DocumentDB resources on create and update. type DocumentDBValidator struct { @@ -129,40 +130,7 @@ func (v *DocumentDBValidator) validateResources(db *dbpreview.DocumentDB) field. // validateSchemaVersionNotExceedsBinary ensures spec.schemaVersion <= binary version. func (v *DocumentDBValidator) validateSchemaVersionNotExceedsBinary(db *dbpreview.DocumentDB) field.ErrorList { - if db.Spec.SchemaVersion == "" || db.Spec.SchemaVersion == "auto" { - return nil - } - - binaryVersion := resolveBinaryVersion(db) - if binaryVersion == "" { - return field.ErrorList{field.Invalid( - field.NewPath("spec", "schemaVersion"), - db.Spec.SchemaVersion, - "cannot set an explicit schemaVersion without also setting spec.documentDBVersion or spec.documentDBImage; "+ - "the webhook needs a binary version to validate against", - )} - } - - schemaExtensionVersion := util.SemverToExtensionVersion(db.Spec.SchemaVersion) - binaryExtensionVersion := util.SemverToExtensionVersion(binaryVersion) - - cmp, err := util.CompareExtensionVersions(schemaExtensionVersion, binaryExtensionVersion) - if err != nil { - return field.ErrorList{field.Invalid( - field.NewPath("spec", "schemaVersion"), - db.Spec.SchemaVersion, - fmt.Sprintf("cannot validate schemaVersion: version comparison failed: %v", err), - )} - } - if cmp > 0 { - return field.ErrorList{field.Invalid( - field.NewPath("spec", "schemaVersion"), - db.Spec.SchemaVersion, - fmt.Sprintf("schemaVersion %s exceeds the binary version %s; schema version must be <= binary version", - db.Spec.SchemaVersion, binaryVersion), - )} - } - return nil + return validation.SchemaVersionNotExceedsBinary(db.Spec.SchemaVersion, resolveBinaryVersion(db), documentDBTerms) } // --------------------------------------------------------------------------- @@ -201,32 +169,7 @@ func (v *DocumentDBValidator) validateImageRollback(newDB, oldDB *dbpreview.Docu return nil } - newBinaryVersion := resolveBinaryVersion(newDB) - if newBinaryVersion == "" { - return nil - } - - newBinaryExtensionVersion := util.SemverToExtensionVersion(newBinaryVersion) - schemaExtensionVersion := util.SemverToExtensionVersion(installedSchemaVersion) - - cmp, err := util.CompareExtensionVersions(newBinaryExtensionVersion, schemaExtensionVersion) - if err != nil { - return field.ErrorList{field.Forbidden( - field.NewPath("spec"), - fmt.Sprintf("cannot validate image rollback: version comparison failed: %v", err), - )} - } - if cmp < 0 { - return field.ErrorList{field.Forbidden( - field.NewPath("spec"), - fmt.Sprintf( - "image rollback blocked: requested version %s is older than installed schema version %s. "+ - "ALTER EXTENSION has no downgrade path — running an older binary with a newer schema may cause data corruption. "+ - "To recover, restore from backup or update to a version >= %s.", - newBinaryVersion, installedSchemaVersion, installedSchemaVersion), - )} - } - return nil + return validation.ImageRollback(installedSchemaVersion, resolveBinaryVersion(newDB)) } // --------------------------------------------------------------------------- @@ -275,124 +218,32 @@ func (v *DocumentDBValidator) validateRestoreSchemaCompatibility(ctx context.Con return warnings, nil } - return compareBinaryToSchema(resolveEffectiveBinaryVersion(newDB), schemaVersion, src) -} - -// restoreSource identifies where a restore draws its data — and thus its schema -// version — from, carrying the diagnostics and the spec field to flag on error. -type restoreSource struct { - kind string // sourceKindBackup or sourceKindPV - name string - fieldPath *field.Path + return validation.CompareBinaryToSchema(resolveEffectiveBinaryVersion(newDB), schemaVersion, src, documentDBTerms) } -const ( - sourceKindBackup = "backup" - sourceKindPV = "PersistentVolume" -) - // restoreSourceFor identifies the restore source from a recovery configuration, // returning false when neither a backup nor a PersistentVolume source is set. -func restoreSourceFor(recovery *dbpreview.RecoveryConfiguration) (restoreSource, bool) { +func restoreSourceFor(recovery *dbpreview.RecoveryConfiguration) (validation.RestoreSource, bool) { + base := field.NewPath("spec", "bootstrap", "recovery") if recovery.Backup.Name != "" { - return restoreSource{ - kind: sourceKindBackup, - name: recovery.Backup.Name, - fieldPath: field.NewPath("spec", "bootstrap", "recovery", "backup"), - }, true + return validation.BackupSource(base, recovery.Backup.Name), true } if recovery.PersistentVolume != nil && recovery.PersistentVolume.Name != "" { - return restoreSource{ - kind: sourceKindPV, - name: recovery.PersistentVolume.Name, - fieldPath: field.NewPath("spec", "bootstrap", "recovery", "persistentVolume"), - }, true + return validation.PVSource(base, recovery.PersistentVolume.Name), true } - return restoreSource{}, false + return validation.RestoreSource{}, false } // resolveSourceSchemaVersion reads the restore source and returns its recorded -// schema version. It only reports warnings for I/O failures (source not found or -// unreadable); a successfully read source with no schema version returns ("", -// nil), leaving the "unknown schema" policy to compareBinaryToSchema so that both -// empty-input cases are decided in one place. Backup and PV are handled -// identically. -func (v *DocumentDBValidator) resolveSourceSchemaVersion(ctx context.Context, namespace string, src restoreSource) (string, admission.Warnings) { - var schemaVersion string - var readErr error - - switch src.kind { - case sourceKindBackup: +// schema version. See validation.SourceSchemaVersion. +func (v *DocumentDBValidator) resolveSourceSchemaVersion(ctx context.Context, namespace string, src validation.RestoreSource) (string, admission.Warnings) { + return validation.SourceSchemaVersion(ctx, v.Client, src, util.AnnotationSchemaVersion, func() (string, error) { backup := &dbpreview.Backup{} - if err := v.Get(ctx, client.ObjectKey{Name: src.name, Namespace: namespace}, backup); err != nil { - readErr = err - } else { - schemaVersion = backup.Status.SchemaVersion - } - case sourceKindPV: - pv := &corev1.PersistentVolume{} - if err := v.Get(ctx, client.ObjectKey{Name: src.name}, pv); err != nil { - readErr = err - } else { - schemaVersion = pv.Annotations[util.AnnotationSchemaVersion] - } - } - - if readErr != nil { - if apierrors.IsNotFound(readErr) { - return "", admission.Warnings{ - fmt.Sprintf("%s %q not found: schema-version compatibility cannot be verified", src.kind, src.name), - } - } - // Transient/API error: don't hard-block the restore, but warn. - return "", admission.Warnings{ - fmt.Sprintf("failed to read %s %q: schema-version compatibility cannot be verified: %v", src.kind, src.name, readErr), + if err := v.Get(ctx, client.ObjectKey{Name: src.Name, Namespace: namespace}, backup); err != nil { + return "", err } - } - return schemaVersion, nil -} - -// compareBinaryToSchema is the single authority on the restore decision. It warns -// (and allows) when either the source schema version or the target DocumentDB -// version is unknown, and rejects only a known target version that is older than a -// known schema. -func compareBinaryToSchema(binaryVersion, schemaVersion string, src restoreSource) (admission.Warnings, field.ErrorList) { - if schemaVersion == "" { - return admission.Warnings{ - fmt.Sprintf("%s %q has no recorded schema version: schema-version compatibility cannot be verified; "+ - "ensure the target DocumentDB version (spec.documentDBVersion or spec.image.documentDB) is >= the source's "+ - "schema version to avoid data corruption", src.kind, src.name), - }, nil - } - if binaryVersion == "" { - return admission.Warnings{ - fmt.Sprintf("cannot determine the target DocumentDB version for restore from %s %q "+ - "(set spec.documentDBVersion or spec.image.documentDB): compatibility with the %s's schema version %s cannot be verified", - src.kind, src.name, src.kind, schemaVersion), - }, nil - } - - binaryExtensionVersion := util.SemverToExtensionVersion(binaryVersion) - schemaExtensionVersion := util.SemverToExtensionVersion(schemaVersion) - - cmp, err := util.CompareExtensionVersions(binaryExtensionVersion, schemaExtensionVersion) - if err != nil { - return admission.Warnings{ - fmt.Sprintf("cannot compare the target DocumentDB version %s with the %s's schema version %s: %v; compatibility not verified", - binaryVersion, src.kind, schemaVersion, err), - }, nil - } - if cmp < 0 { - return nil, field.ErrorList{field.Forbidden( - src.fieldPath, - fmt.Sprintf( - "restore blocked: the target DocumentDB version %s is older than the %s's schema version %s. "+ - "Restoring onto an older DocumentDB version runs it against a newer, irreversible schema and may cause data corruption. "+ - "Set spec.documentDBVersion (or spec.image.documentDB) to %s or newer.", - binaryVersion, src.kind, schemaVersion, schemaVersion), - )} - } - return nil, nil + return backup.Status.SchemaVersion, nil + }) } // validateImmutableFields rejects updates to fields that cannot be changed after creation. @@ -418,44 +269,8 @@ func (v *DocumentDBValidator) validateImmutableFields(newDB, oldDB *dbpreview.Do // validateStorageResize ensures PVC size can only grow, never shrink. func (v *DocumentDBValidator) validateStorageResize(newDB, oldDB *dbpreview.DocumentDB) field.ErrorList { - oldSize := oldDB.Spec.Resource.Storage.PvcSize - newSize := newDB.Spec.Resource.Storage.PvcSize - if oldSize == newSize { - return nil - } - - pvcSizePath := field.NewPath("spec", "resource", "storage", "pvcSize") - var allErrs field.ErrorList - - oldQty, errOld := resource.ParseQuantity(oldSize) - if errOld != nil { - allErrs = append(allErrs, field.Invalid( - pvcSizePath, - oldSize, - fmt.Sprintf("existing pvcSize is not a valid resource quantity: %v", errOld), - )) - } - - newQty, errNew := resource.ParseQuantity(newSize) - if errNew != nil { - allErrs = append(allErrs, field.Invalid( - pvcSizePath, - newSize, - fmt.Sprintf("pvcSize must be a valid resource quantity: %v", errNew), - )) - } - - if len(allErrs) > 0 { - return allErrs - } - - if newQty.Cmp(oldQty) < 0 { - return field.ErrorList{field.Forbidden( - pvcSizePath, - fmt.Sprintf("storage size can only be increased; attempted shrink from %s to %s", oldSize, newSize), - )} - } - return nil + return validation.StorageResize(field.NewPath("spec", "resource", "storage", "pvcSize"), + oldDB.Spec.Resource.Storage.PvcSize, newDB.Spec.Resource.Storage.PvcSize) } // isBootstrapEqual compares two BootstrapConfiguration pointers for equality. @@ -478,19 +293,7 @@ func isBootstrapEqual(a, b *dbpreview.BootstrapConfiguration) bool { // Digest-only references (e.g., "image@sha256:...") are not parseable as versions // and return "". func resolveBinaryVersion(db *dbpreview.DocumentDB) string { - if ref := specImageDocumentDB(db); ref != "" && !strings.Contains(ref, "@sha256:") { - if tagIdx := strings.LastIndex(ref, ":"); tagIdx >= 0 { - tag := ref[tagIdx+1:] - // Extract leading semver (X.Y.Z) from tags like "0.112.0-amd64" - if semver := extractSemver(tag); semver != "" { - return semver - } - } - } - if version := extractSemver(db.Spec.DocumentDBVersion); version != "" { - return version - } - return db.Spec.DocumentDBVersion + return validation.ResolveBinaryVersion(specImageDocumentDB(db), db.Spec.DocumentDBVersion) } // resolveEffectiveBinaryVersion returns the binary version the operator will @@ -508,13 +311,7 @@ func resolveEffectiveBinaryVersion(db *dbpreview.DocumentDB) string { // helper, and read its semver tag. Anything without a parseable semver tag // (a digest, or the changestream image) stays unknown so the restore is warned, // not falsely blocked — keeping the webhook in step with the controller. - image := product.DocumentDBAdapter{}.ExtensionImage(db) - if tagIdx := strings.LastIndex(image, ":"); tagIdx >= 0 { - if semver := extractSemver(image[tagIdx+1:]); semver != "" { - return semver - } - } - return "" + return validation.ImageTagSemver(product.DocumentDBAdapter{}.ExtensionImage(db)) } func specImageDocumentDB(db *dbpreview.DocumentDB) string { @@ -524,33 +321,6 @@ func specImageDocumentDB(db *dbpreview.DocumentDB) string { return db.Spec.Image.DocumentDB } -// extractSemver returns the leading "X.Y.Z" portion from a tag string, -// normalizing "X.Y-RC" tags to the extension version "X.Y.0". -// It returns "" when neither version format is recognized. -func extractSemver(tag string) string { - if matches := rcImageTagPattern.FindStringSubmatch(tag); matches != nil { - return matches[1] + "." + matches[2] + ".0" - } - // Match digits.digits.digits at start of string - parts := strings.SplitN(tag, ".", 3) - if len(parts) < 3 { - return "" - } - // Validate major and minor are numeric - if _, err := strconv.Atoi(parts[0]); err != nil { - return "" - } - if _, err := strconv.Atoi(parts[1]); err != nil { - return "" - } - // Third part may have a suffix (e.g., "0-amd64"), take only leading digits - thirdPart := parts[2] - i := 0 - for i < len(thirdPart) && thirdPart[i] >= '0' && thirdPart[i] <= '9' { - i++ - } - if i == 0 { - return "" - } - return parts[0] + "." + parts[1] + "." + thirdPart[:i] -} +// extractSemver returns the leading "X.Y.Z" portion from a tag string. +// See validation.ExtractSemver. +func extractSemver(tag string) string { return validation.ExtractSemver(tag) } diff --git a/operator/src/internal/webhook/imagevolume.go b/operator/src/internal/webhook/imagevolume.go index 1e9e771b..ec76af94 100644 --- a/operator/src/internal/webhook/imagevolume.go +++ b/operator/src/internal/webhook/imagevolume.go @@ -5,173 +5,32 @@ package webhook import ( "context" - "fmt" "sync/atomic" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "sigs.k8s.io/controller-runtime/pkg/client" + "github.com/documentdb/documentdb-operator/internal/validation" ) -// imageVolumeProbeImage is a syntactically valid image reference used only for -// the dry-run capability probe. It is never pulled or scheduled because every -// probe Pod is submitted with client.DryRunAll. -const imageVolumeProbeImage = "registry.k8s.io/pause:3.10" - -// imageVolumeProbeVolumeName is the volume/mount name shared by both probe Pods. -const imageVolumeProbeVolumeName = "probe" - // imageVolumeConfirmed caches a positive detection of ImageVolume support. -// -// Only positive results are cached: once the cluster is known to support -// ImageVolume, subsequent creates skip the probe entirely (steady-state -// latency ~0). A negative or inconclusive result is never cached, so if an -// administrator enables the feature gate later the next create re-probes and -// succeeds without restarting the operator. +// See validation.ImageVolumeProbe.Confirmed. var imageVolumeConfirmed atomic.Bool -// ensureImageVolumeSupported blocks DocumentDB creation when the cluster does -// not support the Kubernetes ImageVolume feature (feature gate off, or a -// runtime/version that lacks it). The operator mounts the DocumentDB extension -// into the PostgreSQL pod via a corev1.ImageVolumeSource, so without this -// feature the cluster can never become ready. -// -// The check is intentionally fail-open: it blocks only when it can positively -// prove the feature is unavailable. If the cluster environment prevents a -// conclusive answer (e.g. Pod Security Admission or a ResourceQuota rejects the -// baseline probe Pod), creation is allowed and the reconcile loop surfaces any -// real failure later. -func (v *DocumentDBValidator) ensureImageVolumeSupported(ctx context.Context, namespace string) error { - if imageVolumeConfirmed.Load() { - return nil - } - - supported, conclusive := v.probeImageVolume(ctx, namespace) - if !conclusive { - // Could not determine support; do not block on an inconclusive probe. - return nil +func (v *DocumentDBValidator) imageVolumeProbe() *validation.ImageVolumeProbe { + return &validation.ImageVolumeProbe{ + Client: v.Client, + Confirmed: &imageVolumeConfirmed, + Product: "DocumentDB", + Log: documentdbLog, } - if supported { - imageVolumeConfirmed.Store(true) - return nil - } - - return fmt.Errorf( - "the Kubernetes ImageVolume feature is not enabled on this cluster, but it is required by the DocumentDB " + - "operator to mount the DocumentDB extension into PostgreSQL pods. ImageVolume is GA (on by default) in " + - "Kubernetes 1.35+. On Kubernetes 1.33/1.34 you may enable the ImageVolume feature gate on both the " + - "kube-apiserver and every kubelet; the container runtime must also support it (containerd >= 2.1 or " + - "CRI-O >= 1.33), which this preflight cannot verify. " + - "See https://kubernetes.io/docs/concepts/storage/volumes/#image", - ) } -// probeImageVolume performs a differential server-side dry-run to detect -// ImageVolume support without relying on error-string matching. -// -// It submits two dry-run Pods that are identical except for a single volume: -// - baseline: an emptyDir volume (always a valid volume source), and -// - image: an ImageVolumeSource volume. -// -// Interpretation: -// - baseline rejected -> the environment blocks pod creation -// entirely (PSA, quota, other webhooks); inconclusive. -// - baseline ok, image rejected -> the image volume is the sole difference, -// so ImageVolume is unsupported. -// - baseline ok, image accepted but the ImageVolumeSource was pruned from the -// returned Pod -> the feature gate is off, so the API server silently drops -// the disabled field instead of rejecting it; ImageVolume is unsupported. -// - baseline ok, image accepted and the ImageVolumeSource survived -> the -// feature is supported. -// -// The pruning case is the important one: on Kubernetes 1.33/1.34 with the -// ImageVolume beta gate OFF, a dry-run create does NOT fail; the API server -// admits the Pod and simply strips the unknown/disabled volume source. Relying -// on a create error alone therefore false-positives as "supported". We close -// that gap by inspecting the returned object for field retention. -// -// The second return value reports whether the result is conclusive. -func (v *DocumentDBValidator) probeImageVolume(ctx context.Context, namespace string) (supported bool, conclusive bool) { - if _, err := v.dryRunProbePod(ctx, namespace, baselineProbeVolume()); err != nil { - documentdbLog.Info("ImageVolume preflight inconclusive: baseline probe pod rejected", "error", err) - return false, false - } - - returned, err := v.dryRunProbePod(ctx, namespace, imageProbeVolume()) - if err != nil { - documentdbLog.Info("ImageVolume preflight: image-volume probe pod rejected while baseline passed; feature unavailable", "error", err) - return false, true - } - - if !hasImageVolume(returned) { - documentdbLog.Info("ImageVolume preflight: image-volume probe pod admitted but the ImageVolumeSource was pruned from the returned spec; feature gate is off, feature unavailable") - return false, true - } - - return true, true -} - -// hasImageVolume reports whether the server-returned Pod still carries the probe -// volume as an ImageVolumeSource. When the ImageVolume feature gate is off the -// API server prunes the field, so its absence means the feature is unavailable. -func hasImageVolume(pod *corev1.Pod) bool { - if pod == nil { - return false - } - for _, vol := range pod.Spec.Volumes { - if vol.Name == imageVolumeProbeVolumeName { - return vol.Image != nil - } - } - return false -} - -// dryRunProbePod submits a minimal Pod carrying the given probe volume as a -// server-side dry-run (never persisted) and returns the Pod as processed by the -// API server (with defaulting and field pruning applied) alongside its verdict. -func (v *DocumentDBValidator) dryRunProbePod(ctx context.Context, namespace string, volume corev1.Volume) (*corev1.Pod, error) { - pod := &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - GenerateName: "documentdb-imagevolume-preflight-", - Namespace: namespace, - }, - Spec: corev1.PodSpec{ - Containers: []corev1.Container{{ - Name: "probe", - Image: imageVolumeProbeImage, - VolumeMounts: []corev1.VolumeMount{{ - Name: imageVolumeProbeVolumeName, - MountPath: "/probe", - }}, - }}, - Volumes: []corev1.Volume{volume}, - }, - } - err := v.Create(ctx, pod, client.DryRunAll) - return pod, err -} - -// baselineProbeVolume returns an emptyDir volume, which every Kubernetes -// version accepts. It establishes whether the environment permits creating the -// probe Pod at all. -func baselineProbeVolume() corev1.Volume { - return corev1.Volume{ - Name: imageVolumeProbeVolumeName, - VolumeSource: corev1.VolumeSource{ - EmptyDir: &corev1.EmptyDirVolumeSource{}, - }, - } +// ensureImageVolumeSupported blocks DocumentDB creation when the cluster does +// not support the Kubernetes ImageVolume feature. See validation.ImageVolumeProbe. +func (v *DocumentDBValidator) ensureImageVolumeSupported(ctx context.Context, namespace string) error { + return v.imageVolumeProbe().Ensure(ctx, namespace) } -// imageProbeVolume returns an ImageVolumeSource volume, the feature under test. -func imageProbeVolume() corev1.Volume { - return corev1.Volume{ - Name: imageVolumeProbeVolumeName, - VolumeSource: corev1.VolumeSource{ - Image: &corev1.ImageVolumeSource{ - Reference: imageVolumeProbeImage, - PullPolicy: corev1.PullIfNotPresent, - }, - }, - } +// probeImageVolume performs the differential dry-run probe. The second return +// value reports whether the result is conclusive. +func (v *DocumentDBValidator) probeImageVolume(ctx context.Context, namespace string) (supported bool, conclusive bool) { + return v.imageVolumeProbe().Probe(ctx, namespace) } From f6d08a54ae4d3ce9548596632edd6b33e5ab2758 Mon Sep 17 00:00:00 2001 From: Rayhan Hossain Date: Tue, 20 Oct 2026 10:16:54 -0700 Subject: [PATCH 19/27] feat(core): expose generic engines and validation in pkg/core Re-export ClusterReconciler, CertificateReconciler, PersistentVolumeReconciler, Backup/ScheduledBackup reconcilers, adapters and shared admission validation through pkg/core, and add a test driving a fake second product (Account CR) through the cluster reconciler using only the public API. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 33b3cb3c-1b25-4ffb-bff1-ebf0f3d04058 Signed-off-by: Rayhan Hossain --- operator/src/pkg/core/engine.go | 140 ++++++++++++++++ operator/src/pkg/core/engine_test.go | 229 +++++++++++++++++++++++++++ 2 files changed, 369 insertions(+) create mode 100644 operator/src/pkg/core/engine.go create mode 100644 operator/src/pkg/core/engine_test.go diff --git a/operator/src/pkg/core/engine.go b/operator/src/pkg/core/engine.go new file mode 100644 index 00000000..ba37e206 --- /dev/null +++ b/operator/src/pkg/core/engine.go @@ -0,0 +1,140 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package core + +import ( + "context" + "sync/atomic" + + "github.com/go-logr/logr" + "k8s.io/apimachinery/pkg/util/validation/field" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/predicate" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" + + "github.com/documentdb/documentdb-operator/internal/engine" + "github.com/documentdb/documentdb-operator/internal/product" + util "github.com/documentdb/documentdb-operator/internal/utils" + "github.com/documentdb/documentdb-operator/internal/validation" +) + +// Adapter contract. A product implements ClusterAdapter for its cluster custom +// resource, and BackupAdapter/ScheduledBackupAdapter when it exposes backups. +type ( + NamingProfile = product.NamingProfile + ClusterSpec = product.ClusterSpec + ClusterStatus = product.ClusterStatus + ClusterAdapter[T client.Object] = product.ClusterAdapter[T] + BackupAdapter[B client.Object] = engine.BackupAdapter[B] + ScheduledBackupAdapter[S client.Object] = engine.ScheduledBackupAdapter[S] +) + +// Generic reconcilers. Each is configured with the product adapters and +// registered with SetupWithManager. +type ( + ClusterReconciler[T client.Object] = engine.ClusterReconciler[T] + CertificateReconciler[T client.Object] = engine.CertificateReconciler[T] + PersistentVolumeReconciler[T client.Object] = engine.PersistentVolumeReconciler[T] + BackupReconciler[T, B client.Object] = engine.BackupReconciler[T, B] + ScheduledBackupReconciler[T, S, B client.Object] = engine.ScheduledBackupReconciler[T, S, B] + + // ReplicationHooks is the optional multi-region extension point of the + // cluster reconciler; nil runs every cluster as a single-region primary. + ReplicationHooks[T client.Object] = engine.ReplicationHooks[T] + ReplicationContext = util.ReplicationContext + SQLExecutor = engine.SQLExecutor +) + +const ( + // RequeueAfterShort and RequeueAfterLong are the reconcilers' retry delays. + RequeueAfterShort = engine.RequeueAfterShort + RequeueAfterLong = engine.RequeueAfterLong + // BackupClusterField is the Backup field index the scheduled-backup + // reconciler registers. + BackupClusterField = engine.BackupClusterField +) + +// NewPodSQLExecutor returns an SQLExecutor that runs psql in the CNPG primary pod. +func NewPodSQLExecutor(c client.Client, clientset kubernetes.Interface, config *rest.Config) SQLExecutor { + return engine.NewPodSQLExecutor(c, clientset, config) +} + +// NewSingleRegionContext returns the replication context of a single-region +// primary cluster. +func NewSingleRegionContext(name, environment, storageClass string) *ReplicationContext { + return util.NewSingleRegionContext(name, environment, storageClass) +} + +// GatewayPort is the gateway port of a profile, or the GATEWAY_PORT env default. +func GatewayPort(p ProductProfile) int32 { return engine.GatewayPort(p) } + +// ClusterInstanceStatusChangedPredicate triggers on CNPG instance status changes. +func ClusterInstanceStatusChangedPredicate() predicate.Predicate { + return engine.ClusterInstanceStatusChangedPredicate() +} + +// ServicePrefixPredicate matches Services whose name starts with prefix. +func ServicePrefixPredicate(prefix string) predicate.Predicate { + return engine.ServicePrefixPredicate(prefix) +} + +// Admission validation shared by product webhooks. +type ( + ValidationTerms = validation.Terms + RestoreSource = validation.RestoreSource + ImageVolumeProbe = validation.ImageVolumeProbe +) + +// NewImageVolumeProbe returns the ImageVolume preflight for product. confirmed +// caches a positive result and should be shared across calls. +func NewImageVolumeProbe(c client.Client, confirmed *atomic.Bool, productName string, log logr.Logger) *ImageVolumeProbe { + return &validation.ImageVolumeProbe{Client: c, Confirmed: confirmed, Product: productName, Log: log} +} + +// ResolveBinaryVersion returns the extension version pinned by an image tag or +// version field, or "". +func ResolveBinaryVersion(imageRef, version string) string { + return validation.ResolveBinaryVersion(imageRef, version) +} + +// ImageTagSemver returns the semver of an image reference's tag, or "". +func ImageTagSemver(ref string) string { return validation.ImageTagSemver(ref) } + +// ValidateSchemaVersion rejects an explicit schemaVersion above the binary version. +func ValidateSchemaVersion(schemaVersion, binaryVersion string, t ValidationTerms) field.ErrorList { + return validation.SchemaVersionNotExceedsBinary(schemaVersion, binaryVersion, t) +} + +// ValidateImageRollback rejects a binary version below the installed schema. +func ValidateImageRollback(installedSchemaVersion, newBinaryVersion string) field.ErrorList { + return validation.ImageRollback(installedSchemaVersion, newBinaryVersion) +} + +// ValidateStorageResize rejects PVC shrinks. +func ValidateStorageResize(path *field.Path, oldSize, newSize string) field.ErrorList { + return validation.StorageResize(path, oldSize, newSize) +} + +// BackupRestoreSource and PVRestoreSource identify the source of a recovery +// stanza at base. +func BackupRestoreSource(base *field.Path, name string) RestoreSource { + return validation.BackupSource(base, name) +} + +func PVRestoreSource(base *field.Path, name string) RestoreSource { + return validation.PVSource(base, name) +} + +// RestoreSourceSchemaVersion reads the schema version recorded on a restore +// source; readBackup reads it from the product's Backup. +func RestoreSourceSchemaVersion(ctx context.Context, c client.Reader, src RestoreSource, naming NamingProfile, readBackup func() (string, error)) (string, admission.Warnings) { + return validation.SourceSchemaVersion(ctx, c, src, naming.SchemaVersionAnnotation(), readBackup) +} + +// ValidateRestoreCompatibility rejects restoring a schema newer than the binary. +func ValidateRestoreCompatibility(binaryVersion, schemaVersion string, src RestoreSource, t ValidationTerms) (admission.Warnings, field.ErrorList) { + return validation.CompareBinaryToSchema(binaryVersion, schemaVersion, src, t) +} diff --git a/operator/src/pkg/core/engine_test.go b/operator/src/pkg/core/engine_test.go new file mode 100644 index 00000000..1c7c8ea1 --- /dev/null +++ b/operator/src/pkg/core/engine_test.go @@ -0,0 +1,229 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package core_test + +import ( + "context" + "strings" + "testing" + + cnpgv1 "github.com/cloudnative-pg/cloudnative-pg/api/v1" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/tools/record" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + "github.com/documentdb/documentdb-operator/pkg/core" +) + +// account is a second product's cluster custom resource, built only against +// the public pkg/core surface. +type account struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + Spec accountSpec `json:"spec,omitempty"` + Status accountStatus `json:"status,omitempty"` +} + +type accountSpec struct { + Instances int `json:"instances"` + PvcSize string `json:"pvcSize"` +} + +type accountStatus struct { + State string `json:"state,omitempty"` + Endpoint string `json:"endpoint,omitempty"` + SchemaVersion string `json:"schemaVersion,omitempty"` + Engine string `json:"engine,omitempty"` + Gateway string `json:"gateway,omitempty"` +} + +func (a *account) DeepCopyObject() runtime.Object { + out := *a + a.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + return &out +} + +type accountList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []account `json:"items"` +} + +func (l *accountList) DeepCopyObject() runtime.Object { + out := *l + out.Items = make([]account, len(l.Items)) + for i := range l.Items { + out.Items[i] = *l.Items[i].DeepCopyObject().(*account) + } + return &out +} + +var accountGV = schema.GroupVersion{Group: "cosmos.example.com", Version: "v1"} + +type accountAdapter struct{} + +var _ core.ClusterAdapter[*account] = accountAdapter{} + +func (accountAdapter) Profile() core.ProductProfile { + return core.ProductProfile{ + Name: "Cosmos", + SidecarInjectorPlugin: "sidecar.cosmos.example.com", + Postgres: core.PostgresProfile{ + Extension: core.ExtensionSpec{Name: "pgcosmos"}, + }, + Gateway: core.GatewayProfile{ContainerName: "pgcosmos-gateway", Port: 8081}, + Naming: core.NamingProfile{ + Domain: "cosmos.example.com", + ServicePrefix: "cosmos-", + OperatorName: "cosmos-operator", + Kind: "Account", + }, + } +} + +func (accountAdapter) New() *account { return &account{} } + +func (a accountAdapter) Intent(obj *account) core.ClusterIntent { + return core.ClusterIntent{ + Images: core.Images{PostgresExtension: "reg/pgcosmos:0.110.0", Gateway: "reg/pgcosmos-gateway:0.110.0"}, + Topology: core.Topology{Instances: obj.Spec.Instances}, + Storage: core.Storage{PvcSize: obj.Spec.PvcSize}, + Identity: core.Identity{Name: obj.Name, APIVersion: accountGV.String(), Kind: "Account", UID: obj.UID}, + CredentialSecret: "cosmos-credentials", + SidecarInjectorPlugin: "sidecar.cosmos.example.com", + Product: a.Profile(), + } +} + +func (accountAdapter) Spec(*account) core.ClusterSpec { return core.ClusterSpec{} } + +func (accountAdapter) Status(obj *account) core.ClusterStatus { + return core.ClusterStatus{ + Phase: obj.Status.State, + ConnectionString: obj.Status.Endpoint, + SchemaVersion: obj.Status.SchemaVersion, + ExtensionImage: obj.Status.Engine, + GatewayImage: obj.Status.Gateway, + } +} + +func (accountAdapter) SetStatus(obj *account, s core.ClusterStatus) { + obj.Status.State = s.Phase + obj.Status.Endpoint = s.ConnectionString + obj.Status.SchemaVersion = s.SchemaVersion + obj.Status.Engine = s.ExtensionImage + obj.Status.Gateway = s.GatewayImage +} + +func (accountAdapter) ConnectionString(obj *account, host string, port int32, _ bool) string { + return "AccountEndpoint=https://" + host + "/" +} + +// TestClusterReconcilerSecondProduct drives the shared cluster reconciler end +// to end for a non-DocumentDB custom resource on a fake API server. +func TestClusterReconcilerSecondProduct(t *testing.T) { + ctx := context.Background() + scheme := runtime.NewScheme() + if err := clientgoscheme.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + if err := cnpgv1.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + scheme.AddKnownTypes(accountGV, &account{}, &accountList{}) + metav1.AddToGroupVersion(scheme, accountGV) + + acct := &account{ + ObjectMeta: metav1.ObjectMeta{Name: "acct", Namespace: "ns", UID: "uid-1"}, + Spec: accountSpec{Instances: 1, PvcSize: "10Gi"}, + } + c := fake.NewClientBuilder().WithScheme(scheme). + WithObjects(acct). + WithStatusSubresource(&account{}, &cnpgv1.Cluster{}). + Build() + + var sql []string + r := &core.ClusterReconciler[*account]{ + Client: c, + Scheme: scheme, + Recorder: record.NewFakeRecorder(10), + Adapter: accountAdapter{}, + SQLExecutor: func(_ context.Context, _ *cnpgv1.Cluster, cmd string) (string, error) { + sql = append(sql, cmd) + return " default_version | installed_version\n-----------------+-------------------\n 0.110-0 | 0.110-0\n", nil + }, + } + req := ctrl.Request{NamespacedName: types.NamespacedName{Name: "acct", Namespace: "ns"}} + key := client.ObjectKey{Name: "acct", Namespace: "ns"} + + // 1. The product finalizer is added first. + if _, err := r.Reconcile(ctx, req); err != nil { + t.Fatal(err) + } + got := &account{} + if err := c.Get(ctx, key, got); err != nil { + t.Fatal(err) + } + if want := "cosmos.example.com/pv-retention-finalizer"; len(got.Finalizers) != 1 || got.Finalizers[0] != want { + t.Fatalf("finalizers = %v, want [%s]", got.Finalizers, want) + } + + // 2. RBAC and the CNPG Cluster are created from the product intent. + res, err := r.Reconcile(ctx, req) + if err != nil { + t.Fatal(err) + } + if res.RequeueAfter != core.RequeueAfterLong { + t.Fatalf("requeue = %v, want %v", res.RequeueAfter, core.RequeueAfterLong) + } + for _, obj := range []client.Object{&corev1.ServiceAccount{}, &rbacv1.Role{}, &rbacv1.RoleBinding{}} { + if err := c.Get(ctx, key, obj); err != nil { + t.Fatalf("%T: %v", obj, err) + } + } + cluster := &cnpgv1.Cluster{} + if err := c.Get(ctx, key, cluster); err != nil { + t.Fatal(err) + } + if ext := cluster.Spec.PostgresConfiguration.Extensions; len(ext) != 1 || ext[0].Name != "pgcosmos" { + t.Fatalf("extensions = %+v, want pgcosmos", ext) + } + if img := core.ExtensionImage(cluster, "pgcosmos"); img != "reg/pgcosmos:0.110.0" { + t.Fatalf("extension image = %q", img) + } + + // 3. Once CNPG reports a healthy primary, status and schema version are recorded. + cluster.Status.Phase = "Cluster in healthy state" + cluster.Status.CurrentPrimary = "acct-1" + cluster.Status.InstancesStatus = map[cnpgv1.PodStatus][]string{cnpgv1.PodHealthy: {"acct-1"}} + if err := c.Status().Update(ctx, cluster); err != nil { + t.Fatal(err) + } + if _, err := r.Reconcile(ctx, req); err != nil { + t.Fatal(err) + } + if err := c.Get(ctx, key, got); err != nil { + t.Fatal(err) + } + want := accountStatus{ + State: "Cluster in healthy state", + SchemaVersion: "0.110.0", + Engine: "reg/pgcosmos:0.110.0", + Gateway: "reg/pgcosmos-gateway:0.110.0", + } + if got.Status != want { + t.Fatalf("status = %+v, want %+v", got.Status, want) + } + if len(sql) != 1 || !strings.Contains(sql[0], "'pgcosmos'") { + t.Fatalf("sql = %q, want one pgcosmos version query", sql) + } +} From b376b69e9bba3c50f720f65b480764a464dea5d1 Mon Sep 17 00:00:00 2001 From: Rayhan Hossain Date: Tue, 20 Oct 2026 10:19:21 -0700 Subject: [PATCH 20/27] docs: describe the generic engines and validate the full refactor make manifests generate yields no diff; all modules vet; operator and sidecar tests pass; lint findings unchanged versus origin/main. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 33b3cb3c-1b25-4ffb-bff1-ebf0f3d04058 Signed-off-by: Rayhan Hossain --- CHANGELOG.md | 2 +- docs/designs/product-neutral-core.md | 88 ++++++++++++++++++++++------ 2 files changed, 72 insertions(+), 18 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 66f12be3..785f2037 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,7 @@ - Require Go 1.26.10 across all six modules and pin Go builder images to that patch release to address [GO-2026-6809](https://pkg.go.dev/vuln/GO-2026-6809) in `crypto/x509`. Unit-test workflows now resolve Go from the operator module instead of an older hard-coded version. ### Major Features -- **Product-neutral operator core**: The CNPG `Cluster` renderer, sync, PostgreSQL parameter merge, resource split/validation and extension lifecycle SQL are now driven by a `ProductProfile` and `ClusterIntent` instead of the `DocumentDB` custom resource, and are exported for other operator modules through `operator/src/pkg/core`. The sidecar injector accepts optional `gatewayContainerName`, `gatewayPort`, `gatewayArgs`, `gatewayUsernameEnv` and `gatewayPasswordEnv` parameters for non-DocumentDB gateways. Rendered DocumentDB clusters are byte-identical to before, so existing deployments see no change. See [Product-neutral operator core](docs/designs/product-neutral-core.md). +- **Product-neutral operator core**: The CNPG `Cluster` renderer, sync, PostgreSQL parameter merge, resource split/validation and extension lifecycle SQL are now driven by a `ProductProfile` and `ClusterIntent` instead of the `DocumentDB` custom resource, and are exported for other operator modules through `operator/src/pkg/core`. The sidecar injector accepts optional `gatewayContainerName`, `gatewayPort`, `gatewayArgs`, `gatewayUsernameEnv` and `gatewayPasswordEnv` parameters for non-DocumentDB gateways. The cluster, certificate, persistent volume, backup and scheduled-backup reconcilers and the admission validation rules are generic over the custom resource type through a `ClusterAdapter`, with DocumentDB as the first implementation; `pkg/core` exports them too. Rendered DocumentDB clusters are byte-identical to before, CRDs and RBAC are unchanged, and existing deployments see no change. See [Product-neutral operator core](docs/designs/product-neutral-core.md). ## [1.0.0] - 2026-10-19 diff --git a/docs/designs/product-neutral-core.md b/docs/designs/product-neutral-core.md index 16ee8167..594849c2 100644 --- a/docs/designs/product-neutral-core.md +++ b/docs/designs/product-neutral-core.md @@ -10,14 +10,22 @@ follow the same pattern with a different extension and gateway. This design separates the CNPG orchestration (rendering the `Cluster`, syncing it, PostgreSQL parameters, the resource split, and the extension lifecycle) from the DocumentDB product, so a second operator can reuse the same -code. DocumentDB behavior is unchanged: golden snapshots of the rendered -`Cluster` are byte-identical before and after the refactor. +code. The reconcilers (cluster, certificate, persistent volume, backup, +scheduled backup) and the admission rules are generic too. DocumentDB behavior +is unchanged: golden snapshots of the rendered `Cluster` are byte-identical +before and after the refactor, and the controller and webhook tests pass +unmodified. ## Layers ``` product CR (DocumentDB, ...) - │ Adapter.ToClusterIntent + │ ClusterAdapter[T]: Intent · Spec · Status/SetStatus · Naming + ▼ + generic reconcilers + admission rules internal/engine, internal/validation + (cluster, certificate, PV, backup, (public: pkg/core) + scheduled backup) + │ ▼ ClusterIntent + ProductProfile internal/product (public: pkg/core) │ @@ -47,6 +55,26 @@ code. DocumentDB behavior is unchanged: golden snapshots of the rendered custom resource to a `ClusterIntent`. It is the only place that reads the DocumentDB spec for rendering. +- **`ClusterAdapter[T]`** (`internal/product/cluster.go`) extends the adapter + for the reconcilers: `New`, `Intent`, a neutral operational `ClusterSpec` + (TLS, exposure, PV reclaim, environment, replication flags), a neutral + `ClusterStatus` read/write, and `ConnectionString`. `NamingProfile` on the + profile holds the finalizer, CR kind, Service prefix, labels, and the OTel + metric attribute. +- **`internal/engine`** holds `ClusterReconciler[T]`, + `CertificateReconciler[T]`, `PersistentVolumeReconciler[T]`, + `BackupReconciler[T,B]`, and `ScheduledBackupReconciler[T,S,B]`. + Multi-region replication plugs in through `ReplicationHooks[T]`; when it is + nil the cluster runs single-region. +- **`internal/validation`** holds the admission rules (schema version vs. + binary, image rollback, storage resize, restore compatibility, ImageVolume + probe). They take `Terms` so error messages use the product's field names. + +The DocumentDB controllers and webhook in `internal/controller` and +`internal/webhook` are now thin wrappers over these packages. They keep their +types, method names, and RBAC markers, so `cmd/main.go`, tests, and generated +manifests are unchanged. + The renderer (`cnpg.GetCnpgClusterSpecFromIntent`) only reads the intent. The old `GetCnpgClusterSpec`, `MergeParameters`, `ProtectedParameters`, `ComputeResourceSplit`, and `ValidateResources` entry points remain as thin @@ -65,6 +93,10 @@ DocumentDB wrappers. | Image sync | `SyncCnpgCluster` reads the extension by name | first `Extensions` entry | | Resource validation path | `ValidateResourcesFor(..., base)` | `spec.resource` | | Gateway sidecar | `GatewayProfile` → plugin params | injector defaults | +| Client Service port | `GatewayProfile.Port`, else `GATEWAY_PORT` | `10260` | +| Service name prefix, finalizer, labels | `NamingProfile` | `documentdb-service-`, `documentdb.io/...` | +| OTel metric attribute | `NamingProfile` | `documentdb.cluster` | +| Admission error wording | `validation.Terms` | `documentDBVersion`, `spec.documentDBImage`, ... | ## Sidecar-injector gateway contract @@ -97,19 +129,42 @@ Go does not allow importing `internal/` packages from another module, so profile types plus `RenderCluster`, `SyncCluster`, `MergeParameters`, `ProtectedParameters`, `ComputeResourceSplit`, `ValidateResources`, `DefaultSplitConfig`, the extension SQL helpers, and `DocumentDBProfile` as a -reference profile. It adds no behavior of its own. +reference profile. `pkg/core/engine.go` re-exports the reconcilers, adapters, +`ReplicationHooks`, `NewPodSQLExecutor`, predicates, and the validation +helpers (`ValidateSchemaVersion`, `ValidateImageRollback`, +`ValidateStorageResize`, `ValidateRestoreCompatibility`, +`NewImageVolumeProbe`). It adds no behavior of its own. A second product: -1. Defines its own CRD and an adapter that returns a `core.ProductProfile` and - builds a `core.ClusterIntent` from its custom resource. -2. Writes its own reconciler. It calls `core.RenderCluster` to build the - desired `Cluster` and `core.SyncCluster` to apply changes, and runs - `core.ExtensionUpdateSQL` / `core.ReplicationRoleGrantSQL` for the - extension lifecycle. +1. Defines its own CRD and a `core.ClusterAdapter[*MyCR]` that returns its + `core.ProductProfile` (including `Naming`) and maps the CR to + `core.ClusterIntent`, `core.ClusterSpec`, and `core.ClusterStatus`. Backup + support adds a `core.BackupAdapter[*MyBackup]` and + `core.ScheduledBackupAdapter[*MySchedule]`. +2. Wires the generic reconcilers in its `main.go`: + + ```go + adapter := cosmos.AccountAdapter{} + (&core.ClusterReconciler[*v1.Account]{ + Client: mgr.GetClient(), Scheme: mgr.GetScheme(), + Recorder: mgr.GetEventRecorderFor("account-controller"), + SQLExecutor: core.NewPodSQLExecutor(mgr.GetClient(), clientset, mgr.GetConfig()), + Adapter: adapter, // Replication: nil -> single-region + }).SetupWithManager(mgr, "account") + (&core.CertificateReconciler[*v1.Account]{Client: mgr.GetClient(), Scheme: mgr.GetScheme(), Adapter: adapter}).SetupWithManager(mgr, "account-certificate") + (&core.PersistentVolumeReconciler[*v1.Account]{Client: mgr.GetClient(), Adapter: adapter}).SetupWithManager(mgr, "account-pv") + ``` + + It writes its own webhook on top of the `core.Validate*` helpers with its + own `core.ValidationTerms`, and its own RBAC markers for its API group. 3. Ships the shared sidecar-injector with its gateway image, setting `ProductProfile.Gateway` when its gateway contract differs. +`pkg/core/engine_test.go` defines a fake `Account` CR in another API group and +drives `core.ClusterReconciler` with a fake client through finalizer, RBAC, +CNPG `Cluster` creation with the second product's extension, and status, +using only `pkg/core`. `pkg/core/example_test.go` renders a non-DocumentDB cluster through the facade. `internal/cnpg/second_product_test.go` renders a synthetic `pgcosmos` product end to end and fails if any DocumentDB string other than the documented @@ -127,12 +182,11 @@ end to end and fails if any DocumentDB string other than the documented ## Not in scope / follow-ups -- `DocumentDBReconciler`, the webhook, backup CRDs, and the kubectl plugin stay - DocumentDB-specific. A second product writes its own reconciler on top of - `pkg/core`. -- The client `Service` port comes from the `GATEWAY_PORT` environment variable - (default `10260`), not from `GatewayProfile.Port`. -- The OTel collector ConfigMap labels metrics with `documentdb.cluster`. - Making the attribute name a profile field is a follow-up. +- CRD types, `cmd/main.go` wiring, Helm charts, RBAC markers, and the kubectl + plugin stay per product. +- DocumentDB multi-region replication (fleet networking, promotion, and + publication/subscription) stays DocumentDB-specific behind + `ReplicationHooks`. Another product can implement its own hooks or run + single-region. - `MonitoringSpec` and `CertificatesConfiguration` are reused from the DocumentDB API and CNPG respectively as shared types. From 5675078175f168802dd546dfe27b326474f15a0a Mon Sep 17 00:00:00 2001 From: Rayhan Hossain Date: Tue, 20 Oct 2026 11:37:44 -0700 Subject: [PATCH 21/27] fix(core): address cross-product integration review findings Expose replication state construction for external product hooks, reconcile gateway contract parameters and owned Service ports, and configure the PostgreSQL health metric through the product naming profile. Preserve DocumentDB defaults and add external-product regression coverage. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 33b3cb3c-1b25-4ffb-bff1-ebf0f3d04058 Signed-off-by: Rayhan Hossain --- CHANGELOG.md | 2 +- docs/designs/product-neutral-core.md | 63 +++- operator/src/internal/cnpg/cnpg_cluster.go | 21 +- operator/src/internal/cnpg/cnpg_sync.go | 13 +- .../internal/cnpg/gateway_contract_test.go | 96 ++++++ .../src/internal/cnpg/second_product_test.go | 29 ++ operator/src/internal/otel/base_config.yaml | 2 +- operator/src/internal/otel/config.go | 40 ++- operator/src/internal/otel/config_test.go | 17 +- operator/src/internal/otel/options_test.go | 45 +++ operator/src/internal/product/cluster.go | 3 + operator/src/internal/product/documentdb.go | 1 + operator/src/internal/utils/constants.go | 7 + .../src/internal/utils/replication_context.go | 70 +++-- operator/src/internal/utils/util.go | 37 +++ operator/src/internal/utils/util_test.go | 59 ++++ operator/src/pkg/core/engine.go | 27 +- operator/src/pkg/core/engine_test.go | 49 +++ operator/src/pkg/core/example_test.go | 15 + operator/src/pkg/core/replication_test.go | 297 ++++++++++++++++++ 20 files changed, 831 insertions(+), 62 deletions(-) create mode 100644 operator/src/pkg/core/replication_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 785f2037..e053dea2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,7 @@ - Require Go 1.26.10 across all six modules and pin Go builder images to that patch release to address [GO-2026-6809](https://pkg.go.dev/vuln/GO-2026-6809) in `crypto/x509`. Unit-test workflows now resolve Go from the operator module instead of an older hard-coded version. ### Major Features -- **Product-neutral operator core**: The CNPG `Cluster` renderer, sync, PostgreSQL parameter merge, resource split/validation and extension lifecycle SQL are now driven by a `ProductProfile` and `ClusterIntent` instead of the `DocumentDB` custom resource, and are exported for other operator modules through `operator/src/pkg/core`. The sidecar injector accepts optional `gatewayContainerName`, `gatewayPort`, `gatewayArgs`, `gatewayUsernameEnv` and `gatewayPasswordEnv` parameters for non-DocumentDB gateways. The cluster, certificate, persistent volume, backup and scheduled-backup reconcilers and the admission validation rules are generic over the custom resource type through a `ClusterAdapter`, with DocumentDB as the first implementation; `pkg/core` exports them too. Rendered DocumentDB clusters are byte-identical to before, CRDs and RBAC are unchanged, and existing deployments see no change. See [Product-neutral operator core](docs/designs/product-neutral-core.md). +- **Product-neutral operator core**: The CNPG `Cluster` renderer, sync, PostgreSQL parameter merge, resource split/validation and extension lifecycle SQL are now driven by a `ProductProfile` and `ClusterIntent` instead of the `DocumentDB` custom resource, and are exported for other operator modules through `operator/src/pkg/core`. The sidecar injector accepts optional `gatewayContainerName`, `gatewayPort`, `gatewayArgs`, `gatewayUsernameEnv` and `gatewayPasswordEnv` parameters for non-DocumentDB gateways. The operator keeps these parameters in sync on existing clusters, and the client Service port follows the gateway port. Other products can build multi-region topologies with `core.NewReplicationContext` and name the OTel SQL health metric with `NamingProfile.HealthMetricName` (DocumentDB keeps `documentdb.postgres.up`). The cluster, certificate, persistent volume, backup and scheduled-backup reconcilers and the admission validation rules are generic over the custom resource type through a `ClusterAdapter`, with DocumentDB as the first implementation; `pkg/core` exports them too. Rendered DocumentDB clusters are byte-identical to before, CRDs and RBAC are unchanged, and existing deployments see no change. See [Product-neutral operator core](docs/designs/product-neutral-core.md). ## [1.0.0] - 2026-10-19 diff --git a/docs/designs/product-neutral-core.md b/docs/designs/product-neutral-core.md index 594849c2..3df0a816 100644 --- a/docs/designs/product-neutral-core.md +++ b/docs/designs/product-neutral-core.md @@ -65,7 +65,13 @@ unmodified. `CertificateReconciler[T]`, `PersistentVolumeReconciler[T]`, `BackupReconciler[T,B]`, and `ScheduledBackupReconciler[T,S,B]`. Multi-region replication plugs in through `ReplicationHooks[T]`; when it is - nil the cluster runs single-region. + nil the cluster runs single-region. Its `Context` method returns a + `ReplicationContext` built with `NewReplicationContext(state, + currentLocalPrimary, targetLocalPrimary)`. The state is `NoReplication`, + `Primary`, `Replica`, or `NotPresent`; the cluster, Service, and backup + reconcilers read it to set the replica source, skip the cluster or backups, + and hold the client endpoint until a pending promotion finishes (current and + target local primaries differ). - **`internal/validation`** holds the admission rules (schema version vs. binary, image rollback, storage resize, restore compatibility, ImageVolume probe). They take `Terms` so error messages use the product's field names. @@ -96,6 +102,7 @@ DocumentDB wrappers. | Client Service port | `GatewayProfile.Port`, else `GATEWAY_PORT` | `10260` | | Service name prefix, finalizer, labels | `NamingProfile` | `documentdb-service-`, `documentdb.io/...` | | OTel metric attribute | `NamingProfile` | `documentdb.cluster` | +| OTel SQL health metric | `NamingProfile.HealthMetricName` | `documentdb.postgres.up` | | Admission error wording | `validation.Terms` | `documentDBVersion`, `spec.documentDBImage`, ... | ## Sidecar-injector gateway contract @@ -105,9 +112,13 @@ The injector accepts five optional plugin parameters: `gatewayContainerName`, verbatim), `gatewayUsernameEnv`, and `gatewayPasswordEnv`. They default to the DocumentDB gateway (`documentdb-gateway`, `10260`, `--create-user ... --start-pg false --pg-port 5432`, `USERNAME`, `PASSWORD`). The operator sends -them only when `ProductProfile.Gateway` sets them, and the injector serializes -them only when they differ from the defaults, so existing clusters see no -parameter change and no rolling restart. See +them only when `ProductProfile.Gateway` sets them to a non-default value (an +empty `Args` slice is sent as `[]`), matching how the injector serializes +them, so existing clusters see no parameter change and no rolling restart. +`SyncCnpgCluster` adds, updates, and removes these parameters on existing +clusters and requests a rolling restart when they change. When the gateway +port changes, the client Service's ports follow it; the Service keeps its +assigned cluster IPs and node ports. See [the plugin configuration guide](../developer-guides/sidecar-injector-plugin-configuration.md). ## Wire contracts kept on purpose @@ -130,7 +141,9 @@ profile types plus `RenderCluster`, `SyncCluster`, `MergeParameters`, `ProtectedParameters`, `ComputeResourceSplit`, `ValidateResources`, `DefaultSplitConfig`, the extension SQL helpers, and `DocumentDBProfile` as a reference profile. `pkg/core/engine.go` re-exports the reconcilers, adapters, -`ReplicationHooks`, `NewPodSQLExecutor`, predicates, and the validation +`ReplicationHooks`, `NewReplicationContext` with its `ReplicationState` +values (`NoReplication`, `ReplicationPrimary`, `ReplicationReplica`, +`ReplicationNotPresent`), `NewPodSQLExecutor`, predicates, and the validation helpers (`ValidateSchemaVersion`, `ValidateImageRollback`, `ValidateStorageResize`, `ValidateRestoreCompatibility`, `NewImageVolumeProbe`). It adds no behavior of its own. @@ -146,25 +159,51 @@ A second product: ```go adapter := cosmos.AccountAdapter{} - (&core.ClusterReconciler[*v1.Account]{ + if err := (&core.ClusterReconciler[*v1.Account]{ Client: mgr.GetClient(), Scheme: mgr.GetScheme(), Recorder: mgr.GetEventRecorderFor("account-controller"), SQLExecutor: core.NewPodSQLExecutor(mgr.GetClient(), clientset, mgr.GetConfig()), - Adapter: adapter, // Replication: nil -> single-region - }).SetupWithManager(mgr, "account") - (&core.CertificateReconciler[*v1.Account]{Client: mgr.GetClient(), Scheme: mgr.GetScheme(), Adapter: adapter}).SetupWithManager(mgr, "account-certificate") - (&core.PersistentVolumeReconciler[*v1.Account]{Client: mgr.GetClient(), Adapter: adapter}).SetupWithManager(mgr, "account-pv") + Adapter: adapter, + Replication: replication, // nil -> single-region + }).SetupWithManager(mgr, "account"); err != nil { + return err + } + if err := (&core.CertificateReconciler[*v1.Account]{Client: mgr.GetClient(), Scheme: mgr.GetScheme(), Adapter: adapter}).SetupWithManager(mgr, "account-certificate"); err != nil { + return err + } + if err := (&core.PersistentVolumeReconciler[*v1.Account]{Client: mgr.GetClient(), Adapter: adapter}).SetupWithManager(mgr, "account-pv"); err != nil { + return err + } + ``` + + A multi-region product implements `core.ReplicationHooks[*v1.Account]` and + returns its topology from `Context`: + + ```go + func (accountReplication) Context(ctx context.Context, a *v1.Account) (*core.ReplicationContext, error) { + if a.Spec.Region != a.Spec.PrimaryRegion { + rc := core.NewReplicationContext(core.ReplicationReplica, "", "") + rc.PrimaryCNPGClusterName = a.Spec.PrimaryRegion + return rc, nil + } + // Hold the endpoint while a promotion to this region is pending. + return core.NewReplicationContext(core.ReplicationPrimary, a.Status.LocalPrimary, a.Status.TargetPrimary), nil + } ``` It writes its own webhook on top of the `core.Validate*` helpers with its own `core.ValidationTerms`, and its own RBAC markers for its API group. 3. Ships the shared sidecar-injector with its gateway image, setting - `ProductProfile.Gateway` when its gateway contract differs. + `ProductProfile.Gateway` when its gateway contract differs, and + `NamingProfile.HealthMetricName` and `MetricClusterAttribute` for its + monitoring names. `pkg/core/engine_test.go` defines a fake `Account` CR in another API group and drives `core.ClusterReconciler` with a fake client through finalizer, RBAC, CNPG `Cluster` creation with the second product's extension, and status, -using only `pkg/core`. +using only `pkg/core`. `pkg/core/replication_test.go` drives the cluster and +backup reconcilers with replica, not-present, and promotion-pending contexts +built with `core.NewReplicationContext`. `pkg/core/example_test.go` renders a non-DocumentDB cluster through the facade. `internal/cnpg/second_product_test.go` renders a synthetic `pgcosmos` product end to end and fails if any DocumentDB string other than the documented diff --git a/operator/src/internal/cnpg/cnpg_cluster.go b/operator/src/internal/cnpg/cnpg_cluster.go index 1ab7194d..7cb148f2 100644 --- a/operator/src/internal/cnpg/cnpg_cluster.go +++ b/operator/src/internal/cnpg/cnpg_cluster.go @@ -421,11 +421,15 @@ func buildPostgresConfiguration(intent product.ClusterIntent, extensionImageSour } // addGatewayContractParams forwards a product's non-default gateway contract to -// the sidecar injector. Unset fields are omitted so the injector keeps its -// DocumentDB gateway defaults and existing parameter maps are unchanged. +// the sidecar injector. Unset fields and values equal to the injector's +// (DocumentDB gateway) defaults are omitted, matching the injector's own +// normalization so existing parameter maps are unchanged and SyncCnpgCluster +// does not fight the injector's defaulting webhook. func addGatewayContractParams(params map[string]string, gw product.GatewayProfile) { - addPluginParamIfSet(params, util.PLUGIN_PARAM_GATEWAY_CONTAINER_NAME, gw.ContainerName) - if gw.Port > 0 { + if gw.ContainerName != util.PLUGIN_DEFAULT_GATEWAY_CONTAINER_NAME { + addPluginParamIfSet(params, util.PLUGIN_PARAM_GATEWAY_CONTAINER_NAME, gw.ContainerName) + } + if gw.Port > 0 && gw.Port != util.PLUGIN_DEFAULT_GATEWAY_PORT { params[util.PLUGIN_PARAM_GATEWAY_PORT] = strconv.FormatInt(int64(gw.Port), 10) } if gw.Args != nil { @@ -433,8 +437,12 @@ func addGatewayContractParams(params map[string]string, gw product.GatewayProfil args, _ := json.Marshal(gw.Args) params[util.PLUGIN_PARAM_GATEWAY_ARGS] = string(args) } - addPluginParamIfSet(params, util.PLUGIN_PARAM_GATEWAY_USERNAME_ENV, gw.UsernameEnv) - addPluginParamIfSet(params, util.PLUGIN_PARAM_GATEWAY_PASSWORD_ENV, gw.PasswordEnv) + if gw.UsernameEnv != util.PLUGIN_DEFAULT_GATEWAY_USERNAME_ENV { + addPluginParamIfSet(params, util.PLUGIN_PARAM_GATEWAY_USERNAME_ENV, gw.UsernameEnv) + } + if gw.PasswordEnv != util.PLUGIN_DEFAULT_GATEWAY_PASSWORD_ENV { + addPluginParamIfSet(params, util.PLUGIN_PARAM_GATEWAY_PASSWORD_ENV, gw.PasswordEnv) + } } // OTelOptions maps the product naming onto the OTel collector config options. @@ -442,5 +450,6 @@ func OTelOptions(p product.ProductProfile) otelcfg.Options { return otelcfg.Options{ ClusterAttribute: p.Naming.MetricClusterAttribute, Generator: p.Naming.OperatorName, + HealthMetricName: p.Naming.HealthMetricName, } } diff --git a/operator/src/internal/cnpg/cnpg_sync.go b/operator/src/internal/cnpg/cnpg_sync.go index 3bd36865..f9b8ab85 100644 --- a/operator/src/internal/cnpg/cnpg_sync.go +++ b/operator/src/internal/cnpg/cnpg_sync.go @@ -24,8 +24,10 @@ import ( // for ALL CNPG spec mutations (images + plugin params + replication). // // Mutable plugin parameters synced: gatewayImage, gatewayTLSSecret, sidecar -// resource params, and OTel sidecar params (otelCollectorImage, -// otelConfigMapName, prometheusPort, otelConfigHash). +// resource params, OTel sidecar params (otelCollectorImage, +// otelConfigMapName, prometheusPort, otelConfigHash), and the gateway contract +// (gatewayContainerName, gatewayPort, gatewayArgs, gatewayUsernameEnv, +// gatewayPasswordEnv). // Other parameters (e.g., documentDbCredentialSecret) are set at cluster creation // and do not change during the lifecycle of a DocumentDB resource. // @@ -122,6 +124,13 @@ func SyncCnpgCluster( util.PLUGIN_PARAM_OTEL_MEMORY_LIMIT, util.PLUGIN_PARAM_OTEL_CPU_REQUEST, util.PLUGIN_PARAM_OTEL_CPU_LIMIT, + // Gateway contract: the injector renders the gateway + // container from these at pod creation. + util.PLUGIN_PARAM_GATEWAY_CONTAINER_NAME, + util.PLUGIN_PARAM_GATEWAY_PORT, + util.PLUGIN_PARAM_GATEWAY_ARGS, + util.PLUGIN_PARAM_GATEWAY_USERNAME_ENV, + util.PLUGIN_PARAM_GATEWAY_PASSWORD_ENV, } for _, key := range sidecarParamKeys { desiredVal := getParam(desiredPlugin.Parameters, key) diff --git a/operator/src/internal/cnpg/gateway_contract_test.go b/operator/src/internal/cnpg/gateway_contract_test.go index 201f7f14..4cdb3f7e 100644 --- a/operator/src/internal/cnpg/gateway_contract_test.go +++ b/operator/src/internal/cnpg/gateway_contract_test.go @@ -4,10 +4,16 @@ package cnpg import ( + "context" + "reflect" "testing" + cnpgv1 "github.com/cloudnative-pg/cloudnative-pg/api/v1" "github.com/go-logr/logr" + "k8s.io/apimachinery/pkg/runtime" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" "github.com/documentdb/documentdb-operator/internal/product" util "github.com/documentdb/documentdb-operator/internal/utils" @@ -53,3 +59,93 @@ func TestGatewayContractPluginParams(t *testing.T) { } } } + +// TestGatewayContractSync drives a live cluster's gateway contract through +// add, change (including explicit empty args), and a return to the defaults. +func TestGatewayContractSync(t *testing.T) { + ctx := context.Background() + scheme := runtime.NewScheme() + if err := cnpgv1.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + req := ctrl.Request{} + req.Name, req.Namespace = "golden", "golden-ns" + intent := product.DocumentDBAdapter{}.ToClusterIntent(goldenBaseDocumentDB()) + render := func(gw product.GatewayProfile) *cnpgv1.Cluster { + in := intent + in.Product.Gateway = gw + return GetCnpgClusterSpecFromIntent(req, in, "sa", "", true, logr.Discard()) + } + + // Explicit sidecar-injector defaults render exactly like the DocumentDB + // profile, as the injector's defaulting webhook would normalize them. + explicitDefaults := render(product.GatewayProfile{ + ContainerName: "documentdb-gateway", Port: 10260, UsernameEnv: "USERNAME", PasswordEnv: "PASSWORD", + }) + if !reflect.DeepEqual(explicitDefaults.Spec.Plugins, render(product.GatewayProfile{}).Spec.Plugins) { + t.Fatalf("explicit defaults rendered %v", explicitDefaults.Spec.Plugins[0].Parameters) + } + + c := fake.NewClientBuilder().WithScheme(scheme).WithObjects(render(product.GatewayProfile{})).Build() + key := client.ObjectKey{Name: "golden", Namespace: "golden-ns"} + sync := func(desired *cnpgv1.Cluster) (*cnpgv1.Cluster, *cnpgv1.Cluster) { + t.Helper() + before := &cnpgv1.Cluster{} + if err := c.Get(ctx, key, before); err != nil { + t.Fatal(err) + } + if err := SyncCnpgCluster(ctx, c, before.DeepCopy(), desired, nil); err != nil { + t.Fatal(err) + } + after := &cnpgv1.Cluster{} + if err := c.Get(ctx, key, after); err != nil { + t.Fatal(err) + } + for _, k := range []string{ + util.PLUGIN_PARAM_GATEWAY_CONTAINER_NAME, util.PLUGIN_PARAM_GATEWAY_PORT, util.PLUGIN_PARAM_GATEWAY_ARGS, + util.PLUGIN_PARAM_GATEWAY_USERNAME_ENV, util.PLUGIN_PARAM_GATEWAY_PASSWORD_ENV, + } { + got, gotOK := after.Spec.Plugins[0].Parameters[k] + want, wantOK := desired.Spec.Plugins[0].Parameters[k] + if got != want || gotOK != wantOK { + t.Errorf("%s = %q (set %t), want %q (set %t)", k, got, gotOK, want, wantOK) + } + } + return before, after + } + restarted := func(before, after *cnpgv1.Cluster) bool { + const a = "kubectl.kubernetes.io/restartedAt" + return after.Annotations[a] != "" && after.Annotations[a] != before.Annotations[a] + } + + custom := render(product.GatewayProfile{ + ContainerName: "pgcosmos-gateway", Port: 8081, Args: []string{"--port", "8081"}, + UsernameEnv: "COSMOS_USER", PasswordEnv: "COSMOS_KEY", + }) + if before, after := sync(custom); !restarted(before, after) { + t.Error("adding the gateway contract did not request a rolling restart") + } + + changed := render(product.GatewayProfile{ + ContainerName: "pgcosmos-gateway-v2", Port: 8082, Args: []string{}, + UsernameEnv: "COSMOS_USER2", PasswordEnv: "COSMOS_KEY2", + }) + if got := changed.Spec.Plugins[0].Parameters[util.PLUGIN_PARAM_GATEWAY_ARGS]; got != "[]" { + t.Fatalf("explicit empty args rendered %q, want []", got) + } + if before, after := sync(changed); !restarted(before, after) { + t.Error("changing the gateway contract did not request a rolling restart") + } + + // Idempotent: re-syncing the same contract does not patch. + if before, after := sync(changed); after.ResourceVersion != before.ResourceVersion { + t.Error("re-syncing an unchanged gateway contract patched the cluster") + } + + if before, after := sync(explicitDefaults); !restarted(before, after) { + t.Error("returning to the default gateway contract did not request a rolling restart") + } + if before, after := sync(render(product.GatewayProfile{})); after.ResourceVersion != before.ResourceVersion { + t.Error("re-syncing the default gateway contract patched the cluster") + } +} diff --git a/operator/src/internal/cnpg/second_product_test.go b/operator/src/internal/cnpg/second_product_test.go index a5df8d4f..44c7d4bc 100644 --- a/operator/src/internal/cnpg/second_product_test.go +++ b/operator/src/internal/cnpg/second_product_test.go @@ -7,6 +7,7 @@ import ( "encoding/json" "regexp" "slices" + "strings" "testing" "github.com/go-logr/logr" @@ -14,6 +15,7 @@ import ( ctrl "sigs.k8s.io/controller-runtime" dbpreview "github.com/documentdb/documentdb-operator/api/preview" + otelcfg "github.com/documentdb/documentdb-operator/internal/otel" "github.com/documentdb/documentdb-operator/internal/product" util "github.com/documentdb/documentdb-operator/internal/utils" ) @@ -132,3 +134,30 @@ func TestSecondProductResourceValidationUsesProductPath(t *testing.T) { } } } + +// TestSecondProductOtelConfigHash checks that the rendered config hash is the +// hash of the collector config generated with the product's metric naming. +func TestSecondProductOtelConfigHash(t *testing.T) { + req := ctrl.Request{} + req.Name, req.Namespace = "acct", "ns" + intent := pgcosmosIntent() + intent.Product.Naming = product.NamingProfile{ + MetricClusterAttribute: "cosmos.cluster", OperatorName: "cosmos-operator", HealthMetricName: "cosmos.postgres.up", + } + data, err := otelcfg.GenerateConfigMapDataWithOptions(OTelOptions(intent.Product), "acct", "ns", intent.Monitoring) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(data["static.yaml"], "metric_name: cosmos.postgres.up\n") { + t.Fatalf("static.yaml:\n%s", data["static.yaml"]) + } + got := GetCnpgClusterSpecFromIntent(req, intent, "acct-sa", "", true, logr.Discard()).Spec.Plugins[0].Parameters[util.PLUGIN_PARAM_OTEL_CONFIG_HASH] + if want := otelcfg.HashConfigMapData(data); got != want { + t.Fatalf("otelConfigHash = %q, want %q", got, want) + } + + intent.Product.Naming.HealthMetricName = "" + if def := GetCnpgClusterSpecFromIntent(req, intent, "acct-sa", "", true, logr.Discard()).Spec.Plugins[0].Parameters[util.PLUGIN_PARAM_OTEL_CONFIG_HASH]; def == got { + t.Fatal("the health metric name does not affect the config hash") + } +} diff --git a/operator/src/internal/otel/base_config.yaml b/operator/src/internal/otel/base_config.yaml index 1ef124ba..54679125 100644 --- a/operator/src/internal/otel/base_config.yaml +++ b/operator/src/internal/otel/base_config.yaml @@ -12,7 +12,7 @@ receivers: queries: - sql: "SELECT 1 as up" metrics: - - metric_name: documentdb.postgres.up + - metric_name: {{ .HealthMetricName }} value_column: up data_type: gauge diff --git a/operator/src/internal/otel/config.go b/operator/src/internal/otel/config.go index 3b5def04..ee9efb72 100644 --- a/operator/src/internal/otel/config.go +++ b/operator/src/internal/otel/config.go @@ -4,10 +4,12 @@ package otel import ( + "bytes" "crypto/sha256" _ "embed" "fmt" "sort" + "text/template" "gopkg.in/yaml.v3" @@ -17,6 +19,25 @@ import ( //go:embed base_config.yaml var baseConfigYAML []byte +// baseConfigTemplate renders static.yaml: base_config.yaml is a text/template +// over staticConfigData, so product-specific metric names stay structured. +var baseConfigTemplate = template.Must(template.New("base_config.yaml").Option("missingkey=error").Parse(string(baseConfigYAML))) + +// staticConfigData holds the product-specific names base_config.yaml uses. +type staticConfigData struct { + // HealthMetricName is the PostgreSQL liveness gauge. + HealthMetricName string +} + +// renderStaticConfig renders static.yaml from base_config.yaml. +func renderStaticConfig(data staticConfigData) (string, error) { + var out bytes.Buffer + if err := baseConfigTemplate.Execute(&out, data); err != nil { + return "", fmt.Errorf("failed to render static OTel config: %w", err) + } + return out.String(), nil +} + const defaultPrometheusPort = 8888 // MonitorRoleName is the dedicated PostgreSQL identity the OTel Collector @@ -58,8 +79,9 @@ func ConfigMapName(clusterName string) string { // GenerateConfigMapData returns the ConfigMap data entries for the OTel Collector. // The config is split into two files that the collector merges at startup: -// - static.yaml: receivers and static processors from the embedded base_config.yaml. -// Add new SQL metric queries to base_config.yaml — no Go code changes needed. +// - static.yaml: receivers and static processors rendered from the embedded +// base_config.yaml template. Add new SQL metric queries to base_config.yaml — +// no Go code changes needed unless a query needs a new product-specific name. // - dynamic.yaml: per-cluster resource attributes, exporters, and pipeline wiring. // // This function is only called when monitoring is enabled. When monitoring is @@ -77,11 +99,16 @@ type Options struct { ClusterAttribute string // Generator is the operator name written in the generated-file header. Generator string + // HealthMetricName is the gauge the sqlquery receiver reports PostgreSQL + // liveness as. + HealthMetricName string } const ( defaultClusterAttribute = "documentdb.cluster" defaultGenerator = "documentdb-operator" + // defaultHealthMetricName is the DocumentDB PostgreSQL liveness gauge. + defaultHealthMetricName = "documentdb.postgres.up" ) // GenerateConfigMapDataWithOptions is GenerateConfigMapData with product naming. @@ -92,13 +119,20 @@ func GenerateConfigMapDataWithOptions(opts Options, clusterName, namespace strin if opts.Generator == "" { opts.Generator = defaultGenerator } + if opts.HealthMetricName == "" { + opts.HealthMetricName = defaultHealthMetricName + } + staticYAML, err := renderStaticConfig(staticConfigData{HealthMetricName: opts.HealthMetricName}) + if err != nil { + return nil, err + } dynamicYAML, err := generateDynamicConfig(opts.ClusterAttribute, clusterName, namespace, spec) if err != nil { return nil, err } return map[string]string{ - "static.yaml": string(baseConfigYAML), + "static.yaml": staticYAML, "dynamic.yaml": "# Auto-generated by " + opts.Generator + ". Do not edit.\n" + dynamicYAML, }, nil } diff --git a/operator/src/internal/otel/config_test.go b/operator/src/internal/otel/config_test.go index 0032c331..9c8d9ce9 100644 --- a/operator/src/internal/otel/config_test.go +++ b/operator/src/internal/otel/config_test.go @@ -38,10 +38,17 @@ func parseCfg(yamlStr string) collectorConfig { return cfg } +// defaultStaticYAML is base_config.yaml rendered with the DocumentDB names. +func defaultStaticYAML() []byte { + out, err := renderStaticConfig(staticConfigData{HealthMetricName: defaultHealthMetricName}) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + return []byte(out) +} + var _ = Describe("base_config.yaml embed", func() { It("can be parsed as valid YAML with expected static components", func() { var cfg collectorConfig - Expect(yaml.Unmarshal(baseConfigYAML, &cfg)).To(Succeed()) + Expect(yaml.Unmarshal(defaultStaticYAML(), &cfg)).To(Succeed()) Expect(cfg.Receivers).To(HaveKey("sqlquery")) Expect(cfg.Receivers).To(HaveKey("otlp")) Expect(cfg.Processors).To(HaveKey("memory_limiter")) @@ -51,13 +58,13 @@ var _ = Describe("base_config.yaml embed", func() { }) It("does not configure a password while PostgreSQL uses trust authentication", func() { - Expect(string(baseConfigYAML)).To(ContainSubstring("user=${env:PGUSER}")) - Expect(string(baseConfigYAML)).NotTo(ContainSubstring("PGPASSWORD")) + Expect(string(defaultStaticYAML())).To(ContainSubstring("user=${env:PGUSER}")) + Expect(string(defaultStaticYAML())).NotTo(ContainSubstring("PGPASSWORD")) }) It("declares a cgroup-aware memory_limiter processor", func() { var cfg collectorConfig - Expect(yaml.Unmarshal(baseConfigYAML, &cfg)).To(Succeed()) + Expect(yaml.Unmarshal(defaultStaticYAML(), &cfg)).To(Succeed()) memLimiter, ok := cfg.Processors["memory_limiter"].(map[string]any) Expect(ok).To(BeTrue(), "memory_limiter processor must be a map") @@ -68,7 +75,7 @@ var _ = Describe("base_config.yaml embed", func() { It("declares an OTLP gRPC receiver on port 4317", func() { var cfg collectorConfig - Expect(yaml.Unmarshal(baseConfigYAML, &cfg)).To(Succeed()) + Expect(yaml.Unmarshal(defaultStaticYAML(), &cfg)).To(Succeed()) otlp, ok := cfg.Receivers["otlp"].(map[string]any) Expect(ok).To(BeTrue(), "otlp receiver must be a map") diff --git a/operator/src/internal/otel/options_test.go b/operator/src/internal/otel/options_test.go index a51f5c04..61f3294e 100644 --- a/operator/src/internal/otel/options_test.go +++ b/operator/src/internal/otel/options_test.go @@ -8,6 +8,8 @@ import ( "strings" "testing" + "gopkg.in/yaml.v3" + dbpreview "github.com/documentdb/documentdb-operator/api/preview" ) @@ -36,3 +38,46 @@ func TestGenerateConfigMapDataWithOptions(t *testing.T) { t.Fatalf("custom naming not applied:\n%s", dyn) } } + +func TestGenerateConfigMapDataHealthMetricName(t *testing.T) { + spec := &dbpreview.MonitoringSpec{Enabled: true} + + def, err := GenerateConfigMapData("c", "ns", spec) + if err != nil { + t.Fatal(err) + } + explicit, err := GenerateConfigMapDataWithOptions(Options{HealthMetricName: "documentdb.postgres.up"}, "c", "ns", spec) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(def, explicit) { + t.Fatal("the explicit DocumentDB health metric must produce the default config") + } + if static := def["static.yaml"]; !strings.Contains(static, "metric_name: documentdb.postgres.up\n") || strings.Contains(static, "{{") { + t.Fatalf("default static config:\n%s", static) + } + + custom, err := GenerateConfigMapDataWithOptions(Options{ + ClusterAttribute: "cosmos.cluster", Generator: "cosmos-operator", HealthMetricName: "cosmos.postgres.up", + }, "c", "ns", spec) + if err != nil { + t.Fatal(err) + } + for key, data := range custom { + if strings.Contains(data, "documentdb.") { + t.Errorf("%s still uses DocumentDB metric naming:\n%s", key, data) + } + } + var static collectorConfig + if err := yaml.Unmarshal([]byte(custom["static.yaml"]), &static); err != nil { + t.Fatal(err) + } + queries := static.Receivers["sqlquery"].(map[string]any)["queries"].([]any) + metric := queries[0].(map[string]any)["metrics"].([]any)[0].(map[string]any) + if metric["metric_name"] != "cosmos.postgres.up" { + t.Fatalf("health metric = %v, want cosmos.postgres.up", metric["metric_name"]) + } + if HashConfigMapData(custom) == HashConfigMapData(def) { + t.Fatal("the config hash must change with the health metric name") + } +} diff --git a/operator/src/internal/product/cluster.go b/operator/src/internal/product/cluster.go index 9d23cd72..f061b924 100644 --- a/operator/src/internal/product/cluster.go +++ b/operator/src/internal/product/cluster.go @@ -22,6 +22,9 @@ type NamingProfile struct { // MetricClusterAttribute is the OTel resource attribute key carrying the // cluster name. MetricClusterAttribute string + // HealthMetricName is the OTel gauge the monitoring sidecar reports + // PostgreSQL liveness as; empty means "documentdb.postgres.up". + HealthMetricName string // OperatorName identifies the operator in generated artifacts. OperatorName string // Kind is the custom resource kind; it identifies the product owner diff --git a/operator/src/internal/product/documentdb.go b/operator/src/internal/product/documentdb.go index dbccbf8a..2f77e72f 100644 --- a/operator/src/internal/product/documentdb.go +++ b/operator/src/internal/product/documentdb.go @@ -48,6 +48,7 @@ func DocumentDBProfile() ProductProfile { Domain: "documentdb.io", ServicePrefix: util.DOCUMENTDB_SERVICE_PREFIX, MetricClusterAttribute: "documentdb.cluster", + HealthMetricName: "documentdb.postgres.up", OperatorName: "documentdb-operator", Kind: "DocumentDB", }, diff --git a/operator/src/internal/utils/constants.go b/operator/src/internal/utils/constants.go index 0b59e39a..a1026caa 100644 --- a/operator/src/internal/utils/constants.go +++ b/operator/src/internal/utils/constants.go @@ -130,6 +130,13 @@ const ( PLUGIN_PARAM_GATEWAY_ARGS = "gatewayArgs" PLUGIN_PARAM_GATEWAY_USERNAME_ENV = "gatewayUsernameEnv" PLUGIN_PARAM_GATEWAY_PASSWORD_ENV = "gatewayPasswordEnv" + // The sidecar injector's gateway contract defaults; keep in sync with + // cnpg-plugins/sidecar-injector/internal/config. Its defaulting webhook + // drops parameters equal to these, so the operator omits them too. + PLUGIN_DEFAULT_GATEWAY_CONTAINER_NAME = "documentdb-gateway" + PLUGIN_DEFAULT_GATEWAY_PORT = int32(10260) + PLUGIN_DEFAULT_GATEWAY_USERNAME_ENV = "USERNAME" + PLUGIN_DEFAULT_GATEWAY_PASSWORD_ENV = "PASSWORD" // TODO: remove these constants once change stream support is included in the official images. CHANGESTREAM_DOCUMENTDB_IMAGE_REPOSITORY = "ghcr.io/wentingwu666666/documentdb-kubernetes-operator" diff --git a/operator/src/internal/utils/replication_context.go b/operator/src/internal/utils/replication_context.go index d750479b..bb7b646b 100644 --- a/operator/src/internal/utils/replication_context.go +++ b/operator/src/internal/utils/replication_context.go @@ -25,7 +25,7 @@ type ReplicationContext struct { OtherFleetMemberNames []string currentLocalPrimary string targetLocalPrimary string - state replicationState + state ReplicationState } type crossCloudNetworkingStrategy string @@ -36,26 +36,43 @@ const ( Istio crossCloudNetworkingStrategy = "Istio" ) -type replicationState int32 +// ReplicationState is the role of a cluster in its replication topology. +type ReplicationState int32 const ( - NoReplication replicationState = iota + // NoReplication is a single-region cluster: primary, endpoint always enabled. + NoReplication ReplicationState = iota + // Primary is the primary cluster of a replication topology. Primary + // Replica is a cluster replicating from PrimaryCNPGClusterName. Replica + // NotPresent is a cluster that is not a member of its topology; the + // reconciler deletes its resources. NotPresent ) +// NewReplicationContext returns a replication context in state. The endpoint +// of a replicating cluster is enabled only once currentLocalPrimary equals +// targetLocalPrimary (that is, a pending promotion has completed); it is always +// enabled for NoReplication. The caller sets the exported topology fields. +func NewReplicationContext(state ReplicationState, currentLocalPrimary, targetLocalPrimary string) *ReplicationContext { + return &ReplicationContext{ + state: state, + CrossCloudNetworkingStrategy: None, + currentLocalPrimary: currentLocalPrimary, + targetLocalPrimary: targetLocalPrimary, + } +} + // NewSingleRegionContext returns the replication context of a cluster that is // not part of a multi-region topology: it is primary, serves its endpoint, and // its CNPG cluster is named after the custom resource. func NewSingleRegionContext(name, environment, storageClass string) *ReplicationContext { - return &ReplicationContext{ - state: NoReplication, - CrossCloudNetworkingStrategy: None, - Environment: environment, - StorageClass: storageClass, - CNPGClusterName: name, - } + rc := NewReplicationContext(NoReplication, "", "") + rc.Environment = environment + rc.StorageClass = storageClass + rc.CNPGClusterName = name + return rc } func GetReplicationContext(ctx context.Context, client client.Client, documentdb dbpreview.DocumentDB) (*ReplicationContext, error) { @@ -73,13 +90,7 @@ func GetReplicationContext(ctx context.Context, client client.Client, documentdb // This edge case can happen when the Hub cluster is also a member, and we are not // putting the documentdb instance on it if self == nil { - return &ReplicationContext{ - state: NotPresent, - CrossCloudNetworkingStrategy: None, - Environment: "", - StorageClass: "", - CNPGClusterName: "", - }, nil + return NewReplicationContext(NotPresent, "", ""), nil } // If no remote clusters, then just proceed with a regular cluster @@ -103,19 +114,16 @@ func GetReplicationContext(ctx context.Context, client client.Client, documentdb environment = self.EnvironmentOverride } - return &ReplicationContext{ - CNPGClusterName: generateCNPGClusterName(documentdb.Name, self.Name), - OtherCNPGClusterNames: otherCNPGClusterNames, - CrossCloudNetworkingStrategy: crossCloudNetworkingStrategy(documentdb.Spec.ClusterReplication.CrossCloudNetworkingStrategy), - PrimaryCNPGClusterName: primaryCluster, - Environment: environment, - StorageClass: storageClass, - state: replicationState, - FleetMemberName: self.Name, - OtherFleetMemberNames: others, - targetLocalPrimary: documentdb.Status.TargetPrimary, - currentLocalPrimary: documentdb.Status.LocalPrimary, - }, nil + rc := NewReplicationContext(replicationState, documentdb.Status.LocalPrimary, documentdb.Status.TargetPrimary) + rc.CNPGClusterName = generateCNPGClusterName(documentdb.Name, self.Name) + rc.OtherCNPGClusterNames = otherCNPGClusterNames + rc.CrossCloudNetworkingStrategy = crossCloudNetworkingStrategy(documentdb.Spec.ClusterReplication.CrossCloudNetworkingStrategy) + rc.PrimaryCNPGClusterName = primaryCluster + rc.Environment = environment + rc.StorageClass = storageClass + rc.FleetMemberName = self.Name + rc.OtherFleetMemberNames = others + return rc, nil } // String implements fmt.Stringer interface for better logging output @@ -229,7 +237,7 @@ func (r *ReplicationContext) CreateStandbyNamesList() []string { return standbyNames } -func getTopology(ctx context.Context, client client.Client, documentdb dbpreview.DocumentDB) (*dbpreview.MemberCluster, []string, replicationState, error) { +func getTopology(ctx context.Context, client client.Client, documentdb dbpreview.DocumentDB) (*dbpreview.MemberCluster, []string, ReplicationState, error) { memberClusterName := documentdb.Name var err error diff --git a/operator/src/internal/utils/util.go b/operator/src/internal/utils/util.go index fdfe07dd..14b92977 100644 --- a/operator/src/internal/utils/util.go +++ b/operator/src/internal/utils/util.go @@ -184,6 +184,10 @@ func EnsureServiceIP(ctx context.Context, service *corev1.Service) (string, erro } // UpsertService checks if the Service already exists, and creates it if not. +// An existing Service controlled by the desired Service's controller owner +// has its ports converged to the desired ports (for example after a gateway +// port change); server-assigned fields such as ClusterIP(s) and NodePorts and +// all other spec and metadata fields are preserved. func UpsertService(ctx context.Context, c client.Client, service *corev1.Service) (*corev1.Service, error) { log := log.FromContext(ctx) foundService := &corev1.Service{} @@ -203,6 +207,9 @@ func UpsertService(ctx context.Context, c client.Client, service *corev1.Service return nil, err } } else { + if sameController(foundService, service) { + foundService.Spec.Ports = convergeServicePorts(foundService.Spec.Ports, service.Spec.Ports) + } if err := c.Update(ctx, foundService); err != nil { return nil, err } @@ -210,6 +217,36 @@ func UpsertService(ctx context.Context, c client.Client, service *corev1.Service return foundService, nil } +// sameController reports whether found is controlled by desired's controller +// owner, so UpsertService never rewrites a Service it does not own. +func sameController(found, desired *corev1.Service) bool { + owner := metav1.GetControllerOf(desired) + if owner == nil { + return false + } + current := metav1.GetControllerOf(found) + return current != nil && current.UID == owner.UID +} + +// convergeServicePorts returns the desired ports, keeping the NodePort the API +// server allocated to a same-named current port when the desired port leaves +// it unset. +func convergeServicePorts(current, desired []corev1.ServicePort) []corev1.ServicePort { + ports := make([]corev1.ServicePort, len(desired)) + for i, port := range desired { + if port.NodePort == 0 { + for _, cur := range current { + if cur.Name == port.Name && cur.Protocol == port.Protocol { + port.NodePort = cur.NodePort + break + } + } + } + ports[i] = port + } + return ports +} + func GetPortFor(name string) int32 { switch name { case POSTGRES_PORT: diff --git a/operator/src/internal/utils/util_test.go b/operator/src/internal/utils/util_test.go index b069f44d..4a6ac1b4 100644 --- a/operator/src/internal/utils/util_test.go +++ b/operator/src/internal/utils/util_test.go @@ -4,6 +4,7 @@ package util import ( + "context" "strings" "testing" @@ -11,6 +12,9 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" ) func TestGenerateServiceName(t *testing.T) { @@ -1017,3 +1021,58 @@ func TestBuildGatewayService_ProductNeutral(t *testing.T) { t.Fatalf("unexpected owner/selector/annotations: %+v", svc.ObjectMeta) } } + +// TestUpsertServiceConvergesPorts updates an existing gateway Service to a new +// gateway port, keeping server-assigned and unrelated fields, and leaves a +// Service controlled by another owner untouched. +func TestUpsertServiceConvergesPorts(t *testing.T) { + ctx := context.Background() + opts := GatewayServiceOptions{ + Name: "svc", Namespace: "ns", ClusterName: "acct", + OwnerAPIVersion: "cosmos.example.com/v1", OwnerKind: "Account", OwnerUID: "uid-1", + Port: 10260, EndpointEnabled: true, ServiceType: corev1.ServiceTypeLoadBalancer, + } + existing := BuildGatewayService(opts) + existing.Labels = map[string]string{"team": "data"} + existing.Annotations = map[string]string{"example.com/keep": "true"} + existing.Spec.ClusterIP = "10.0.0.5" + existing.Spec.ClusterIPs = []string{"10.0.0.5"} + existing.Spec.Ports[0].NodePort = 31000 + existing.Spec.SessionAffinity = corev1.ServiceAffinityNone + + opts.Port = 8081 + foreign := existing.DeepCopy() + foreign.Name = "foreign" + foreign.OwnerReferences[0].UID = "other-uid" + + c := fake.NewClientBuilder().WithObjects(existing, foreign).Build() + got, err := UpsertService(ctx, c, BuildGatewayService(opts)) + if err != nil { + t.Fatal(err) + } + if err := c.Get(ctx, client.ObjectKeyFromObject(existing), got); err != nil { + t.Fatal(err) + } + want := corev1.ServicePort{Name: "gateway", Protocol: corev1.ProtocolTCP, Port: 8081, TargetPort: intstr.FromInt(8081), NodePort: 31000} + if len(got.Spec.Ports) != 1 || got.Spec.Ports[0] != want { + t.Errorf("ports = %+v, want [%+v]", got.Spec.Ports, want) + } + if got.Spec.ClusterIP != "10.0.0.5" || len(got.Spec.ClusterIPs) != 1 || got.Spec.ClusterIPs[0] != "10.0.0.5" { + t.Errorf("cluster IPs not preserved: %q %v", got.Spec.ClusterIP, got.Spec.ClusterIPs) + } + if got.Labels["team"] != "data" || got.Annotations["example.com/keep"] != "true" || got.Spec.SessionAffinity != corev1.ServiceAffinityNone { + t.Errorf("unrelated fields not preserved: %+v", got) + } + + foreignOpts := opts + foreignOpts.Name = "foreign" + if _, err := UpsertService(ctx, c, BuildGatewayService(foreignOpts)); err != nil { + t.Fatal(err) + } + if err := c.Get(ctx, client.ObjectKeyFromObject(foreign), got); err != nil { + t.Fatal(err) + } + if got.Spec.Ports[0].Port != 10260 { + t.Errorf("a Service controlled by another owner was rewritten: %+v", got.Spec.Ports) + } +} diff --git a/operator/src/pkg/core/engine.go b/operator/src/pkg/core/engine.go index ba37e206..9b1e9308 100644 --- a/operator/src/pkg/core/engine.go +++ b/operator/src/pkg/core/engine.go @@ -45,7 +45,22 @@ type ( // cluster reconciler; nil runs every cluster as a single-region primary. ReplicationHooks[T client.Object] = engine.ReplicationHooks[T] ReplicationContext = util.ReplicationContext - SQLExecutor = engine.SQLExecutor + // ReplicationState is the role of a cluster in its replication topology. + ReplicationState = util.ReplicationState + SQLExecutor = engine.SQLExecutor +) + +// Replication states for NewReplicationContext. +const ( + // NoReplication is a single-region primary; its endpoint is always enabled. + NoReplication = util.NoReplication + // ReplicationPrimary is the primary cluster of a replication topology. + ReplicationPrimary = util.Primary + // ReplicationReplica replicates from ReplicationContext.PrimaryCNPGClusterName. + ReplicationReplica = util.Replica + // ReplicationNotPresent is not a member of its topology; the cluster + // reconciler deletes the cluster's resources. + ReplicationNotPresent = util.NotPresent ) const ( @@ -68,6 +83,16 @@ func NewSingleRegionContext(name, environment, storageClass string) *Replication return util.NewSingleRegionContext(name, environment, storageClass) } +// NewReplicationContext returns the replication context of a cluster in state, +// for ReplicationHooks.Context. A replicating (primary or replica) cluster +// serves its gateway endpoint only when currentLocalPrimary equals +// targetLocalPrimary, so a pending promotion keeps the endpoint disabled. +// The caller sets the exported topology fields (CNPGClusterName, +// PrimaryCNPGClusterName, OtherCNPGClusterNames, StorageClass, ...). +func NewReplicationContext(state ReplicationState, currentLocalPrimary, targetLocalPrimary string) *ReplicationContext { + return util.NewReplicationContext(state, currentLocalPrimary, targetLocalPrimary) +} + // GatewayPort is the gateway port of a profile, or the GATEWAY_PORT env default. func GatewayPort(p ProductProfile) int32 { return engine.GatewayPort(p) } diff --git a/operator/src/pkg/core/engine_test.go b/operator/src/pkg/core/engine_test.go index 1c7c8ea1..35563c78 100644 --- a/operator/src/pkg/core/engine_test.go +++ b/operator/src/pkg/core/engine_test.go @@ -21,6 +21,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" + dbpreview "github.com/documentdb/documentdb-operator/api/preview" "github.com/documentdb/documentdb-operator/pkg/core" ) @@ -227,3 +228,51 @@ func TestClusterReconcilerSecondProduct(t *testing.T) { t.Fatalf("sql = %q, want one pgcosmos version query", sql) } } + +// monitoredAccountAdapter enables monitoring with product metric naming. +type monitoredAccountAdapter struct{ accountAdapter } + +func (monitoredAccountAdapter) Profile() core.ProductProfile { + p := accountAdapter{}.Profile() + p.Naming.MetricClusterAttribute = "cosmos.cluster" + p.Naming.HealthMetricName = "cosmos.postgres.up" + return p +} + +func (monitoredAccountAdapter) Spec(*account) core.ClusterSpec { + return core.ClusterSpec{Monitoring: &dbpreview.MonitoringSpec{Enabled: true}} +} + +// TestSecondProductMonitoringConfig checks the collector config the cluster +// reconciler writes for a product with its own metric naming. +func TestSecondProductMonitoringConfig(t *testing.T) { + ctx := context.Background() + scheme := runtime.NewScheme() + if err := clientgoscheme.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + scheme.AddKnownTypes(accountGV, &account{}, &accountList{}) + metav1.AddToGroupVersion(scheme, accountGV) + acct := &account{ObjectMeta: metav1.ObjectMeta{Name: "acct", Namespace: "ns", UID: "uid-1"}} + c := fake.NewClientBuilder().WithScheme(scheme).Build() + r := &core.ClusterReconciler[*account]{Client: c, Scheme: scheme, Adapter: monitoredAccountAdapter{}} + if err := r.ReconcileOtelConfigMap(ctx, acct, "ns"); err != nil { + t.Fatal(err) + } + cm := &corev1.ConfigMap{} + if err := c.Get(ctx, client.ObjectKey{Name: "acct-otel-config", Namespace: "ns"}, cm); err != nil { + t.Fatal(err) + } + if static := cm.Data["static.yaml"]; !strings.Contains(static, "metric_name: cosmos.postgres.up\n") { + t.Errorf("static.yaml does not report cosmos.postgres.up:\n%s", static) + } + if dynamic := cm.Data["dynamic.yaml"]; !strings.Contains(dynamic, "key: cosmos.cluster") || + !strings.HasPrefix(dynamic, "# Auto-generated by cosmos-operator.") { + t.Errorf("dynamic.yaml does not use the Cosmos naming:\n%s", dynamic) + } + for key, data := range cm.Data { + if strings.Contains(data, "documentdb.") { + t.Errorf("%s uses DocumentDB metric naming:\n%s", key, data) + } + } +} diff --git a/operator/src/pkg/core/example_test.go b/operator/src/pkg/core/example_test.go index d8749794..3260adfd 100644 --- a/operator/src/pkg/core/example_test.go +++ b/operator/src/pkg/core/example_test.go @@ -42,3 +42,18 @@ func Example() { // operator // ALTER EXTENSION pgcosmos UPDATE } + +// ExampleNewReplicationContext builds the contexts a product's +// ReplicationHooks returns for a primary awaiting promotion and a replica. +func ExampleNewReplicationContext() { + promoting := core.NewReplicationContext(core.ReplicationPrimary, "account-east", "account-west") + fmt.Println(promoting.IsPrimary(), promoting.EndpointEnabled()) + + replica := core.NewReplicationContext(core.ReplicationReplica, "", "") + replica.PrimaryCNPGClusterName = "account-east" + replica.OtherCNPGClusterNames = []string{"account-east"} + fmt.Println(replica.IsPrimary(), replica.IsReplicating(), replica.GetReplicationSource()) + // Output: + // true false + // false true account-east +} diff --git a/operator/src/pkg/core/replication_test.go b/operator/src/pkg/core/replication_test.go new file mode 100644 index 00000000..a9f23eee --- /dev/null +++ b/operator/src/pkg/core/replication_test.go @@ -0,0 +1,297 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package core_test + +import ( + "context" + "strings" + "testing" + "time" + + cnpgv1 "github.com/cloudnative-pg/cloudnative-pg/api/v1" + snapshotv1 "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/tools/record" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + dbpreview "github.com/documentdb/documentdb-operator/api/preview" + "github.com/documentdb/documentdb-operator/pkg/core" +) + +// accountReplication is a second product's multi-region hooks: it reports a +// fixed replication context built through the public constructor and marks +// non-primary CNPG clusters as replicas of the primary. +type accountReplication struct { + rc *core.ReplicationContext + added int +} + +var _ core.ReplicationHooks[*account] = (*accountReplication)(nil) + +func (h *accountReplication) Context(context.Context, *account) (*core.ReplicationContext, error) { + return h.rc, nil +} + +func (h *accountReplication) AddToClusterSpec(_ context.Context, _ *account, rc *core.ReplicationContext, desired *cnpgv1.Cluster) error { + h.added++ + if !rc.IsPrimary() { + enabled := true + desired.Spec.ReplicaCluster = &cnpgv1.ReplicaClusterConfiguration{Enabled: &enabled, Source: rc.PrimaryCNPGClusterName} + } + return nil +} + +func (h *accountReplication) SyncChanges(context.Context, *cnpgv1.Cluster, *cnpgv1.Cluster, *account, *core.ReplicationContext) ([]core.JSONPatch, error, time.Duration) { + return nil, nil, 0 +} + +func (h *accountReplication) ReconcilePrimary(context.Context, *account, *core.ReplicationContext, *cnpgv1.Cluster) *ctrl.Result { + return nil +} + +func (h *accountReplication) ReconcileNetworking(context.Context, *account, *core.ReplicationContext) *ctrl.Result { + return nil +} + +// replicatedAccountAdapter grants a replication admin role on primaries. +type replicatedAccountAdapter struct{ accountAdapter } + +func (replicatedAccountAdapter) Profile() core.ProductProfile { + p := accountAdapter{}.Profile() + p.Postgres.ReplicationAdminRole = "cosmos_admin" + return p +} + +func (a replicatedAccountAdapter) Intent(obj *account) core.ClusterIntent { + intent := accountAdapter{}.Intent(obj) + intent.Product = a.Profile() + return intent +} + +// regionContext returns the context of region "acct" in a two-region topology. +func regionContext(state core.ReplicationState, currentLocalPrimary, targetLocalPrimary string) *core.ReplicationContext { + rc := core.NewReplicationContext(state, currentLocalPrimary, targetLocalPrimary) + rc.CNPGClusterName = "acct" + rc.PrimaryCNPGClusterName = "acct-primary" + rc.OtherCNPGClusterNames = []string{"acct-primary"} + return rc +} + +func replicationScheme(t *testing.T) *runtime.Scheme { + t.Helper() + scheme := runtime.NewScheme() + for _, add := range []func(*runtime.Scheme) error{clientgoscheme.AddToScheme, cnpgv1.AddToScheme, dbpreview.AddToScheme, snapshotv1.AddToScheme} { + if err := add(scheme); err != nil { + t.Fatal(err) + } + } + scheme.AddKnownTypes(accountGV, &account{}, &accountList{}) + metav1.AddToGroupVersion(scheme, accountGV) + return scheme +} + +func TestReplicationContextStates(t *testing.T) { + for _, tc := range []struct { + name string + rc *core.ReplicationContext + primary, replicating, notPresent, endpoint bool + }{ + {"single region", core.NewSingleRegionContext("acct", "", ""), true, false, false, true}, + {"no replication ignores promotion", core.NewReplicationContext(core.NoReplication, "a-1", "a-2"), true, false, false, true}, + {"primary", regionContext(core.ReplicationPrimary, "acct-1", "acct-1"), true, true, false, true}, + {"primary awaiting promotion", regionContext(core.ReplicationPrimary, "acct-1", "acct-2"), true, true, false, false}, + {"replica", regionContext(core.ReplicationReplica, "", ""), false, true, false, true}, + {"replica awaiting promotion", regionContext(core.ReplicationReplica, "", "acct-2"), false, true, false, false}, + {"not present", core.NewReplicationContext(core.ReplicationNotPresent, "", ""), false, false, true, true}, + } { + t.Run(tc.name, func(t *testing.T) { + if got := tc.rc.IsPrimary(); got != tc.primary { + t.Errorf("IsPrimary = %t", got) + } + if got := tc.rc.IsReplicating(); got != tc.replicating { + t.Errorf("IsReplicating = %t", got) + } + if got := tc.rc.IsNotPresent(); got != tc.notPresent { + t.Errorf("IsNotPresent = %t", got) + } + if got := tc.rc.EndpointEnabled(); got != tc.endpoint { + t.Errorf("EndpointEnabled = %t", got) + } + }) + } +} + +// TestClusterReconcilerReplicationStates drives the shared cluster reconciler +// with publicly constructed replication contexts. +func TestClusterReconcilerReplicationStates(t *testing.T) { + for _, tc := range []struct { + name string + rc *core.ReplicationContext + wantHookCalls int + wantReplica bool + wantRoleGrants bool + }{ + {"single region", core.NewSingleRegionContext("acct", "", ""), 0, false, true}, + {"primary", regionContext(core.ReplicationPrimary, "acct-1", "acct-1"), 1, false, true}, + {"replica", regionContext(core.ReplicationReplica, "", ""), 1, true, false}, + } { + t.Run(tc.name, func(t *testing.T) { + ctx := context.Background() + scheme := replicationScheme(t) + acct := &account{ + ObjectMeta: metav1.ObjectMeta{Name: "acct", Namespace: "ns", UID: "uid-1"}, + Spec: accountSpec{Instances: 1, PvcSize: "10Gi"}, + } + c := fake.NewClientBuilder().WithScheme(scheme).WithObjects(acct). + WithStatusSubresource(&account{}, &cnpgv1.Cluster{}).Build() + hooks := &accountReplication{rc: tc.rc} + var sql []string + r := &core.ClusterReconciler[*account]{ + Client: c, Scheme: scheme, Recorder: record.NewFakeRecorder(10), + Adapter: replicatedAccountAdapter{}, Replication: hooks, + SQLExecutor: func(_ context.Context, _ *cnpgv1.Cluster, cmd string) (string, error) { + sql = append(sql, cmd) + return "(1 row)", nil + }, + } + req := ctrl.Request{NamespacedName: types.NamespacedName{Name: "acct", Namespace: "ns"}} + for range 2 { // finalizer, then create + if _, err := r.Reconcile(ctx, req); err != nil { + t.Fatal(err) + } + } + cluster := &cnpgv1.Cluster{} + if err := c.Get(ctx, client.ObjectKey{Name: "acct", Namespace: "ns"}, cluster); err != nil { + t.Fatal(err) + } + if hooks.added != tc.wantHookCalls { + t.Errorf("AddToClusterSpec calls = %d, want %d", hooks.added, tc.wantHookCalls) + } + if isReplica := cluster.Spec.ReplicaCluster != nil && cluster.Spec.ReplicaCluster.Source == "acct-primary"; isReplica != tc.wantReplica { + t.Errorf("replica cluster = %+v, want replica %t", cluster.Spec.ReplicaCluster, tc.wantReplica) + } + + cluster.Status.Phase = "Cluster in healthy state" + cluster.Status.CurrentPrimary = "acct-1" + cluster.Status.InstancesStatus = map[cnpgv1.PodStatus][]string{cnpgv1.PodHealthy: {"acct-1"}} + if err := c.Status().Update(ctx, cluster); err != nil { + t.Fatal(err) + } + if _, err := r.Reconcile(ctx, req); err != nil { + t.Fatal(err) + } + roleCheck := core.ReplicationRoleCheckSQL("cosmos_admin") + checked := false + for _, cmd := range sql { + checked = checked || strings.Contains(cmd, roleCheck) + } + if checked != tc.wantRoleGrants { + t.Errorf("replication role checked = %t, want %t (sql %q)", checked, tc.wantRoleGrants, sql) + } + }) + } +} + +func TestClusterReconcilerNotPresentSkipsCluster(t *testing.T) { + ctx := context.Background() + scheme := replicationScheme(t) + acct := &account{ObjectMeta: metav1.ObjectMeta{Name: "acct", Namespace: "ns", UID: "uid-1"}} + c := fake.NewClientBuilder().WithScheme(scheme).WithObjects(acct).Build() + hooks := &accountReplication{rc: core.NewReplicationContext(core.ReplicationNotPresent, "", "")} + r := &core.ClusterReconciler[*account]{ + Client: c, Scheme: scheme, Recorder: record.NewFakeRecorder(10), + Adapter: accountAdapter{}, Replication: hooks, + } + req := ctrl.Request{NamespacedName: types.NamespacedName{Name: "acct", Namespace: "ns"}} + for range 2 { + if _, err := r.Reconcile(ctx, req); err != nil { + t.Fatal(err) + } + } + clusters := &cnpgv1.ClusterList{} + if err := c.List(ctx, clusters); err != nil { + t.Fatal(err) + } + if len(clusters.Items) != 0 || hooks.added != 0 { + t.Fatalf("not-present region rendered %d clusters and %d hook calls", len(clusters.Items), hooks.added) + } +} + +type backupAdapter struct{} + +func (backupAdapter) New() *dbpreview.Backup { return &dbpreview.Backup{} } +func (backupAdapter) NewList() client.ObjectList { return &dbpreview.BackupList{} } +func (backupAdapter) Spec(b *dbpreview.Backup) *dbpreview.BackupSpec { return &b.Spec } +func (backupAdapter) Status(b *dbpreview.Backup) *dbpreview.BackupStatus { return &b.Status } +func (backupAdapter) Items(list client.ObjectList) []*dbpreview.Backup { + items := []*dbpreview.Backup{} + for i := range list.(*dbpreview.BackupList).Items { + items = append(items, &list.(*dbpreview.BackupList).Items[i]) + } + return items +} + +// TestBackupReconcilerReplicationGating shows that backups run only on a +// primary whose endpoint is enabled (no promotion pending). +func TestBackupReconcilerReplicationGating(t *testing.T) { + for _, tc := range []struct { + name string + rc *core.ReplicationContext + wantCreated bool + wantPhase cnpgv1.BackupPhase + wantRequeue time.Duration + }{ + {"primary", regionContext(core.ReplicationPrimary, "acct-1", "acct-1"), true, "", 5 * time.Second}, + {"primary awaiting promotion", regionContext(core.ReplicationPrimary, "acct-1", "acct-2"), false, "", time.Minute}, + {"replica", regionContext(core.ReplicationReplica, "", ""), false, dbpreview.BackupPhaseSkipped, 0}, // requeues for expiry + } { + t.Run(tc.name, func(t *testing.T) { + ctx := context.Background() + scheme := replicationScheme(t) + acct := &account{ObjectMeta: metav1.ObjectMeta{Name: "acct", Namespace: "ns", UID: "uid-1"}} + backup := &dbpreview.Backup{ + ObjectMeta: metav1.ObjectMeta{Name: "b1", Namespace: "ns", UID: "uid-b"}, + Spec: dbpreview.BackupSpec{Cluster: cnpgv1.LocalObjectReference{Name: "acct"}}, + } + vsc := &snapshotv1.VolumeSnapshotClass{ + ObjectMeta: metav1.ObjectMeta{Name: "default", Annotations: map[string]string{"snapshot.storage.kubernetes.io/is-default-class": "true"}}, + Driver: "disk.csi.example.com", + } + c := fake.NewClientBuilder().WithScheme(scheme).WithObjects(acct, backup, vsc). + WithStatusSubresource(&dbpreview.Backup{}).Build() + r := &core.BackupReconciler[*account, *dbpreview.Backup]{ + Client: c, Scheme: scheme, Recorder: record.NewFakeRecorder(10), + Cluster: accountAdapter{}, Backups: backupAdapter{}, + Replication: &accountReplication{rc: tc.rc}, + } + res, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: types.NamespacedName{Name: "b1", Namespace: "ns"}}) + if err != nil { + t.Fatal(err) + } + if tc.wantPhase == "" && res.RequeueAfter != tc.wantRequeue { + t.Errorf("requeue = %v, want %v", res.RequeueAfter, tc.wantRequeue) + } + cnpgBackup := &cnpgv1.Backup{} + err = c.Get(ctx, client.ObjectKey{Name: "b1", Namespace: "ns"}, cnpgBackup) + if created := err == nil; created != tc.wantCreated { + t.Fatalf("CNPG backup created = %t, want %t (err %v)", created, tc.wantCreated, err) + } + if tc.wantCreated && cnpgBackup.Spec.Cluster.Name != "acct" { + t.Errorf("CNPG backup cluster = %q, want acct", cnpgBackup.Spec.Cluster.Name) + } + got := &dbpreview.Backup{} + if err := c.Get(ctx, client.ObjectKeyFromObject(backup), got); err != nil { + t.Fatal(err) + } + if got.Status.Phase != tc.wantPhase { + t.Errorf("backup phase = %q, want %q", got.Status.Phase, tc.wantPhase) + } + }) + } +} From 62d7c330dd7aa480bf53672b8a3756af77b77f8f Mon Sep 17 00:00:00 2001 From: Rayhan Hossain Date: Tue, 20 Oct 2026 12:31:40 -0700 Subject: [PATCH 22/27] test(core): cover reconciler lifecycle and failure paths Exercise cluster, certificate, backup and scheduled-backup reconciliation, including client failures, replication hooks, retention and PV recovery. Cover public validation and resource helpers and SQL execution failures. Changed-statement coverage is 94.55% with the CI coverage command and unchanged 90% threshold. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 33b3cb3c-1b25-4ffb-bff1-ebf0f3d04058 Signed-off-by: Rayhan Hossain --- operator/src/internal/engine/backup_test.go | 424 ++++++++++ .../src/internal/engine/certificate_test.go | 365 +++++++++ .../engine/cluster_reconciler_test.go | 774 ++++++++++++++++++ operator/src/internal/engine/helpers_test.go | 124 +++ .../internal/engine/predicates_sql_test.go | 112 +++ .../internal/engine/scheduledbackup_test.go | 326 ++++++++ operator/src/pkg/core/facade_test.go | 179 ++++ 7 files changed, 2304 insertions(+) create mode 100644 operator/src/internal/engine/backup_test.go create mode 100644 operator/src/internal/engine/certificate_test.go create mode 100644 operator/src/internal/engine/cluster_reconciler_test.go create mode 100644 operator/src/internal/engine/helpers_test.go create mode 100644 operator/src/internal/engine/predicates_sql_test.go create mode 100644 operator/src/internal/engine/scheduledbackup_test.go create mode 100644 operator/src/pkg/core/facade_test.go diff --git a/operator/src/internal/engine/backup_test.go b/operator/src/internal/engine/backup_test.go new file mode 100644 index 00000000..e5e2bdaf --- /dev/null +++ b/operator/src/internal/engine/backup_test.go @@ -0,0 +1,424 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package engine + +import ( + "context" + "strings" + "testing" + "time" + + cnpgv1 "github.com/cloudnative-pg/cloudnative-pg/api/v1" + snapshotv1 "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/tools/record" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" + + dbpreview "github.com/documentdb/documentdb-operator/api/preview" + "github.com/documentdb/documentdb-operator/internal/product" + util "github.com/documentdb/documentdb-operator/internal/utils" +) + +type backupReconciler = BackupReconciler[*dbpreview.DocumentDB, *dbpreview.Backup] + +func newBackupReconciler(c client.Client, s *runtime.Scheme, rec record.EventRecorder) *backupReconciler { + return &backupReconciler{ + Client: c, + Scheme: s, + Recorder: rec, + Cluster: product.DocumentDBAdapter{}, + Backups: product.DocumentDBBackupAdapter{}, + } +} + +var backupKey = ctrl.Request{NamespacedName: types.NamespacedName{Name: "bk", Namespace: testNamespace}} + +func pendingBackup() *dbpreview.Backup { + return testBackupFor("bk", "", time.Now()) +} + +func defaultSnapshotClass() *snapshotv1.VolumeSnapshotClass { + return &snapshotv1.VolumeSnapshotClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: "default-snap", + Annotations: map[string]string{"snapshot.storage.kubernetes.io/is-default-class": "true"}, + }, + Driver: "test.csi", + DeletionPolicy: snapshotv1.VolumeSnapshotContentDelete, + } +} + +func getBackup(t *testing.T, c client.Client) *dbpreview.Backup { + t.Helper() + b := &dbpreview.Backup{} + if err := c.Get(context.Background(), backupKey.NamespacedName, b); err != nil { + t.Fatalf("get backup: %v", err) + } + return b +} + +func TestBackupReconcileNotFoundAndGetError(t *testing.T) { + s := newTestScheme(t) + r := newBackupReconciler(newFakeClient(t, s, nil), s, record.NewFakeRecorder(10)) + if res, err := r.Reconcile(context.Background(), backupKey); err != nil || res != (ctrl.Result{}) { + t.Fatalf("expected empty result for a missing backup, got %v, %v", res, err) + } + + c := newFakeClient(t, s, &interceptor.Funcs{Get: func(context.Context, client.WithWatch, client.ObjectKey, client.Object, ...client.GetOption) error { + return errInjected + }}) + r = newBackupReconciler(c, s, record.NewFakeRecorder(10)) + if _, err := r.Reconcile(context.Background(), backupKey); err != errInjected { + t.Fatalf("expected injected error, got %v", err) + } +} + +func TestBackupReconcileDeletesExpiredBackup(t *testing.T) { + s := newTestScheme(t) + b := testBackupFor("bk", cnpgv1.BackupPhaseCompleted, time.Now().Add(-48*time.Hour)) + expired := metav1.NewTime(time.Now().Add(-time.Hour)) + b.Status.ExpiredAt = &expired + rec := record.NewFakeRecorder(10) + c := newFakeClient(t, s, nil, b) + r := newBackupReconciler(c, s, rec) + + if _, err := r.Reconcile(context.Background(), backupKey); err != nil { + t.Fatalf("reconcile: %v", err) + } + if err := c.Get(context.Background(), backupKey.NamespacedName, &dbpreview.Backup{}); !apierrors.IsNotFound(err) { + t.Fatalf("expected expired backup to be deleted, got %v", err) + } + if events := drainEvents(rec); !hasEvent(events, "BackupExpired") || !hasEvent(events, "BackupDeleted") { + t.Fatalf("expected expiry events, got %v", events) + } +} + +func TestBackupReconcileExpiredDeleteFailure(t *testing.T) { + s := newTestScheme(t) + b := testBackupFor("bk", cnpgv1.BackupPhaseCompleted, time.Now().Add(-48*time.Hour)) + expired := metav1.NewTime(time.Now().Add(-time.Hour)) + b.Status.ExpiredAt = &expired + rec := record.NewFakeRecorder(10) + c := newFakeClient(t, s, &interceptor.Funcs{Delete: func(context.Context, client.WithWatch, client.Object, ...client.DeleteOption) error { + return errInjected + }}, b) + r := newBackupReconciler(c, s, rec) + if _, err := r.Reconcile(context.Background(), backupKey); err != errInjected { + t.Fatalf("expected injected error, got %v", err) + } + if !hasEvent(drainEvents(rec), "BackupDeleteFailed") { + t.Fatal("expected BackupDeleteFailed event") + } +} + +func TestBackupReconcileDoneBackupRequeuesUntilExpiry(t *testing.T) { + s := newTestScheme(t) + b := testBackupFor("bk", cnpgv1.BackupPhaseCompleted, time.Now()) + expires := metav1.NewTime(time.Now().Add(2 * time.Hour)) + b.Status.ExpiredAt = &expires + r := newBackupReconciler(newFakeClient(t, s, nil, b), s, record.NewFakeRecorder(10)) + res, err := r.Reconcile(context.Background(), backupKey) + if err != nil || res.RequeueAfter <= time.Hour || res.RequeueAfter > 2*time.Hour { + t.Fatalf("expected requeue at expiry, got %v, %v", res, err) + } +} + +func TestBackupReconcileFailsWithoutCluster(t *testing.T) { + s := newTestScheme(t) + rec := record.NewFakeRecorder(10) + c := newFakeClient(t, s, nil, pendingBackup()) + r := newBackupReconciler(c, s, rec) + res, err := r.Reconcile(context.Background(), backupKey) + if err != nil { + t.Fatalf("reconcile: %v", err) + } + got := getBackup(t, c) + if got.Status.Phase != cnpgv1.BackupPhaseFailed || !strings.Contains(got.Status.Message, "Failed to get associated DocumentDB cluster") { + t.Fatalf("expected failed phase naming the cluster kind, got %+v", got.Status) + } + // Default 30-day retention from creation time. + if got.Status.ExpiredAt == nil || res.RequeueAfter < 29*24*time.Hour { + t.Fatalf("expected default retention expiry, got %v / %v", got.Status.ExpiredAt, res.RequeueAfter) + } + if !hasEvent(drainEvents(rec), "BackupFailed") { + t.Fatal("expected BackupFailed event") + } +} + +func TestBackupReconcileFailsWithoutSnapshotClass(t *testing.T) { + s := newTestScheme(t) + cluster := testCluster() + cluster.Spec.Backup = &dbpreview.BackupConfiguration{RetentionDays: 2} + c := newFakeClient(t, s, nil, pendingBackup(), cluster) + r := newBackupReconciler(c, s, record.NewFakeRecorder(10)) + res, err := r.Reconcile(context.Background(), backupKey) + if err != nil { + t.Fatalf("reconcile: %v", err) + } + got := getBackup(t, c) + if got.Status.Phase != cnpgv1.BackupPhaseFailed || !strings.Contains(got.Status.Message, "Please create a default VolumeSnapshotClass") { + t.Fatalf("expected failure asking for a VolumeSnapshotClass, got %+v", got.Status) + } + // The cluster's retention applies. + if res.RequeueAfter > 2*24*time.Hour || res.RequeueAfter < 47*time.Hour { + t.Fatalf("expected cluster retention of 2 days, got %v", res.RequeueAfter) + } +} + +func TestBackupReconcileCreatesCNPGBackupForPrimary(t *testing.T) { + s := newTestScheme(t) + cluster := testCluster() + cluster.Spec.Environment = "aks" + rec := record.NewFakeRecorder(10) + c := newFakeClient(t, s, nil, pendingBackup(), cluster) + r := newBackupReconciler(c, s, rec) + + res, err := r.Reconcile(context.Background(), backupKey) + if err != nil || res.RequeueAfter != 5*time.Second { + t.Fatalf("expected 5s requeue after creating the CNPG backup, got %v, %v", res, err) + } + vsc := &snapshotv1.VolumeSnapshotClass{} + if err := c.Get(context.Background(), types.NamespacedName{Name: "azure-disk-snapclass"}, vsc); err != nil { + t.Fatalf("expected the AKS VolumeSnapshotClass to be created: %v", err) + } + cnpgBackup := &cnpgv1.Backup{} + if err := c.Get(context.Background(), backupKey.NamespacedName, cnpgBackup); err != nil { + t.Fatalf("expected CNPG backup: %v", err) + } + if cnpgBackup.Spec.Cluster.Name != "db" { + t.Fatalf("expected CNPG backup of the single-region CNPG cluster, got %q", cnpgBackup.Spec.Cluster.Name) + } + if owner := metav1.GetControllerOf(cnpgBackup); owner == nil || owner.Name != "bk" { + t.Fatalf("expected CNPG backup owned by the Backup, got %v", cnpgBackup.OwnerReferences) + } + if events := drainEvents(rec); !hasEvent(events, "BackupInitialized") || !hasEvent(events, "azure-disk-snapclass") { + t.Fatalf("expected snapshot class and initialization events, got %v", events) + } +} + +func TestBackupReconcileReplicationRoles(t *testing.T) { + notPrimary := util.NewReplicationContext(util.Replica, "", "") + pendingPromotion := util.NewReplicationContext(util.Primary, "old", "new") + + t.Run("replica is skipped", func(t *testing.T) { + s := newTestScheme(t) + c := newFakeClient(t, s, nil, pendingBackup(), testCluster(), defaultSnapshotClass()) + r := newBackupReconciler(c, s, record.NewFakeRecorder(10)) + r.Replication = &fakeReplication[*dbpreview.DocumentDB]{rc: notPrimary} + if _, err := r.Reconcile(context.Background(), backupKey); err != nil { + t.Fatal(err) + } + got := getBackup(t, c) + if got.Status.Phase != dbpreview.BackupPhaseSkipped || !strings.Contains(got.Status.Message, "primary cluster") { + t.Fatalf("expected skipped backup, got %+v", got.Status) + } + }) + + t.Run("primary with disabled endpoint is deferred", func(t *testing.T) { + s := newTestScheme(t) + c := newFakeClient(t, s, nil, pendingBackup(), testCluster(), defaultSnapshotClass()) + r := newBackupReconciler(c, s, record.NewFakeRecorder(10)) + r.Replication = &fakeReplication[*dbpreview.DocumentDB]{rc: pendingPromotion} + res, err := r.Reconcile(context.Background(), backupKey) + if err != nil || res.RequeueAfter != time.Minute { + t.Fatalf("expected one-minute deferral, got %v, %v", res, err) + } + if err := c.Get(context.Background(), backupKey.NamespacedName, &cnpgv1.Backup{}); !apierrors.IsNotFound(err) { + t.Fatalf("no CNPG backup should be created while deferred, got %v", err) + } + }) + + t.Run("replication context error", func(t *testing.T) { + s := newTestScheme(t) + c := newFakeClient(t, s, nil, pendingBackup(), testCluster(), defaultSnapshotClass()) + r := newBackupReconciler(c, s, record.NewFakeRecorder(10)) + r.Replication = &fakeReplication[*dbpreview.DocumentDB]{contextErr: errInjected} + if _, err := r.Reconcile(context.Background(), backupKey); err != errInjected { + t.Fatalf("expected injected error, got %v", err) + } + }) +} + +func TestBackupReconcileMirrorsCNPGBackupStatus(t *testing.T) { + s := newTestScheme(t) + cluster := testCluster() + cluster.Status.SchemaVersion = "0.110.0" + stopped := metav1.NewTime(time.Now()) + cnpgBackup := &cnpgv1.Backup{ + ObjectMeta: metav1.ObjectMeta{Name: "bk", Namespace: testNamespace}, + Status: cnpgv1.BackupStatus{Phase: cnpgv1.BackupPhaseCompleted, StartedAt: &stopped, StoppedAt: &stopped}, + } + c := newFakeClient(t, s, nil, pendingBackup(), cluster, defaultSnapshotClass(), cnpgBackup) + r := newBackupReconciler(c, s, record.NewFakeRecorder(10)) + + res, err := r.Reconcile(context.Background(), backupKey) + if err != nil || res.RequeueAfter < 29*24*time.Hour { + t.Fatalf("expected requeue at default expiry, got %v, %v", res, err) + } + got := getBackup(t, c) + if got.Status.Phase != cnpgv1.BackupPhaseCompleted || got.Status.SchemaVersion != "0.110.0" || got.Status.ExpiredAt == nil { + t.Fatalf("expected status mirrored from the CNPG backup, got %+v", got.Status) + } +} + +func TestBackupReconcileInProgressCNPGBackupRequeues(t *testing.T) { + s := newTestScheme(t) + cnpgBackup := &cnpgv1.Backup{ + ObjectMeta: metav1.ObjectMeta{Name: "bk", Namespace: testNamespace}, + Status: cnpgv1.BackupStatus{Phase: cnpgv1.BackupPhaseRunning}, + } + c := newFakeClient(t, s, nil, pendingBackup(), testCluster(), defaultSnapshotClass(), cnpgBackup) + r := newBackupReconciler(c, s, record.NewFakeRecorder(10)) + res, err := r.Reconcile(context.Background(), backupKey) + if err != nil || res.RequeueAfter != 10*time.Second { + t.Fatalf("expected 10s requeue for a running backup, got %v, %v", res, err) + } + if got := getBackup(t, c); got.Status.Phase != cnpgv1.BackupPhaseRunning { + t.Fatalf("expected running phase, got %q", got.Status.Phase) + } +} + +func TestBackupReconcileClientErrors(t *testing.T) { + isCNPGBackup := func(obj client.Object) bool { _, ok := obj.(*cnpgv1.Backup); return ok } + isSnapshotClass := func(obj client.Object) bool { _, ok := obj.(*snapshotv1.VolumeSnapshotClass); return ok } + isSnapshotClassList := func(obj client.ObjectList) bool { _, ok := obj.(*snapshotv1.VolumeSnapshotClassList); return ok } + + tests := []struct { + name string + environment string + funcs interceptor.Funcs + cnpgBackup bool + wantErr bool + wantMessage string + }{ + { + name: "listing snapshot classes fails", + funcs: interceptor.Funcs{List: func(ctx context.Context, c client.WithWatch, list client.ObjectList, opts ...client.ListOption) error { + if isSnapshotClassList(list) { + return errInjected + } + return c.List(ctx, list, opts...) + }}, + wantMessage: "Failed to ensure VolumeSnapshotClass: injected failure", + }, + { + name: "creating snapshot class fails", + environment: "aks", + funcs: interceptor.Funcs{Create: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.CreateOption) error { + if isSnapshotClass(obj) { + return errInjected + } + return c.Create(ctx, obj, opts...) + }}, + wantMessage: "Failed to ensure VolumeSnapshotClass: injected failure", + }, + { + name: "getting CNPG backup fails", + funcs: interceptor.Funcs{Get: func(ctx context.Context, c client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + if isCNPGBackup(obj) { + return errInjected + } + return c.Get(ctx, key, obj, opts...) + }}, + wantErr: true, + }, + { + name: "creating CNPG backup fails", + funcs: interceptor.Funcs{Create: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.CreateOption) error { + if isCNPGBackup(obj) { + return errInjected + } + return c.Create(ctx, obj, opts...) + }}, + wantMessage: "Failed to initialize backup: injected failure", + }, + { + name: "patching mirrored status fails", + cnpgBackup: true, + funcs: interceptor.Funcs{SubResourcePatch: func(context.Context, client.Client, string, client.Object, client.Patch, ...client.SubResourcePatchOption) error { + return errInjected + }}, + wantErr: true, + }, + { + name: "patching failed status fails", + funcs: interceptor.Funcs{ + Create: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.CreateOption) error { + if isCNPGBackup(obj) { + return errInjected + } + return c.Create(ctx, obj, opts...) + }, + SubResourcePatch: func(context.Context, client.Client, string, client.Object, client.Patch, ...client.SubResourcePatchOption) error { + return errInjected + }, + }, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := newTestScheme(t) + cluster := testCluster() + cluster.Spec.Environment = tt.environment + objs := []client.Object{pendingBackup(), cluster} + if tt.environment == "" { + objs = append(objs, defaultSnapshotClass()) + } + if tt.cnpgBackup { + objs = append(objs, &cnpgv1.Backup{ + ObjectMeta: metav1.ObjectMeta{Name: "bk", Namespace: testNamespace}, + Status: cnpgv1.BackupStatus{Phase: cnpgv1.BackupPhaseRunning}, + }) + } + c := newFakeClient(t, s, &tt.funcs, objs...) + r := newBackupReconciler(c, s, record.NewFakeRecorder(10)) + _, err := r.Reconcile(context.Background(), backupKey) + if tt.wantErr { + if err == nil { + t.Fatal("expected error") + } + return + } + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if got := getBackup(t, c); got.Status.Phase != cnpgv1.BackupPhaseFailed || got.Status.Message != tt.wantMessage { + t.Fatalf("expected failed phase with %q, got %+v", tt.wantMessage, got.Status) + } + }) + } +} + +func TestBackupCreateCNPGBackupInitializationError(t *testing.T) { + s := newTestScheme(t) + b := pendingBackup() + c := newFakeClient(t, s, nil, b) + // A scheme without the Backup kind cannot set the CNPG backup owner. + r := newBackupReconciler(c, runtime.NewScheme(), record.NewFakeRecorder(10)) + if _, err := r.CreateCNPGBackup(context.Background(), b, nil, util.NewSingleRegionContext("db", "", "")); err != nil { + t.Fatalf("unexpected error: %v", err) + } + if got := getBackup(t, c); got.Status.Phase != cnpgv1.BackupPhaseFailed || !strings.HasPrefix(got.Status.Message, "Failed to initialize backup") { + t.Fatalf("expected initialization failure, got %+v", got.Status) + } +} + +func TestBuildVolumeSnapshotClass(t *testing.T) { + vsc := BuildVolumeSnapshotClass("aks") + if vsc == nil || vsc.Driver != "disk.csi.azure.com" || vsc.Annotations["snapshot.storage.kubernetes.io/is-default-class"] != "true" || + vsc.DeletionPolicy != snapshotv1.VolumeSnapshotContentDelete { + t.Fatalf("unexpected AKS snapshot class: %+v", vsc) + } + for _, env := range []string{"", "eks", "gke"} { + if BuildVolumeSnapshotClass(env) != nil { + t.Fatalf("expected no default snapshot class for %q", env) + } + } +} diff --git a/operator/src/internal/engine/certificate_test.go b/operator/src/internal/engine/certificate_test.go new file mode 100644 index 00000000..7a2dd9ed --- /dev/null +++ b/operator/src/internal/engine/certificate_test.go @@ -0,0 +1,365 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package engine + +import ( + "context" + "slices" + "testing" + + cmapi "github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1" + cmmeta "github.com/cert-manager/cert-manager/pkg/apis/meta/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" + + dbpreview "github.com/documentdb/documentdb-operator/api/preview" + "github.com/documentdb/documentdb-operator/internal/product" +) + +type certReconciler = CertificateReconciler[*dbpreview.DocumentDB] + +var certKey = ctrl.Request{NamespacedName: types.NamespacedName{Name: "db", Namespace: testNamespace}} + +func newCertReconciler(c client.Client, s *runtime.Scheme) *certReconciler { + return &certReconciler{Client: c, Scheme: s, Adapter: product.DocumentDBAdapter{}} +} + +func clusterWithTLS(gw *dbpreview.GatewayTLS) *dbpreview.DocumentDB { + db := testCluster() + if gw != nil { + db.Spec.TLS = &dbpreview.TLSConfiguration{Gateway: gw} + } + return db +} + +func getTLSStatus(t *testing.T, c client.Client) *dbpreview.TLSStatus { + t.Helper() + db := &dbpreview.DocumentDB{} + if err := c.Get(context.Background(), certKey.NamespacedName, db); err != nil { + t.Fatal(err) + } + if db.Status.TLS == nil { + t.Fatal("expected TLS status to be initialized") + } + return db.Status.TLS +} + +func readyCert(name, secret string) *cmapi.Certificate { + return &cmapi.Certificate{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: testNamespace}, + Spec: cmapi.CertificateSpec{SecretName: secret}, + Status: cmapi.CertificateStatus{Conditions: []cmapi.CertificateCondition{ + {Type: cmapi.CertificateConditionReady, Status: cmmeta.ConditionTrue}, + }}, + } +} + +func TestCertificateReconcileNotFoundAndGetError(t *testing.T) { + s := newTestScheme(t) + r := newCertReconciler(newFakeClient(t, s, nil), s) + if res, err := r.Reconcile(context.Background(), certKey); err != nil || res != (ctrl.Result{}) { + t.Fatalf("expected no-op for missing cluster, got %v, %v", res, err) + } + c := newFakeClient(t, s, &interceptor.Funcs{Get: func(context.Context, client.WithWatch, client.ObjectKey, client.Object, ...client.GetOption) error { + return errInjected + }}) + if _, err := newCertReconciler(c, s).Reconcile(context.Background(), certKey); err != errInjected { + t.Fatalf("expected injected error, got %v", err) + } +} + +func TestCertificateSelfSignedLifecycle(t *testing.T) { + for _, tt := range []struct { + name string + gw *dbpreview.GatewayTLS + }{ + {name: "tls unset defaults to self-signed"}, + {name: "empty mode defaults to self-signed", gw: &dbpreview.GatewayTLS{}}, + {name: "legacy mode falls back to self-signed", gw: &dbpreview.GatewayTLS{Mode: "Disabled"}}, + } { + t.Run(tt.name, func(t *testing.T) { + s := newTestScheme(t) + c := newFakeClient(t, s, nil, clusterWithTLS(tt.gw)) + r := newCertReconciler(c, s) + + res, err := r.Reconcile(context.Background(), certKey) + if err != nil || res.RequeueAfter != RequeueAfterShort { + t.Fatalf("expected short requeue while creating, got %v, %v", res, err) + } + issuer := &cmapi.Issuer{} + if err := c.Get(context.Background(), types.NamespacedName{Name: "db-gateway-selfsigned", Namespace: testNamespace}, issuer); err != nil { + t.Fatalf("expected self-signed issuer: %v", err) + } + if issuer.Spec.SelfSigned == nil || metav1.GetControllerOf(issuer) == nil { + t.Fatalf("expected owned self-signed issuer, got %+v", issuer) + } + cert := &cmapi.Certificate{} + if err := c.Get(context.Background(), types.NamespacedName{Name: "db-gateway-cert", Namespace: testNamespace}, cert); err != nil { + t.Fatalf("expected certificate: %v", err) + } + svc := product.DocumentDBProfile().Naming.ServicePrefix + "db" + if cert.Spec.SecretName != "db-gateway-cert-tls" || cert.Spec.IssuerRef.Name != "db-gateway-selfsigned" || + !slices.Equal(cert.Spec.DNSNames, []string{svc, svc + ".default", svc + ".default.svc"}) { + t.Fatalf("unexpected certificate spec: %+v", cert.Spec) + } + if st := getTLSStatus(t, c); st.Ready || st.Message != "Creating self-signed certificate" || st.SecretName != "db-gateway-cert-tls" { + t.Fatalf("unexpected TLS status: %+v", st) + } + + // Not ready yet: waiting. + if res, err := r.Reconcile(context.Background(), certKey); err != nil || res.RequeueAfter != RequeueAfterShort { + t.Fatalf("expected requeue while waiting, got %v, %v", res, err) + } + if st := getTLSStatus(t, c); st.Ready || st.Message != "Waiting for gateway TLS certificate to become ready" { + t.Fatalf("unexpected TLS status: %+v", st) + } + + // Ready: status flips to ready and no requeue. + cert.Status.Conditions = []cmapi.CertificateCondition{{Type: cmapi.CertificateConditionReady, Status: cmmeta.ConditionTrue}} + if err := c.Update(context.Background(), cert); err != nil { + t.Fatal(err) + } + if res, err := r.Reconcile(context.Background(), certKey); err != nil || res != (ctrl.Result{}) { + t.Fatalf("expected no requeue when ready, got %v, %v", res, err) + } + if st := getTLSStatus(t, c); !st.Ready || st.Message != "Gateway TLS certificate ready" { + t.Fatalf("unexpected TLS status: %+v", st) + } + // Already ready: idempotent. + if res, err := r.Reconcile(context.Background(), certKey); err != nil || res != (ctrl.Result{}) { + t.Fatalf("expected idempotent ready reconcile, got %v, %v", res, err) + } + }) + } +} + +func TestCertificateProvidedSecret(t *testing.T) { + secret := func(data map[string][]byte) *corev1.Secret { + return &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "my-tls", Namespace: testNamespace}, Data: data} + } + provided := &dbpreview.GatewayTLS{Mode: "Provided", Provided: &dbpreview.ProvidedTLS{SecretName: "my-tls"}} + tests := []struct { + name string + gw *dbpreview.GatewayTLS + secret *corev1.Secret + wantRequeue bool + wantReady bool + wantMessage string + }{ + {name: "secret name missing", gw: &dbpreview.GatewayTLS{Mode: "Provided"}, wantMessage: "Provided TLS secret name missing"}, + {name: "secret absent", gw: provided, wantRequeue: true, wantMessage: "Waiting for provided TLS secret"}, + {name: "missing tls.crt", gw: provided, secret: secret(map[string][]byte{"tls.key": []byte("k")}), wantRequeue: true, wantMessage: "Provided secret missing tls.crt"}, + {name: "missing tls.key", gw: provided, secret: secret(map[string][]byte{"tls.crt": []byte("c")}), wantRequeue: true, wantMessage: "Provided secret missing tls.key"}, + {name: "complete secret", gw: provided, secret: secret(map[string][]byte{"tls.crt": []byte("c"), "tls.key": []byte("k")}), wantReady: true, wantMessage: "Using provided TLS secret"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := newTestScheme(t) + objs := []client.Object{clusterWithTLS(tt.gw)} + if tt.secret != nil { + objs = append(objs, tt.secret) + } + c := newFakeClient(t, s, nil, objs...) + res, err := newCertReconciler(c, s).Reconcile(context.Background(), certKey) + if err != nil { + t.Fatal(err) + } + if (res.RequeueAfter == RequeueAfterShort) != tt.wantRequeue { + t.Fatalf("unexpected requeue %v", res) + } + st := getTLSStatus(t, c) + if st.Ready != tt.wantReady || st.Message != tt.wantMessage { + t.Fatalf("unexpected TLS status: %+v", st) + } + if tt.gw.Provided != nil && st.SecretName != "my-tls" { + t.Fatalf("expected provided secret name in status, got %q", st.SecretName) + } + }) + } +} + +func TestCertificateCertManagerMode(t *testing.T) { + t.Run("configuration missing", func(t *testing.T) { + s := newTestScheme(t) + c := newFakeClient(t, s, nil, clusterWithTLS(&dbpreview.GatewayTLS{Mode: "CertManager"})) + if res, err := newCertReconciler(c, s).Reconcile(context.Background(), certKey); err != nil || res != (ctrl.Result{}) { + t.Fatalf("expected no requeue, got %v, %v", res, err) + } + if st := getTLSStatus(t, c); st.Ready || st.Message != "CertManager configuration missing" { + t.Fatalf("unexpected TLS status: %+v", st) + } + }) + + t.Run("defaults issuer kind, group and secret and merges DNS names", func(t *testing.T) { + s := newTestScheme(t) + gw := &dbpreview.GatewayTLS{Mode: "CertManager", CertManager: &dbpreview.CertManagerTLS{ + IssuerRef: dbpreview.IssuerRef{Name: "ca"}, + DNSNames: []string{"db.example.com", "", "db.example.com"}, + }} + c := newFakeClient(t, s, nil, clusterWithTLS(gw)) + r := newCertReconciler(c, s) + if res, err := r.Reconcile(context.Background(), certKey); err != nil || res.RequeueAfter != RequeueAfterShort { + t.Fatalf("expected requeue after creating, got %v, %v", res, err) + } + cert := &cmapi.Certificate{} + if err := c.Get(context.Background(), types.NamespacedName{Name: "db-gateway-cert", Namespace: testNamespace}, cert); err != nil { + t.Fatal(err) + } + svc := product.DocumentDBProfile().Naming.ServicePrefix + "db" + if cert.Spec.IssuerRef != (cmmeta.ObjectReference{Name: "ca", Kind: "Issuer", Group: "cert-manager.io"}) || + cert.Spec.SecretName != "db-gateway-cert-tls" || + !slices.Equal(cert.Spec.DNSNames, []string{"db.example.com", svc, svc + ".default", svc + ".default.svc"}) { + t.Fatalf("unexpected certificate spec: %+v", cert.Spec) + } + if st := getTLSStatus(t, c); st.Ready || st.Message != "Creating cert-manager certificate" { + t.Fatalf("unexpected TLS status: %+v", st) + } + if _, err := r.Reconcile(context.Background(), certKey); err != nil { + t.Fatal(err) + } + if st := getTLSStatus(t, c); st.Ready || st.Message != "Waiting for cert-manager certificate to become ready" { + t.Fatalf("unexpected TLS status: %+v", st) + } + }) + + t.Run("honors explicit issuer and reports readiness", func(t *testing.T) { + s := newTestScheme(t) + gw := &dbpreview.GatewayTLS{Mode: "CertManager", CertManager: &dbpreview.CertManagerTLS{ + IssuerRef: dbpreview.IssuerRef{Name: "ca", Kind: "ClusterIssuer", Group: "example.io"}, + SecretName: "custom-tls", + }} + c := newFakeClient(t, s, nil, clusterWithTLS(gw)) + r := newCertReconciler(c, s) + if _, err := r.Reconcile(context.Background(), certKey); err != nil { + t.Fatal(err) + } + cert := &cmapi.Certificate{} + if err := c.Get(context.Background(), types.NamespacedName{Name: "db-gateway-cert", Namespace: testNamespace}, cert); err != nil { + t.Fatal(err) + } + if cert.Spec.IssuerRef.Kind != "ClusterIssuer" || cert.Spec.IssuerRef.Group != "example.io" || cert.Spec.SecretName != "custom-tls" { + t.Fatalf("explicit issuer settings not honored: %+v", cert.Spec) + } + cert.Status.Conditions = []cmapi.CertificateCondition{{Type: cmapi.CertificateConditionReady, Status: cmmeta.ConditionTrue}} + if err := c.Update(context.Background(), cert); err != nil { + t.Fatal(err) + } + if res, err := r.Reconcile(context.Background(), certKey); err != nil || res != (ctrl.Result{}) { + t.Fatalf("expected ready without requeue, got %v, %v", res, err) + } + if st := getTLSStatus(t, c); !st.Ready || st.SecretName != "custom-tls" || st.Message != "Gateway TLS certificate ready (cert-manager)" { + t.Fatalf("unexpected TLS status: %+v", st) + } + }) +} + +func TestCertificateClientErrors(t *testing.T) { + isKind := func(obj client.Object, kind string) bool { + switch obj.(type) { + case *cmapi.Certificate: + return kind == "cert" + case *cmapi.Issuer: + return kind == "issuer" + case *corev1.Secret: + return kind == "secret" + } + return false + } + getErr := func(kind string) interceptor.Funcs { + return interceptor.Funcs{Get: func(ctx context.Context, c client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + if isKind(obj, kind) { + return errInjected + } + return c.Get(ctx, key, obj, opts...) + }} + } + createErr := func(kind string) interceptor.Funcs { + return interceptor.Funcs{Create: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.CreateOption) error { + if isKind(obj, kind) { + return errInjected + } + return c.Create(ctx, obj, opts...) + }} + } + statusErr := interceptor.Funcs{SubResourceUpdate: func(context.Context, client.Client, string, client.Object, ...client.SubResourceUpdateOption) error { + return errInjected + }} + certManager := &dbpreview.GatewayTLS{Mode: "CertManager", CertManager: &dbpreview.CertManagerTLS{IssuerRef: dbpreview.IssuerRef{Name: "ca"}}} + provided := &dbpreview.GatewayTLS{Mode: "Provided", Provided: &dbpreview.ProvidedTLS{SecretName: "my-tls"}} + initialized := &dbpreview.TLSStatus{} + + tests := []struct { + name string + gw *dbpreview.GatewayTLS + status *dbpreview.TLSStatus + extra []client.Object + funcs interceptor.Funcs + }{ + {name: "initial status update", funcs: statusErr}, + {name: "self-signed issuer get", funcs: getErr("issuer")}, + {name: "self-signed issuer create", funcs: createErr("issuer")}, + {name: "self-signed cert get", funcs: getErr("cert")}, + {name: "self-signed cert create", funcs: createErr("cert")}, + {name: "self-signed creating status", status: initialized, funcs: statusErr}, + {name: "self-signed waiting status", status: initialized, funcs: statusErr, extra: []client.Object{&cmapi.Certificate{ObjectMeta: metav1.ObjectMeta{Name: "db-gateway-cert", Namespace: testNamespace}}}}, + {name: "self-signed ready status", status: initialized, funcs: statusErr, extra: []client.Object{readyCert("db-gateway-cert", "s")}}, + {name: "cert-manager missing config status", gw: &dbpreview.GatewayTLS{Mode: "CertManager"}, status: initialized, funcs: statusErr}, + {name: "cert-manager cert get", gw: certManager, funcs: getErr("cert")}, + {name: "cert-manager cert create", gw: certManager, funcs: createErr("cert")}, + {name: "cert-manager creating status", gw: certManager, status: initialized, funcs: statusErr}, + {name: "cert-manager waiting status", gw: certManager, status: initialized, funcs: statusErr, extra: []client.Object{&cmapi.Certificate{ObjectMeta: metav1.ObjectMeta{Name: "db-gateway-cert", Namespace: testNamespace}}}}, + {name: "cert-manager ready status", gw: certManager, status: initialized, funcs: statusErr, extra: []client.Object{readyCert("db-gateway-cert", "s")}}, + {name: "provided missing name status", gw: &dbpreview.GatewayTLS{Mode: "Provided"}, status: initialized, funcs: statusErr}, + {name: "provided secret get", gw: provided, funcs: getErr("secret")}, + {name: "provided secret absent status", gw: provided, status: initialized, funcs: statusErr}, + {name: "provided missing crt status", gw: provided, status: initialized, funcs: statusErr, extra: []client.Object{&corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "my-tls", Namespace: testNamespace}}}}, + {name: "provided missing key status", gw: provided, status: initialized, funcs: statusErr, extra: []client.Object{&corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "my-tls", Namespace: testNamespace}, Data: map[string][]byte{"tls.crt": nil}}}}, + {name: "provided ready status", gw: provided, status: initialized, funcs: statusErr, extra: []client.Object{&corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "my-tls", Namespace: testNamespace}, Data: map[string][]byte{"tls.crt": nil, "tls.key": nil}}}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := newTestScheme(t) + db := clusterWithTLS(tt.gw) + db.Status.TLS = tt.status + c := newFakeClient(t, s, &tt.funcs, append(tt.extra, db)...) + res, err := newCertReconciler(c, s).Reconcile(context.Background(), certKey) + if err != errInjected { + t.Fatalf("expected injected error, got %v (result %v)", err, res) + } + }) + } +} + +func TestCertificateOwnerReferenceErrors(t *testing.T) { + for _, tt := range []struct { + name string + gw *dbpreview.GatewayTLS + extra []client.Object + }{ + {name: "self-signed issuer"}, + {name: "self-signed certificate", extra: []client.Object{&cmapi.Issuer{ObjectMeta: metav1.ObjectMeta{Name: "db-gateway-selfsigned", Namespace: testNamespace}}}}, + {name: "cert-manager certificate", gw: &dbpreview.GatewayTLS{Mode: "CertManager", CertManager: &dbpreview.CertManagerTLS{IssuerRef: dbpreview.IssuerRef{Name: "ca"}}}}, + } { + t.Run(tt.name, func(t *testing.T) { + s := newTestScheme(t) + c := newFakeClient(t, s, nil, append(tt.extra, clusterWithTLS(tt.gw))...) + // The reconciler scheme lacks the DocumentDB kind, so owner + // references cannot be set and nothing is created. + r := newCertReconciler(c, runtime.NewScheme()) + if _, err := r.Reconcile(context.Background(), certKey); err == nil { + t.Fatal("expected owner reference error") + } + err := c.Get(context.Background(), types.NamespacedName{Name: "db-gateway-cert", Namespace: testNamespace}, &cmapi.Certificate{}) + if !apierrors.IsNotFound(err) { + t.Fatalf("certificate must not be created without an owner, got %v", err) + } + }) + } +} diff --git a/operator/src/internal/engine/cluster_reconciler_test.go b/operator/src/internal/engine/cluster_reconciler_test.go new file mode 100644 index 00000000..abbd0e2d --- /dev/null +++ b/operator/src/internal/engine/cluster_reconciler_test.go @@ -0,0 +1,774 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package engine + +import ( + "context" + "fmt" + "slices" + "strings" + "sync" + "testing" + "time" + + cnpgv1 "github.com/cloudnative-pg/cloudnative-pg/api/v1" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/tools/record" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" + + dbpreview "github.com/documentdb/documentdb-operator/api/preview" + otelcfg "github.com/documentdb/documentdb-operator/internal/otel" + "github.com/documentdb/documentdb-operator/internal/product" + util "github.com/documentdb/documentdb-operator/internal/utils" +) + +type clusterReconciler = ClusterReconciler[*dbpreview.DocumentDB] + +var clusterKey = ctrl.Request{NamespacedName: types.NamespacedName{Name: "db", Namespace: testNamespace}} + +var finalizer = product.DocumentDBProfile().Naming.Finalizer() + +const extensionVersionsUpToDate = " default_version | installed_version\n" + + "-----------------+-------------------\n" + + " 0.110-0 | 0.110-0\n(1 row)" + +// sqlRecorder is a scripted SQLExecutor that records the statements it runs. +type sqlRecorder struct { + mu sync.Mutex + commands []string + roleOut string + roleErr error + grantErr error + extErr error +} + +func (s *sqlRecorder) exec(_ context.Context, _ *cnpgv1.Cluster, sql string) (string, error) { + s.mu.Lock() + defer s.mu.Unlock() + s.commands = append(s.commands, sql) + switch { + case strings.Contains(sql, "pg_available_extensions"): + return extensionVersionsUpToDate, s.extErr + case strings.HasPrefix(strings.TrimSpace(strings.ToUpper(sql)), "GRANT"): + return "", s.grantErr + case strings.Contains(sql, "documentdb_admin_role"): + return s.roleOut, s.roleErr + } + return "", nil +} + +func (s *sqlRecorder) ran(substr string) bool { + for _, c := range s.commands { + if strings.Contains(c, substr) { + return true + } + } + return false +} + +func newClusterReconciler(c client.Client, s *runtime.Scheme, sql *sqlRecorder) (*clusterReconciler, *record.FakeRecorder) { + rec := record.NewFakeRecorder(20) + return &clusterReconciler{ + Client: c, + Scheme: s, + Recorder: rec, + SQLExecutor: sql.exec, + Adapter: product.DocumentDBAdapter{}, + }, rec +} + +// reconcilableCluster returns a DocumentDB with the finalizer already present. +func reconcilableCluster() *dbpreview.DocumentDB { + db := testCluster() + db.Finalizers = []string{finalizer} + db.Spec.InstancesPerNode = 1 + db.Spec.Resource.Storage.PvcSize = "1Gi" + return db +} + +func gatewayService(ip string) *corev1.Service { + return &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{Name: product.DocumentDBProfile().Naming.ServiceName("db"), Namespace: testNamespace}, + Spec: corev1.ServiceSpec{ + Type: corev1.ServiceTypeClusterIP, + ClusterIP: ip, + Ports: []corev1.ServicePort{{Name: "gateway", Port: 10260}}, + }, + } +} + +// healthyCNPGCluster returns a healthy CNPG cluster that already matches the +// images the reconciler renders, so no rollout is needed. +func healthyCNPGCluster(t *testing.T, s *runtime.Scheme, db *dbpreview.DocumentDB) *cnpgv1.Cluster { + t.Helper() + seed := db.DeepCopy() + seed.Spec.ExposeViaService.ServiceType = "" + c := newFakeClient(t, s, nil, seed) + r, _ := newClusterReconciler(c, s, &sqlRecorder{}) + if _, err := r.Reconcile(context.Background(), clusterKey); err != nil { + t.Fatalf("seed reconcile: %v", err) + } + cluster := &cnpgv1.Cluster{} + if err := c.Get(context.Background(), clusterKey.NamespacedName, cluster); err != nil { + t.Fatalf("seed CNPG cluster: %v", err) + } + cluster.ResourceVersion = "" + cluster.Status = cnpgv1.ClusterStatus{ + Phase: CNPGClusterHealthyPhase, + CurrentPrimary: "db-1", + InstancesStatus: map[cnpgv1.PodStatus][]string{cnpgv1.PodHealthy: {"db-1"}}, + } + return cluster +} + +func getDocumentDB(t *testing.T, c client.Client) *dbpreview.DocumentDB { + t.Helper() + db := &dbpreview.DocumentDB{} + if err := c.Get(context.Background(), clusterKey.NamespacedName, db); err != nil { + t.Fatal(err) + } + return db +} + +func TestClusterReconcileMissingResourceCleansUpRBAC(t *testing.T) { + s := newTestScheme(t) + meta := metav1.ObjectMeta{Name: "db", Namespace: testNamespace} + c := newFakeClient(t, s, nil, &rbacv1.Role{ObjectMeta: meta}, &rbacv1.RoleBinding{ObjectMeta: meta}, &corev1.ServiceAccount{ObjectMeta: meta}) + r, _ := newClusterReconciler(c, s, &sqlRecorder{}) + r.Mutex = &sync.Mutex{} + + if res, err := r.Reconcile(context.Background(), clusterKey); err != nil || res != (ctrl.Result{}) { + t.Fatalf("expected clean no-op, got %v, %v", res, err) + } + for _, obj := range []client.Object{&rbacv1.Role{}, &rbacv1.RoleBinding{}, &corev1.ServiceAccount{}} { + if err := c.Get(context.Background(), clusterKey.NamespacedName, obj); !apierrors.IsNotFound(err) { + t.Fatalf("expected %T to be cleaned up, got %v", obj, err) + } + } +} + +func TestClusterCleanupToleratesDeleteFailures(t *testing.T) { + s := newTestScheme(t) + meta := metav1.ObjectMeta{Name: "db", Namespace: testNamespace} + c := newFakeClient(t, s, &interceptor.Funcs{Delete: func(context.Context, client.WithWatch, client.Object, ...client.DeleteOption) error { + return errInjected + }}, &rbacv1.Role{ObjectMeta: meta}, &rbacv1.RoleBinding{ObjectMeta: meta}, &corev1.ServiceAccount{ObjectMeta: meta}) + r, _ := newClusterReconciler(c, s, &sqlRecorder{}) + if err := r.Cleanup(context.Background(), clusterKey); err != nil { + t.Fatalf("cleanup must continue past delete failures, got %v", err) + } +} + +func TestClusterReconcileGetError(t *testing.T) { + s := newTestScheme(t) + c := newFakeClient(t, s, &interceptor.Funcs{Get: func(context.Context, client.WithWatch, client.ObjectKey, client.Object, ...client.GetOption) error { + return errInjected + }}) + r, _ := newClusterReconciler(c, s, &sqlRecorder{}) + if _, err := r.Reconcile(context.Background(), clusterKey); err != errInjected { + t.Fatalf("expected injected error, got %v", err) + } +} + +func TestClusterReconcileFinalizerLifecycle(t *testing.T) { + t.Run("adds finalizer and requeues", func(t *testing.T) { + s := newTestScheme(t) + c := newFakeClient(t, s, nil, testCluster()) + r, _ := newClusterReconciler(c, s, &sqlRecorder{}) + res, err := r.Reconcile(context.Background(), clusterKey) + if err != nil || !res.Requeue { + t.Fatalf("expected requeue after adding finalizer, got %v, %v", res, err) + } + if got := getDocumentDB(t, c); !slices.Contains(got.Finalizers, finalizer) { + t.Fatalf("expected finalizer %q, got %v", finalizer, got.Finalizers) + } + }) + + t.Run("add finalizer update fails", func(t *testing.T) { + s := newTestScheme(t) + c := newFakeClient(t, s, &interceptor.Funcs{Update: func(context.Context, client.WithWatch, client.Object, ...client.UpdateOption) error { + return errInjected + }}, testCluster()) + r, _ := newClusterReconciler(c, s, &sqlRecorder{}) + if _, err := r.Reconcile(context.Background(), clusterKey); err != errInjected { + t.Fatalf("expected injected error, got %v", err) + } + }) + + deleting := func(finalizers ...string) *dbpreview.DocumentDB { + db := testCluster() + now := metav1.Now() + db.DeletionTimestamp = &now + db.Finalizers = finalizers + return db + } + retainedPV := func(name string) *corev1.PersistentVolume { + naming := product.DocumentDBProfile().Naming + return &corev1.PersistentVolume{ObjectMeta: metav1.ObjectMeta{Name: name, Labels: map[string]string{ + naming.ClusterLabel(): "db", + naming.NamespaceLabel(): testNamespace, + }}} + } + + t.Run("deletion warns about retained PVs and removes finalizer", func(t *testing.T) { + s := newTestScheme(t) + c := newFakeClient(t, s, nil, deleting(finalizer, "other/keep"), retainedPV("pv-a"), retainedPV("pv-b")) + r, rec := newClusterReconciler(c, s, &sqlRecorder{}) + if res, err := r.Reconcile(context.Background(), clusterKey); err != nil || res != (ctrl.Result{}) { + t.Fatalf("expected deletion to complete, got %v, %v", res, err) + } + events := drainEvents(rec) + if !hasEvent(events, "PVsRetained") || !hasEvent(events, "kubectl delete pv pv-a pv-b") { + t.Fatalf("expected PV retention warning listing PVs, got %v", events) + } + if got := getDocumentDB(t, c); slices.Contains(got.Finalizers, finalizer) { + t.Fatalf("expected finalizer removed, got %v", got.Finalizers) + } + }) + + t.Run("deletion without our finalizer is a no-op", func(t *testing.T) { + s := newTestScheme(t) + c := newFakeClient(t, s, nil, deleting("other/keep")) + r, rec := newClusterReconciler(c, s, &sqlRecorder{}) + if res, err := r.Reconcile(context.Background(), clusterKey); err != nil || res != (ctrl.Result{}) { + t.Fatalf("expected no-op, got %v, %v", res, err) + } + if events := drainEvents(rec); len(events) != 0 { + t.Fatalf("expected no events, got %v", events) + } + }) + + t.Run("deletion continues when listing PVs fails", func(t *testing.T) { + s := newTestScheme(t) + c := newFakeClient(t, s, &interceptor.Funcs{List: func(context.Context, client.WithWatch, client.ObjectList, ...client.ListOption) error { + return errInjected + }}, deleting(finalizer, "other/keep")) + r, _ := newClusterReconciler(c, s, &sqlRecorder{}) + if _, err := r.Reconcile(context.Background(), clusterKey); err != nil { + t.Fatalf("PV warning failure must not block deletion, got %v", err) + } + if got := getDocumentDB(t, c); slices.Contains(got.Finalizers, finalizer) { + t.Fatal("expected finalizer removed") + } + }) + + t.Run("remove finalizer update fails", func(t *testing.T) { + s := newTestScheme(t) + c := newFakeClient(t, s, &interceptor.Funcs{Update: func(context.Context, client.WithWatch, client.Object, ...client.UpdateOption) error { + return errInjected + }}, deleting(finalizer)) + r, _ := newClusterReconciler(c, s, &sqlRecorder{}) + r.Recorder = nil + if _, err := r.Reconcile(context.Background(), clusterKey); err != errInjected { + t.Fatalf("expected injected error, got %v", err) + } + }) +} + +func TestClusterEmitPVRetentionWarningWithoutPVs(t *testing.T) { + s := newTestScheme(t) + c := newFakeClient(t, s, nil) + r, rec := newClusterReconciler(c, s, &sqlRecorder{}) + if err := r.EmitPVRetentionWarning(context.Background(), testCluster()); err != nil { + t.Fatal(err) + } + if events := drainEvents(rec); len(events) != 0 { + t.Fatalf("expected no warning without PVs, got %v", events) + } +} + +func TestClusterReconcileReplicationContext(t *testing.T) { + t.Run("context error", func(t *testing.T) { + s := newTestScheme(t) + c := newFakeClient(t, s, nil, reconcilableCluster()) + r, _ := newClusterReconciler(c, s, &sqlRecorder{}) + r.Replication = &fakeReplication[*dbpreview.DocumentDB]{contextErr: errInjected} + if _, err := r.Reconcile(context.Background(), clusterKey); err != errInjected { + t.Fatalf("expected injected error, got %v", err) + } + }) + + t.Run("not present deletes owned resources", func(t *testing.T) { + s := newTestScheme(t) + db := reconcilableCluster() + owned := metav1.OwnerReference{APIVersion: "documentdb.io/preview", Kind: "DocumentDB", Name: "db", UID: db.UID} + svc := gatewayService("10.0.0.1") + svc.OwnerReferences = []metav1.OwnerReference{owned} + cnpgCluster := &cnpgv1.Cluster{ObjectMeta: metav1.ObjectMeta{Name: "db", Namespace: testNamespace, OwnerReferences: []metav1.OwnerReference{owned}}} + role := &rbacv1.Role{ObjectMeta: metav1.ObjectMeta{Name: "db", Namespace: testNamespace}} + c := newFakeClient(t, s, nil, db, svc, cnpgCluster, role) + r, _ := newClusterReconciler(c, s, &sqlRecorder{}) + r.Replication = &fakeReplication[*dbpreview.DocumentDB]{rc: util.NewReplicationContext(util.NotPresent, "", "")} + if res, err := r.Reconcile(context.Background(), clusterKey); err != nil || res != (ctrl.Result{}) { + t.Fatalf("expected no-op result, got %v, %v", res, err) + } + for _, obj := range []client.Object{&corev1.Service{}, &cnpgv1.Cluster{}, &rbacv1.Role{}} { + key := types.NamespacedName{Name: "db", Namespace: testNamespace} + if _, ok := obj.(*corev1.Service); ok { + key.Name = svc.Name + } + if err := c.Get(context.Background(), key, obj); !apierrors.IsNotFound(err) { + t.Fatalf("expected %T deleted, got %v", obj, err) + } + } + }) + + t.Run("not present propagates owned resource deletion failure", func(t *testing.T) { + s := newTestScheme(t) + c := newFakeClient(t, s, &interceptor.Funcs{List: func(context.Context, client.WithWatch, client.ObjectList, ...client.ListOption) error { + return errInjected + }}, reconcilableCluster()) + r, _ := newClusterReconciler(c, s, &sqlRecorder{}) + r.Replication = &fakeReplication[*dbpreview.DocumentDB]{rc: util.NewReplicationContext(util.NotPresent, "", "")} + if _, err := r.Reconcile(context.Background(), clusterKey); err == nil { + t.Fatal("expected error") + } + }) +} + +func TestClusterReconcileCreatesResourcesThenConverges(t *testing.T) { + s := newTestScheme(t) + db := reconcilableCluster() + db.Spec.ExposeViaService.ServiceType = "ClusterIP" + db.Spec.Monitoring = &dbpreview.MonitoringSpec{Enabled: true} + c := newFakeClient(t, s, nil, db, gatewayService("10.0.0.7")) + sql := &sqlRecorder{roleOut: "(0 rows)"} + r, _ := newClusterReconciler(c, s, sql) + + res, err := r.Reconcile(context.Background(), clusterKey) + if err != nil || res.RequeueAfter != RequeueAfterLong { + t.Fatalf("expected long requeue after creating the CNPG cluster, got %v, %v", res, err) + } + for _, obj := range []client.Object{&rbacv1.Role{}, &rbacv1.RoleBinding{}, &corev1.ServiceAccount{}, &cnpgv1.Cluster{}} { + if err := c.Get(context.Background(), clusterKey.NamespacedName, obj); err != nil { + t.Fatalf("expected %T to be created: %v", obj, err) + } + } + cmKey := types.NamespacedName{Name: otelcfg.ConfigMapName("db"), Namespace: testNamespace} + if err := c.Get(context.Background(), cmKey, &corev1.ConfigMap{}); err != nil { + t.Fatalf("expected OTel ConfigMap while monitoring is enabled: %v", err) + } + + // The CNPG cluster becomes healthy. + cluster := &cnpgv1.Cluster{} + if err := c.Get(context.Background(), clusterKey.NamespacedName, cluster); err != nil { + t.Fatal(err) + } + cluster.Status = cnpgv1.ClusterStatus{ + Phase: CNPGClusterHealthyPhase, + CurrentPrimary: "db-1", + InstancesStatus: map[cnpgv1.PodStatus][]string{cnpgv1.PodHealthy: {"db-1"}}, + } + if err := c.Update(context.Background(), cluster); err != nil { + t.Fatal(err) + } + + // Disable monitoring: the ConfigMap is removed after the cluster sync. + got := getDocumentDB(t, c) + got.Spec.Monitoring = nil + if err := c.Update(context.Background(), got); err != nil { + t.Fatal(err) + } + + res, err = r.Reconcile(context.Background(), clusterKey) + if err != nil || res != (ctrl.Result{}) { + t.Fatalf("expected converged reconcile, got %v, %v", res, err) + } + if !sql.ran("GRANT") { + t.Fatalf("expected replication role grant, ran %v", sql.commands) + } + if err := c.Get(context.Background(), cmKey, &corev1.ConfigMap{}); !apierrors.IsNotFound(err) { + t.Fatalf("expected OTel ConfigMap deleted after disabling monitoring, got %v", err) + } + got = getDocumentDB(t, c) + if got.Status.Status != CNPGClusterHealthyPhase || !strings.Contains(got.Status.ConnectionString, "10.0.0.7") || + got.Status.SchemaVersion != "0.110.0" { + t.Fatalf("expected status with phase, connection string and schema version, got %+v", got.Status) + } +} + +func TestClusterReconcileLoadBalancerServiceWithReplicationHooks(t *testing.T) { + s := newTestScheme(t) + db := reconcilableCluster() + db.Spec.ExposeViaService.ServiceType = "LoadBalancer" + svc := gatewayService("") + svc.Spec.Type = corev1.ServiceTypeLoadBalancer + svc.Status.LoadBalancer.Ingress = []corev1.LoadBalancerIngress{{Hostname: "db.example.com"}} + cluster := healthyCNPGCluster(t, s, db) + c := newFakeClient(t, s, nil, db, svc, cluster) + sql := &sqlRecorder{roleOut: "(1 row)"} + r, _ := newClusterReconciler(c, s, sql) + hooks := &fakeReplication[*dbpreview.DocumentDB]{rc: util.NewReplicationContext(util.Primary, "db", "db")} + hooks.rc.CNPGClusterName = "db" + r.Replication = hooks + + if res, err := r.Reconcile(context.Background(), clusterKey); err != nil || res != (ctrl.Result{}) { + t.Fatalf("expected converged reconcile, got %v, %v", res, err) + } + if !hooks.addCalled || !hooks.syncCalled { + t.Fatal("expected replication hooks to extend and sync the cluster") + } + if sql.ran("GRANT") { + t.Fatal("role grant must be skipped when already granted") + } + updated := &corev1.Service{} + if err := c.Get(context.Background(), types.NamespacedName{Name: svc.Name, Namespace: testNamespace}, updated); err != nil { + t.Fatal(err) + } + if updated.Spec.Type != corev1.ServiceTypeLoadBalancer { + t.Fatalf("expected LoadBalancer service, got %v", updated.Spec.Type) + } + if got := getDocumentDB(t, c); !strings.Contains(got.Status.ConnectionString, "db.example.com") { + t.Fatalf("expected connection string through the load balancer hostname, got %q", got.Status.ConnectionString) + } +} + +func TestClusterReconcileReplicationHookResults(t *testing.T) { + stop := &ctrl.Result{RequeueAfter: 42 * time.Second} + tests := []struct { + name string + hooks *fakeReplication[*dbpreview.DocumentDB] + want ctrl.Result + }{ + {name: "add to spec fails", hooks: &fakeReplication[*dbpreview.DocumentDB]{addErr: errInjected}, want: ctrl.Result{RequeueAfter: RequeueAfterShort}}, + {name: "sync requests a wait", hooks: &fakeReplication[*dbpreview.DocumentDB]{syncErr: errInjected, syncRequeue: 7 * time.Second}, want: ctrl.Result{RequeueAfter: 7 * time.Second}}, + {name: "primary reconciliation stops", hooks: &fakeReplication[*dbpreview.DocumentDB]{primaryResult: stop}, want: *stop}, + {name: "networking reconciliation stops", hooks: &fakeReplication[*dbpreview.DocumentDB]{netResult: stop}, want: *stop}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := newTestScheme(t) + db := reconcilableCluster() + c := newFakeClient(t, s, nil, db, healthyCNPGCluster(t, s, db)) + r, _ := newClusterReconciler(c, s, &sqlRecorder{roleOut: "(1 row)"}) + tt.hooks.rc = util.NewReplicationContext(util.Primary, "db", "db") + tt.hooks.rc.CNPGClusterName = "db" + r.Replication = tt.hooks + res, err := r.Reconcile(context.Background(), clusterKey) + if err != nil || res != tt.want { + t.Fatalf("expected %v, got %v, %v", tt.want, res, err) + } + }) + } +} + +func TestClusterReconcileStageFailuresRequeue(t *testing.T) { + isType := func(obj client.Object, want client.Object) bool { return typeName(obj) == typeName(want) } + failGet := func(want client.Object) interceptor.Funcs { + return interceptor.Funcs{Get: func(ctx context.Context, c client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + if isType(obj, want) { + return errInjected + } + return c.Get(ctx, key, obj, opts...) + }} + } + failCreate := func(want client.Object) interceptor.Funcs { + return interceptor.Funcs{Create: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.CreateOption) error { + if isType(obj, want) { + return errInjected + } + return c.Create(ctx, obj, opts...) + }} + } + + tests := []struct { + name string + service bool + serviceIP string + monitoring bool + healthy bool + sql *sqlRecorder + funcs interceptor.Funcs + want ctrl.Result + wantSQLRun string + wantNoCNPG bool + staleImage bool + }{ + {name: "service upsert fails", service: true, serviceIP: "10.0.0.1", funcs: failGet(&corev1.Service{}), want: ctrl.Result{RequeueAfter: RequeueAfterShort}, wantNoCNPG: true}, + {name: "service IP not assigned pauses", service: true, want: ctrl.Result{}, wantNoCNPG: true}, + {name: "RBAC fails", funcs: failCreate(&rbacv1.Role{}), want: ctrl.Result{RequeueAfter: RequeueAfterShort}, wantNoCNPG: true}, + {name: "OTel ConfigMap fails", monitoring: true, funcs: failCreate(&corev1.ConfigMap{}), want: ctrl.Result{RequeueAfter: RequeueAfterShort}, wantNoCNPG: true}, + {name: "CNPG cluster create fails", funcs: failCreate(&cnpgv1.Cluster{}), want: ctrl.Result{RequeueAfter: RequeueAfterShort}, wantNoCNPG: true}, + {name: "CNPG cluster get fails", funcs: failGet(&cnpgv1.Cluster{}), want: ctrl.Result{RequeueAfter: RequeueAfterShort}, wantNoCNPG: true}, + { + name: "CNPG cluster sync fails", healthy: true, staleImage: true, + funcs: interceptor.Funcs{Patch: func(context.Context, client.WithWatch, client.Object, client.Patch, ...client.PatchOption) error { + return errInjected + }}, + want: ctrl.Result{RequeueAfter: RequeueAfterShort}, + }, + { + name: "OTel ConfigMap cleanup fails", healthy: true, + funcs: interceptor.Funcs{Delete: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.DeleteOption) error { + if _, ok := obj.(*corev1.ConfigMap); ok { + return errInjected + } + return c.Delete(ctx, obj, opts...) + }}, + want: ctrl.Result{RequeueAfter: RequeueAfterShort}, + }, + {name: "role check fails", healthy: true, sql: &sqlRecorder{roleErr: errInjected}, want: ctrl.Result{RequeueAfter: RequeueAfterLong}}, + {name: "role grant fails", healthy: true, sql: &sqlRecorder{roleOut: "(0 rows)", grantErr: errInjected}, want: ctrl.Result{RequeueAfter: RequeueAfterShort}, wantSQLRun: "GRANT"}, + {name: "extension upgrade check fails", healthy: true, sql: &sqlRecorder{roleOut: "(1 row)", extErr: errInjected}, want: ctrl.Result{RequeueAfter: RequeueAfterShort}}, + { + name: "status update failure is tolerated", healthy: true, sql: &sqlRecorder{roleOut: "(1 row)"}, + funcs: interceptor.Funcs{SubResourceUpdate: func(context.Context, client.Client, string, client.Object, ...client.SubResourceUpdateOption) error { + return errInjected + }}, + want: ctrl.Result{RequeueAfter: RequeueAfterShort}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := newTestScheme(t) + db := reconcilableCluster() + if tt.monitoring { + db.Spec.Monitoring = &dbpreview.MonitoringSpec{Enabled: true} + } + var objs []client.Object + if tt.healthy { + cluster := healthyCNPGCluster(t, s, db) + if tt.staleImage { + cluster.Spec.Plugins[0].Parameters[util.PLUGIN_PARAM_GATEWAY_IMAGE] = "example.com/gateway:old" + } + objs = append(objs, cluster) + } + if tt.service { + db.Spec.ExposeViaService.ServiceType = "ClusterIP" + objs = append(objs, gatewayService(tt.serviceIP)) + } + objs = append(objs, db) + c := newFakeClient(t, s, &tt.funcs, objs...) + sql := tt.sql + if sql == nil { + sql = &sqlRecorder{roleOut: "(1 row)"} + } + r, _ := newClusterReconciler(c, s, sql) + res, err := r.Reconcile(context.Background(), clusterKey) + if err != nil || res != tt.want { + t.Fatalf("expected %v, got %v, %v", tt.want, res, err) + } + if tt.wantSQLRun != "" && !sql.ran(tt.wantSQLRun) { + t.Fatalf("expected %q to run, ran %v", tt.wantSQLRun, sql.commands) + } + if tt.wantNoCNPG { + clusters := &cnpgv1.ClusterList{} + if err := c.List(context.Background(), clusters); err != nil || len(clusters.Items) != 0 { + t.Fatalf("expected no CNPG cluster after an earlier stage failed, got %d (%v)", len(clusters.Items), err) + } + } + }) + } +} + +func TestClusterReconcilePVRecovery(t *testing.T) { + recoveringCluster := func() *dbpreview.DocumentDB { + db := reconcilableCluster() + db.Spec.Bootstrap = &dbpreview.BootstrapConfiguration{Recovery: &dbpreview.RecoveryConfiguration{ + PersistentVolume: &dbpreview.PVRecoveryConfiguration{Name: "old-pv"}, + }} + return db + } + pv := func(phase corev1.PersistentVolumePhase, claimed bool) *corev1.PersistentVolume { + p := &corev1.PersistentVolume{ + ObjectMeta: metav1.ObjectMeta{Name: "old-pv"}, + Spec: corev1.PersistentVolumeSpec{ + Capacity: corev1.ResourceList{corev1.ResourceStorage: resource.MustParse("1Gi")}, + AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, + }, + Status: corev1.PersistentVolumeStatus{Phase: phase}, + } + if claimed { + p.Spec.ClaimRef = &corev1.ObjectReference{Name: "old-claim", Namespace: testNamespace} + } + return p + } + tempPVCKey := types.NamespacedName{Name: util.TempPVCNameForPVRecovery("db"), Namespace: testNamespace} + + t.Run("reconcile creates temp PVC and requeues", func(t *testing.T) { + s := newTestScheme(t) + c := newFakeClient(t, s, nil, recoveringCluster(), pv(corev1.VolumeAvailable, false)) + r, _ := newClusterReconciler(c, s, &sqlRecorder{}) + res, err := r.Reconcile(context.Background(), clusterKey) + if err != nil || res.RequeueAfter != RequeueAfterShort { + t.Fatalf("expected short requeue, got %v, %v", res, err) + } + pvc := &corev1.PersistentVolumeClaim{} + if err := c.Get(context.Background(), tempPVCKey, pvc); err != nil { + t.Fatalf("expected temp PVC: %v", err) + } + if pvc.Spec.VolumeName != "old-pv" || metav1.GetControllerOf(pvc) == nil { + t.Fatalf("expected owned temp PVC bound to the PV, got %+v", pvc) + } + if err := c.Get(context.Background(), clusterKey.NamespacedName, &cnpgv1.Cluster{}); !apierrors.IsNotFound(err) { + t.Fatal("CNPG cluster must wait for the temp PVC to bind") + } + }) + + t.Run("reconcile surfaces missing PV", func(t *testing.T) { + s := newTestScheme(t) + c := newFakeClient(t, s, nil, recoveringCluster()) + r, _ := newClusterReconciler(c, s, &sqlRecorder{}) + if _, err := r.Reconcile(context.Background(), clusterKey); err == nil || !strings.Contains(err.Error(), "old-pv not found") { + t.Fatalf("expected missing PV error, got %v", err) + } + }) + + errorCases := []struct { + name string + objs []client.Object + funcs interceptor.Funcs + scheme *runtime.Scheme + wantErr string + }{ + { + name: "deleting temp PVC after recovery fails", + objs: []client.Object{ + &cnpgv1.Cluster{ObjectMeta: metav1.ObjectMeta{Name: "db", Namespace: testNamespace}, Status: cnpgv1.ClusterStatus{Phase: CNPGClusterHealthyPhase}}, + &corev1.PersistentVolumeClaim{ObjectMeta: metav1.ObjectMeta{Name: tempPVCKey.Name, Namespace: testNamespace}}, + }, + funcs: interceptor.Funcs{Delete: func(context.Context, client.WithWatch, client.Object, ...client.DeleteOption) error { + return errInjected + }}, + wantErr: "failed to delete temp PVC", + }, + { + name: "getting CNPG cluster fails", + funcs: interceptor.Funcs{Get: func(ctx context.Context, c client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + if _, ok := obj.(*cnpgv1.Cluster); ok { + return errInjected + } + return c.Get(ctx, key, obj, opts...) + }}, + wantErr: "failed to get CNPG cluster", + }, + { + name: "getting temp PVC fails", + funcs: interceptor.Funcs{Get: func(ctx context.Context, c client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + if _, ok := obj.(*corev1.PersistentVolumeClaim); ok { + return errInjected + } + return c.Get(ctx, key, obj, opts...) + }}, + wantErr: "failed to get temp PVC", + }, + { + name: "getting PV fails", + funcs: interceptor.Funcs{Get: func(ctx context.Context, c client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + if _, ok := obj.(*corev1.PersistentVolume); ok { + return errInjected + } + return c.Get(ctx, key, obj, opts...) + }}, + wantErr: "failed to get PV old-pv", + }, + { + name: "PV is bound elsewhere", + objs: []client.Object{pv(corev1.VolumeBound, true)}, + wantErr: "must be Available or Released", + }, + { + name: "clearing claimRef fails", + objs: []client.Object{pv(corev1.VolumeReleased, true)}, + funcs: interceptor.Funcs{Update: func(context.Context, client.WithWatch, client.Object, ...client.UpdateOption) error { + return errInjected + }}, + wantErr: "failed to clear claimRef", + }, + { + name: "temp PVC owner reference fails", + objs: []client.Object{pv(corev1.VolumeAvailable, false)}, + scheme: runtime.NewScheme(), + wantErr: "failed to set owner reference on temp PVC", + }, + { + name: "creating temp PVC fails", + objs: []client.Object{pv(corev1.VolumeAvailable, false)}, + funcs: interceptor.Funcs{Create: func(context.Context, client.WithWatch, client.Object, ...client.CreateOption) error { + return errInjected + }}, + wantErr: "failed to create temp PVC", + }, + } + for _, tt := range errorCases { + t.Run(tt.name, func(t *testing.T) { + s := newTestScheme(t) + db := recoveringCluster() + c := newFakeClient(t, s, &tt.funcs, append(tt.objs, db)...) + r, _ := newClusterReconciler(c, s, &sqlRecorder{}) + if tt.scheme != nil { + r.Scheme = tt.scheme + } + _, err := r.ReconcilePVRecovery(context.Background(), db, testNamespace, "db") + if err == nil || !strings.Contains(err.Error(), tt.wantErr) { + t.Fatalf("expected error containing %q, got %v", tt.wantErr, err) + } + }) + } + + t.Run("released PV claimRef is cleared before binding", func(t *testing.T) { + s := newTestScheme(t) + db := recoveringCluster() + c := newFakeClient(t, s, nil, db, pv(corev1.VolumeReleased, true)) + r, _ := newClusterReconciler(c, s, &sqlRecorder{}) + res, err := r.ReconcilePVRecovery(context.Background(), db, testNamespace, "db") + if err != nil || res.RequeueAfter != RequeueAfterShort { + t.Fatalf("expected requeue after clearing claimRef, got %v, %v", res, err) + } + got := &corev1.PersistentVolume{} + if err := c.Get(context.Background(), types.NamespacedName{Name: "old-pv"}, got); err != nil { + t.Fatal(err) + } + if got.Spec.ClaimRef != nil { + t.Fatalf("expected claimRef cleared, got %v", got.Spec.ClaimRef) + } + }) +} + +func TestClusterEnsureRBACFailures(t *testing.T) { + for _, target := range []client.Object{&rbacv1.Role{}, &corev1.ServiceAccount{}, &rbacv1.RoleBinding{}} { + t.Run(typeName(target), func(t *testing.T) { + s := newTestScheme(t) + c := newFakeClient(t, s, &interceptor.Funcs{Create: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.CreateOption) error { + if typeName(obj) == typeName(target) { + return errInjected + } + return c.Create(ctx, obj, opts...) + }}) + r, _ := newClusterReconciler(c, s, &sqlRecorder{}) + if err := r.EnsureRBAC(context.Background(), testCluster(), testNamespace); err != errInjected { + t.Fatalf("expected injected error, got %v", err) + } + }) + } +} + +func TestClusterOtelConfigMapErrors(t *testing.T) { + s := newTestScheme(t) + r, _ := newClusterReconciler(newFakeClient(t, s, nil), runtime.NewScheme(), &sqlRecorder{}) + if err := r.ReconcileOtelConfigMap(context.Background(), testCluster(), testNamespace); err == nil || + !strings.Contains(err.Error(), "failed to set owner reference") { + t.Fatalf("expected owner reference error, got %v", err) + } + if err := r.DeleteOtelConfigMap(context.Background(), "db", testNamespace); err != nil { + t.Fatalf("deleting an absent ConfigMap must succeed, got %v", err) + } +} + +func TestClusterSetupWithManagerRequiresSQLExecutor(t *testing.T) { + r := &clusterReconciler{Adapter: product.DocumentDBAdapter{}} + if err := r.SetupWithManager(nil, "documentdb"); err == nil || !strings.Contains(err.Error(), "SQLExecutor must be configured") { + t.Fatalf("expected SQLExecutor configuration error, got %v", err) + } +} + +func typeName(obj client.Object) string { return fmt.Sprintf("%T", obj) } diff --git a/operator/src/internal/engine/helpers_test.go b/operator/src/internal/engine/helpers_test.go new file mode 100644 index 00000000..27a4de73 --- /dev/null +++ b/operator/src/internal/engine/helpers_test.go @@ -0,0 +1,124 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package engine + +import ( + "context" + "errors" + "strings" + "testing" + "time" + + cmapi "github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1" + cnpgv1 "github.com/cloudnative-pg/cloudnative-pg/api/v1" + snapshotv1 "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" + "k8s.io/apimachinery/pkg/runtime" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/tools/record" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" + + dbpreview "github.com/documentdb/documentdb-operator/api/preview" + cnpg "github.com/documentdb/documentdb-operator/internal/cnpg" + util "github.com/documentdb/documentdb-operator/internal/utils" +) + +const testNamespace = "default" + +var errInjected = errors.New("injected failure") + +func newTestScheme(t *testing.T) *runtime.Scheme { + t.Helper() + s := runtime.NewScheme() + for _, add := range []func(*runtime.Scheme) error{ + clientgoscheme.AddToScheme, + dbpreview.AddToScheme, + cnpgv1.AddToScheme, + snapshotv1.AddToScheme, + cmapi.AddToScheme, + } { + if err := add(s); err != nil { + t.Fatalf("add to scheme: %v", err) + } + } + return s +} + +// newFakeClient builds a fake client with status subresources and the Backup +// cluster index registered, plus optional interceptors. +func newFakeClient(t *testing.T, s *runtime.Scheme, funcs *interceptor.Funcs, objs ...client.Object) client.Client { + t.Helper() + b := fake.NewClientBuilder(). + WithScheme(s). + WithObjects(objs...). + WithStatusSubresource(&dbpreview.DocumentDB{}, &dbpreview.Backup{}, &dbpreview.ScheduledBackup{}). + WithIndex(&dbpreview.Backup{}, BackupClusterField, func(o client.Object) []string { + return []string{o.(*dbpreview.Backup).Spec.Cluster.Name} + }) + if funcs != nil { + b = b.WithInterceptorFuncs(*funcs) + } + return b.Build() +} + +// drainEvents returns all events currently buffered in a fake recorder. +func drainEvents(rec *record.FakeRecorder) []string { + var events []string + for { + select { + case e := <-rec.Events: + events = append(events, e) + default: + return events + } + } +} + +func hasEvent(events []string, substr string) bool { + for _, e := range events { + if strings.Contains(e, substr) { + return true + } + } + return false +} + +// fakeReplication is a configurable ReplicationHooks implementation. +type fakeReplication[T client.Object] struct { + rc *util.ReplicationContext + contextErr error + addErr error + syncOps []cnpg.JSONPatch + syncErr error + syncRequeue time.Duration + primaryResult *ctrl.Result + netResult *ctrl.Result + + addCalled bool + syncCalled bool +} + +func (f *fakeReplication[T]) Context(context.Context, T) (*util.ReplicationContext, error) { + return f.rc, f.contextErr +} + +func (f *fakeReplication[T]) AddToClusterSpec(context.Context, T, *util.ReplicationContext, *cnpgv1.Cluster) error { + f.addCalled = true + return f.addErr +} + +func (f *fakeReplication[T]) SyncChanges(context.Context, *cnpgv1.Cluster, *cnpgv1.Cluster, T, *util.ReplicationContext) ([]cnpg.JSONPatch, error, time.Duration) { + f.syncCalled = true + return f.syncOps, f.syncErr, f.syncRequeue +} + +func (f *fakeReplication[T]) ReconcilePrimary(context.Context, T, *util.ReplicationContext, *cnpgv1.Cluster) *ctrl.Result { + return f.primaryResult +} + +func (f *fakeReplication[T]) ReconcileNetworking(context.Context, T, *util.ReplicationContext) *ctrl.Result { + return f.netResult +} diff --git a/operator/src/internal/engine/predicates_sql_test.go b/operator/src/internal/engine/predicates_sql_test.go new file mode 100644 index 00000000..c194b62c --- /dev/null +++ b/operator/src/internal/engine/predicates_sql_test.go @@ -0,0 +1,112 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package engine + +import ( + "context" + "net/http" + "net/http/httptest" + "strings" + "testing" + + cnpgv1 "github.com/cloudnative-pg/cloudnative-pg/api/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/event" +) + +func TestClusterInstanceStatusChangedPredicate(t *testing.T) { + p := ClusterInstanceStatusChangedPredicate() + cluster := func(phase string, healthy ...string) *cnpgv1.Cluster { + return &cnpgv1.Cluster{Status: cnpgv1.ClusterStatus{ + Phase: phase, + InstancesStatus: map[cnpgv1.PodStatus][]string{cnpgv1.PodHealthy: healthy}, + }} + } + tests := []struct { + name string + old, new client.Object + want bool + }{ + {name: "unchanged", old: cluster("ok", "a"), new: cluster("ok", "a"), want: false}, + {name: "healthy instances changed", old: cluster("ok", "a"), new: cluster("ok", "a", "b"), want: true}, + {name: "phase changed", old: cluster("ok", "a"), new: cluster("failover", "a"), want: true}, + {name: "old is not a CNPG cluster", old: &corev1.Pod{}, new: cluster("ok"), want: true}, + {name: "new is not a CNPG cluster", old: cluster("ok"), new: &corev1.Pod{}, want: true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + e := event.UpdateEvent{ObjectOld: tt.old, ObjectNew: tt.new} + if got := p.Update(e); got != tt.want { + t.Fatalf("Update() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestServicePrefixPredicate(t *testing.T) { + p := ServicePrefixPredicate("documentdb-service-") + match := &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: "documentdb-service-db"}} + other := &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: "kubernetes"}} + for _, tc := range []struct { + svc *corev1.Service + want bool + }{{match, true}, {other, false}} { + if got := p.Create(event.CreateEvent{Object: tc.svc}); got != tc.want { + t.Errorf("Create(%s) = %v", tc.svc.Name, got) + } + if got := p.Update(event.UpdateEvent{ObjectOld: other, ObjectNew: tc.svc}); got != tc.want { + t.Errorf("Update(%s) = %v", tc.svc.Name, got) + } + if got := p.Delete(event.DeleteEvent{Object: tc.svc}); got != tc.want { + t.Errorf("Delete(%s) = %v", tc.svc.Name, got) + } + if got := p.Generic(event.GenericEvent{Object: tc.svc}); got != tc.want { + t.Errorf("Generic(%s) = %v", tc.svc.Name, got) + } + } +} + +func TestPodSQLExecutorMissingPrimaryPod(t *testing.T) { + s := newTestScheme(t) + exec := NewPodSQLExecutor(newFakeClient(t, s, nil), nil, nil) + cluster := &cnpgv1.Cluster{ObjectMeta: metav1.ObjectMeta{Name: "db", Namespace: testNamespace}, Status: cnpgv1.ClusterStatus{CurrentPrimary: "db-1"}} + if _, err := exec(context.Background(), cluster, "SELECT 1"); err == nil || !strings.Contains(err.Error(), "failed to get primary pod") { + t.Fatalf("expected primary pod lookup error, got %v", err) + } +} + +func TestPodSQLExecutorExecFailureTargetsPrimaryPod(t *testing.T) { + var gotPath, gotQuery string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotPath, gotQuery = r.URL.Path, r.URL.RawQuery + http.Error(w, "exec forbidden", http.StatusForbidden) + })) + defer server.Close() + + config := &rest.Config{Host: server.URL} + clientset, err := kubernetes.NewForConfig(config) + if err != nil { + t.Fatal(err) + } + s := newTestScheme(t) + pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "db-1", Namespace: testNamespace}} + exec := NewPodSQLExecutor(newFakeClient(t, s, nil, pod), clientset, config) + cluster := &cnpgv1.Cluster{ObjectMeta: metav1.ObjectMeta{Name: "db", Namespace: testNamespace}, Status: cnpgv1.ClusterStatus{CurrentPrimary: "db-1"}} + + if _, err := exec(context.Background(), cluster, "SELECT 1"); err == nil || !strings.Contains(err.Error(), "failed to execute command") { + t.Fatalf("expected exec failure, got %v", err) + } + if gotPath != "/api/v1/namespaces/default/pods/db-1/exec" { + t.Fatalf("expected exec on the primary pod, got %q", gotPath) + } + for _, want := range []string{"container=postgres", "command=psql", "command=SELECT+1"} { + if !strings.Contains(gotQuery, want) { + t.Fatalf("expected %q in exec query %q", want, gotQuery) + } + } +} diff --git a/operator/src/internal/engine/scheduledbackup_test.go b/operator/src/internal/engine/scheduledbackup_test.go new file mode 100644 index 00000000..c3e3f6bc --- /dev/null +++ b/operator/src/internal/engine/scheduledbackup_test.go @@ -0,0 +1,326 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package engine + +import ( + "context" + "testing" + "time" + + cnpgv1 "github.com/cloudnative-pg/cloudnative-pg/api/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/tools/record" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" + "sigs.k8s.io/controller-runtime/pkg/manager" + + dbpreview "github.com/documentdb/documentdb-operator/api/preview" + "github.com/documentdb/documentdb-operator/internal/product" +) + +type scheduledReconciler = ScheduledBackupReconciler[*dbpreview.DocumentDB, *dbpreview.ScheduledBackup, *dbpreview.Backup] + +func newScheduledReconciler(c client.Client, s *runtime.Scheme, rec record.EventRecorder) *scheduledReconciler { + return &scheduledReconciler{ + Client: c, + Scheme: s, + Recorder: rec, + Cluster: product.DocumentDBAdapter{}, + Scheduled: product.DocumentDBScheduledBackupAdapter{}, + Backups: product.DocumentDBBackupAdapter{}, + } +} + +func testCluster() *dbpreview.DocumentDB { + return &dbpreview.DocumentDB{ + TypeMeta: metav1.TypeMeta{APIVersion: "documentdb.io/preview", Kind: "DocumentDB"}, + ObjectMeta: metav1.ObjectMeta{Name: "db", Namespace: testNamespace, UID: "db-uid"}, + } +} + +func testScheduledBackup(schedule string) *dbpreview.ScheduledBackup { + return &dbpreview.ScheduledBackup{ + ObjectMeta: metav1.ObjectMeta{Name: "nightly", Namespace: testNamespace}, + Spec: dbpreview.ScheduledBackupSpec{ + Schedule: schedule, + Cluster: cnpgv1.LocalObjectReference{Name: "db"}, + }, + } +} + +func testBackupFor(name string, phase cnpgv1.BackupPhase, created time.Time) *dbpreview.Backup { + return &dbpreview.Backup{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: testNamespace, CreationTimestamp: metav1.NewTime(created)}, + Spec: dbpreview.BackupSpec{Cluster: cnpgv1.LocalObjectReference{Name: "db"}}, + Status: dbpreview.BackupStatus{Phase: phase}, + } +} + +var scheduledKey = ctrl.Request{NamespacedName: types.NamespacedName{Name: "nightly", Namespace: testNamespace}} + +func TestScheduledBackupReconcileNotFound(t *testing.T) { + s := newTestScheme(t) + r := newScheduledReconciler(newFakeClient(t, s, nil), s, record.NewFakeRecorder(10)) + res, err := r.Reconcile(context.Background(), scheduledKey) + if err != nil || res != (ctrl.Result{}) { + t.Fatalf("expected empty result without error, got %v, %v", res, err) + } +} + +func TestScheduledBackupReconcileGetError(t *testing.T) { + s := newTestScheme(t) + c := newFakeClient(t, s, &interceptor.Funcs{ + Get: func(ctx context.Context, c client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + return errInjected + }, + }) + r := newScheduledReconciler(c, s, record.NewFakeRecorder(10)) + if _, err := r.Reconcile(context.Background(), scheduledKey); err != errInjected { + t.Fatalf("expected injected error, got %v", err) + } +} + +func TestScheduledBackupReconcileMissingClusterRecordsEvent(t *testing.T) { + s := newTestScheme(t) + rec := record.NewFakeRecorder(10) + r := newScheduledReconciler(newFakeClient(t, s, nil, testScheduledBackup("@hourly")), s, rec) + if _, err := r.Reconcile(context.Background(), scheduledKey); err == nil { + t.Fatal("expected error when the referenced cluster does not exist") + } + if events := drainEvents(rec); !hasEvent(events, "ClusterNotFound") || !hasEvent(events, "DocumentDB cluster") { + t.Fatalf("expected ClusterNotFound event naming the product kind, got %v", events) + } +} + +func TestScheduledBackupReconcileCreatesDueBackup(t *testing.T) { + s := newTestScheme(t) + rec := record.NewFakeRecorder(10) + sb := testScheduledBackup("* * * * *") + sb.Spec.RetentionDays = ptrInt(7) + past := metav1.NewTime(time.Now().Add(-time.Hour)) + sb.Status.NextScheduledTime = &past + c := newFakeClient(t, s, nil, sb, testCluster()) + r := newScheduledReconciler(c, s, rec) + + res, err := r.Reconcile(context.Background(), scheduledKey) + if err != nil { + t.Fatalf("reconcile: %v", err) + } + if res.RequeueAfter <= 0 || res.RequeueAfter > time.Minute { + t.Fatalf("expected requeue within a minute for a per-minute schedule, got %v", res.RequeueAfter) + } + + var backups dbpreview.BackupList + if err := c.List(context.Background(), &backups); err != nil { + t.Fatal(err) + } + if len(backups.Items) != 1 { + t.Fatalf("expected one created backup, got %d", len(backups.Items)) + } + b := backups.Items[0] + if b.Labels[dbpreview.ScheduledBackupLabel] != "nightly" || b.Spec.Cluster.Name != "db" || + b.Spec.RetentionDays == nil || *b.Spec.RetentionDays != 7 { + t.Fatalf("backup not derived from the scheduled backup: %+v", b) + } + + var got dbpreview.ScheduledBackup + if err := c.Get(context.Background(), scheduledKey.NamespacedName, &got); err != nil { + t.Fatal(err) + } + if owner := metav1.GetControllerOf(&got); owner == nil || owner.Name != "db" || owner.Kind != "DocumentDB" { + t.Fatalf("expected the cluster to own the scheduled backup, got %v", got.OwnerReferences) + } + if got.Status.LastScheduledTime == nil || got.Status.NextScheduledTime == nil || + !got.Status.NextScheduledTime.After(got.Status.LastScheduledTime.Time) { + t.Fatalf("expected last and next scheduled times to be recorded, got %+v", got.Status) + } + if events := drainEvents(rec); !hasEvent(events, "BackupSchedule") { + t.Fatalf("expected BackupSchedule event, got %v", events) + } +} + +func TestScheduledBackupReconcileWaitsForRunningBackup(t *testing.T) { + s := newTestScheme(t) + sb := testScheduledBackup("* * * * *") + running := testBackupFor("running", cnpgv1.BackupPhaseRunning, time.Now()) + c := newFakeClient(t, s, nil, sb, testCluster(), running) + r := newScheduledReconciler(c, s, record.NewFakeRecorder(10)) + + res, err := r.Reconcile(context.Background(), scheduledKey) + if err != nil || res.RequeueAfter != time.Minute { + t.Fatalf("expected one-minute requeue while a backup runs, got %v, %v", res, err) + } + var backups dbpreview.BackupList + if err := c.List(context.Background(), &backups); err != nil { + t.Fatal(err) + } + if len(backups.Items) != 1 { + t.Fatalf("expected no new backup while one is running, got %d", len(backups.Items)) + } +} + +func TestScheduledBackupReconcileSchedulesFromLatestBackup(t *testing.T) { + now := time.Now().UTC().Truncate(time.Second) + s := newTestScheme(t) + sb := testScheduledBackup("@every 24h") + // A schedule computed from the older backup would already be due. + older := testBackupFor("older", cnpgv1.BackupPhaseCompleted, now.Add(-48*time.Hour)) + newer := testBackupFor("newer", cnpgv1.BackupPhaseFailed, now.Add(-time.Hour)) + c := newFakeClient(t, s, nil, sb, testCluster(), older, newer) + r := newScheduledReconciler(c, s, record.NewFakeRecorder(10)) + + res, err := r.Reconcile(context.Background(), scheduledKey) + if err != nil { + t.Fatalf("reconcile: %v", err) + } + next := newer.CreationTimestamp.Add(24 * time.Hour) + if res.RequeueAfter <= 22*time.Hour || res.RequeueAfter > 24*time.Hour { + t.Fatalf("expected requeue about 23h away, 24h after the latest backup, got %v", res.RequeueAfter) + } + var got dbpreview.ScheduledBackup + if err := c.Get(context.Background(), scheduledKey.NamespacedName, &got); err != nil { + t.Fatal(err) + } + if got.Status.LastScheduledTime != nil { + t.Fatal("no backup should have been scheduled before the next run time") + } + if got.Status.NextScheduledTime == nil || !got.Status.NextScheduledTime.UTC().Equal(next) { + t.Fatalf("expected next run at %v, got %v", next, got.Status.NextScheduledTime) + } + var backups dbpreview.BackupList + if err := c.List(context.Background(), &backups); err != nil { + t.Fatal(err) + } + if len(backups.Items) != 2 { + t.Fatalf("expected no new backup, got %d", len(backups.Items)) + } +} + +func TestScheduledBackupReconcileErrors(t *testing.T) { + isScheduled := func(obj client.Object) bool { _, ok := obj.(*dbpreview.ScheduledBackup); return ok } + tests := []struct { + name string + funcs interceptor.Funcs + schedule string + wantEvent string + }{ + { + name: "owner reference update fails", + schedule: "* * * * *", + funcs: interceptor.Funcs{Update: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.UpdateOption) error { + return errInjected + }}, + }, + { + name: "invalid schedule", + schedule: "not a cron", + wantEvent: "InvalidSchedule", + }, + { + name: "listing backups fails", + schedule: "* * * * *", + funcs: interceptor.Funcs{List: func(ctx context.Context, c client.WithWatch, list client.ObjectList, opts ...client.ListOption) error { + return errInjected + }}, + }, + { + name: "creating backup fails", + schedule: "* * * * *", + wantEvent: "BackupCreation", + funcs: interceptor.Funcs{Create: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.CreateOption) error { + return errInjected + }}, + }, + { + name: "status update fails", + schedule: "0 0 1 1 *", + funcs: interceptor.Funcs{SubResourceUpdate: func(ctx context.Context, c client.Client, sub string, obj client.Object, opts ...client.SubResourceUpdateOption) error { + if isScheduled(obj) { + return errInjected + } + return c.SubResource(sub).Update(ctx, obj, opts...) + }}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := newTestScheme(t) + rec := record.NewFakeRecorder(10) + sb := testScheduledBackup(tt.schedule) + past := metav1.NewTime(time.Now().Add(-time.Hour)) + sb.Status.NextScheduledTime = &past + c := newFakeClient(t, s, &tt.funcs, sb, testCluster()) + r := newScheduledReconciler(c, s, rec) + if _, err := r.Reconcile(context.Background(), scheduledKey); err == nil { + t.Fatal("expected error") + } + if tt.wantEvent != "" && !hasEvent(drainEvents(rec), tt.wantEvent) { + t.Fatalf("expected %s event", tt.wantEvent) + } + }) + } +} + +func TestScheduledBackupEnsureOwnerReferenceKeepsExistingOwner(t *testing.T) { + s := newTestScheme(t) + sb := testScheduledBackup("@daily") + sb.OwnerReferences = []metav1.OwnerReference{{APIVersion: "v1", Kind: "ConfigMap", Name: "other", UID: "x"}} + // No cluster exists: an existing owner must short-circuit the lookup. + r := newScheduledReconciler(newFakeClient(t, s, nil, sb), s, record.NewFakeRecorder(10)) + if err := r.EnsureOwnerReference(context.Background(), sb); err != nil { + t.Fatalf("expected no error, got %v", err) + } + if len(sb.OwnerReferences) != 1 || sb.OwnerReferences[0].Name != "other" { + t.Fatalf("existing owner reference changed: %v", sb.OwnerReferences) + } +} + +func TestScheduledBackupEnsureOwnerReferenceSchemeError(t *testing.T) { + s := newTestScheme(t) + sb := testScheduledBackup("@daily") + c := newFakeClient(t, s, nil, sb, testCluster()) + // The reconciler scheme does not know the cluster kind, so the owner + // reference cannot be built. + r := newScheduledReconciler(c, runtime.NewScheme(), record.NewFakeRecorder(10)) + if err := r.EnsureOwnerReference(context.Background(), sb); err == nil { + t.Fatal("expected error setting the owner reference") + } +} + +// indexerManager records the field index registered by SetupWithManager. +type indexerManager struct { + manager.Manager + field string + extract client.IndexerFunc +} + +func (m *indexerManager) GetFieldIndexer() client.FieldIndexer { return m } + +func (m *indexerManager) IndexField(_ context.Context, _ client.Object, field string, extract client.IndexerFunc) error { + m.field, m.extract = field, extract + return errInjected +} + +func TestScheduledBackupSetupWithManagerIndexesBackupsByCluster(t *testing.T) { + s := newTestScheme(t) + r := newScheduledReconciler(nil, s, nil) + mgr := &indexerManager{} + if err := r.SetupWithManager(mgr); err != errInjected { + t.Fatalf("expected index registration error to propagate, got %v", err) + } + if mgr.field != BackupClusterField { + t.Fatalf("expected index on %q, got %q", BackupClusterField, mgr.field) + } + if got := mgr.extract(testBackupFor("b", "", time.Now())); len(got) != 1 || got[0] != "db" { + t.Fatalf("expected the backup's cluster name, got %v", got) + } + if got := mgr.extract(testCluster()); got != nil { + t.Fatalf("expected nil for non-Backup objects, got %v", got) + } +} + +func ptrInt(v int) *int { return &v } diff --git a/operator/src/pkg/core/facade_test.go b/operator/src/pkg/core/facade_test.go new file mode 100644 index 00000000..e6aa7533 --- /dev/null +++ b/operator/src/pkg/core/facade_test.go @@ -0,0 +1,179 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package core_test + +import ( + "context" + "errors" + "strings" + "testing" + + cnpgv1 "github.com/cloudnative-pg/cloudnative-pg/api/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/validation/field" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + utils "github.com/documentdb/documentdb-operator/internal/utils" + "github.com/documentdb/documentdb-operator/pkg/core" +) + +// A second product's admission webhook validates its custom resource through +// the facade alone. +func TestFacadeAdmissionValidation(t *testing.T) { + terms := core.ValidationTerms{ + Product: "Widget", + VersionField: "spec.widgetVersion", + ImageField: "spec.image.widget", + SchemaWithoutBinaryFields: "spec.widgetVersion", + } + + if got := core.ImageTagSemver("example.com/widget:1.2.3"); got != "1.2.3" { + t.Fatalf("ImageTagSemver = %q", got) + } + if got := core.ResolveBinaryVersion("example.com/widget@sha256:abc", "0.110.0"); got != "0.110.0" { + t.Fatalf("digest-only image should fall back to the version field, got %q", got) + } + + if errs := core.ValidateSchemaVersion("0.111.0", "0.110.0", terms); len(errs) != 1 { + t.Fatalf("schema above binary should be rejected, got %v", errs) + } + if errs := core.ValidateSchemaVersion("0.110.0", "", terms); len(errs) != 1 || !strings.Contains(errs[0].Detail, "spec.widgetVersion") { + t.Fatalf("schema without a binary should name the product's fields, got %v", errs) + } + if errs := core.ValidateImageRollback("0.111.0", "0.110.0"); len(errs) != 1 { + t.Fatalf("binary below installed schema should be rejected, got %v", errs) + } + if errs := core.ValidateImageRollback("0.110.0", "0.111.0"); len(errs) != 0 { + t.Fatalf("upgrade should be allowed, got %v", errs) + } + sizePath := field.NewPath("spec", "storage", "size") + if errs := core.ValidateStorageResize(sizePath, "10Gi", "5Gi"); len(errs) != 1 || errs[0].Field != "spec.storage.size" { + t.Fatalf("shrink should be rejected at the product path, got %v", errs) + } + + base := field.NewPath("spec", "bootstrap", "recovery") + backupSrc := core.BackupRestoreSource(base, "nightly") + if backupSrc.FieldPath.String() != "spec.bootstrap.recovery.backup" || backupSrc.Name != "nightly" { + t.Fatalf("unexpected backup source %+v", backupSrc) + } + schema, warnings := core.RestoreSourceSchemaVersion(context.Background(), nil, backupSrc, core.DocumentDBProfile().Naming, + func() (string, error) { return "0.111.0", nil }) + if schema != "0.111.0" || len(warnings) != 0 { + t.Fatalf("backup schema = %q, warnings %v", schema, warnings) + } + if _, errs := core.ValidateRestoreCompatibility("0.110.0", schema, backupSrc, terms); len(errs) != 1 { + t.Fatalf("restoring a newer schema should be rejected, got %v", errs) + } + + naming := core.DocumentDBProfile().Naming + pv := &corev1.PersistentVolume{ObjectMeta: metav1.ObjectMeta{ + Name: "pv-1", + Annotations: map[string]string{naming.SchemaVersionAnnotation(): "0.109.0"}, + }} + c := fake.NewClientBuilder().WithObjects(pv).Build() + pvSrc := core.PVRestoreSource(base, "pv-1") + schema, warnings = core.RestoreSourceSchemaVersion(context.Background(), c, pvSrc, naming, + func() (string, error) { return "", errors.New("unused") }) + if schema != "0.109.0" || len(warnings) != 0 { + t.Fatalf("pv schema = %q, warnings %v", schema, warnings) + } + if warnings, errs := core.ValidateRestoreCompatibility("0.110.0", schema, pvSrc, terms); len(errs) != 0 || len(warnings) != 0 { + t.Fatalf("restoring an older schema should be allowed, got %v %v", warnings, errs) + } +} + +func TestFacadeParametersAndResources(t *testing.T) { + intent := core.ClusterIntent{ + Product: core.ProductProfile{Postgres: core.PostgresProfile{ + ProtectedParameters: map[string]string{"widget.mode": "strict"}, + }}, + Postgres: core.Postgres{Parameters: map[string]string{"widget.mode": "loose", "work_mem": "8MB"}}, + FeatureGates: core.FeatureGates{ChangeStreams: true}, + } + protected := core.ProtectedParameters(intent) + if protected["widget.mode"] != "strict" || protected["wal_level"] != "logical" { + t.Fatalf("unexpected protected parameters %v", protected) + } + merged := core.MergeParameters(intent, 4<<30) + if merged["widget.mode"] != "strict" || merged["work_mem"] != "8MB" || merged["wal_level"] != "logical" { + t.Fatalf("user overrides must not beat protected parameters: %v", merged) + } + + cfg := core.DefaultSplitConfig() + split := core.ComputeResourceSplit(core.Resources{Memory: "4Gi"}, false, cfg) + if split.MonitoringEnabled || split.PostgresMemoryBytes <= 0 || split.PostgresMemoryBytes >= 4<<30 || split.Gateway.MemoryLimit == "" { + t.Fatalf("expected the gateway to be carved out of the envelope, got %+v", split) + } + base := field.NewPath("spec", "resources") + if errs := core.ValidateResources(core.Resources{Memory: "4Gi"}, false, base, cfg); len(errs) != 0 { + t.Fatalf("valid envelope rejected: %v", errs) + } + errs := core.ValidateResources(core.Resources{Memory: "1Gi", Gateway: &core.ComponentResources{Memory: "2Gi"}}, false, base, cfg) + if len(errs) == 0 || !strings.HasPrefix(errs[0].Field, "spec.resources") { + t.Fatalf("over-committed envelope should be rejected under the product path, got %v", errs) + } +} + +func TestFacadeExtensionLifecycleHelpers(t *testing.T) { + if got := core.ExtensionVersionSQL("widget"); !strings.Contains(got, "name = 'widget'") { + t.Fatalf("ExtensionVersionSQL = %q", got) + } + if got := core.ExtensionUpdateSQL("widget", "1.2-3"); got != "ALTER EXTENSION widget UPDATE TO '1.2-3'" { + t.Fatalf("ExtensionUpdateSQL = %q", got) + } + if got := core.ReplicationRoleGrantSQL("widget_admin"); got != "GRANT widget_admin TO streaming_replica;" { + t.Fatalf("ReplicationRoleGrantSQL = %q", got) + } + + cluster := &cnpgv1.Cluster{} + if core.ExtensionImage(cluster, "widget") != "" || core.GatewayImage(cluster) != "" { + t.Fatal("expected no images on an empty cluster") + } + cluster.Spec.PostgresConfiguration.Extensions = []cnpgv1.ExtensionConfiguration{{ + Name: "widget", ImageVolumeSource: corev1.ImageVolumeSource{Reference: "example.com/widget:1.0"}, + }} + cluster.Spec.Plugins = []cnpgv1.PluginConfiguration{{ + Name: "sidecar", Parameters: map[string]string{utils.PLUGIN_PARAM_GATEWAY_IMAGE: "example.com/gw:1.0"}, + }} + if got := core.ExtensionImage(cluster, "widget"); got != "example.com/widget:1.0" { + t.Fatalf("ExtensionImage = %q", got) + } + if got := core.GatewayImage(cluster); got != "example.com/gw:1.0" { + t.Fatalf("GatewayImage = %q", got) + } + + if core.ParsePullPolicy("Always") != corev1.PullAlways || core.ParsePullPolicy("sometimes") != "" { + t.Fatal("ParsePullPolicy should accept only Kubernetes pull policies") + } +} + +func TestFacadeSyncClusterPatchesImages(t *testing.T) { + current := &cnpgv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{Name: "w", Namespace: "default"}, + Spec: cnpgv1.ClusterSpec{Plugins: []cnpgv1.PluginConfiguration{{ + Name: "sidecar", Parameters: map[string]string{utils.PLUGIN_PARAM_GATEWAY_IMAGE: "example.com/gw:1.0"}, + }}}, + } + desired := current.DeepCopy() + desired.Spec.Plugins[0].Parameters[utils.PLUGIN_PARAM_GATEWAY_IMAGE] = "example.com/gw:2.0" + + s := runtime.NewScheme() + if err := cnpgv1.AddToScheme(s); err != nil { + t.Fatal(err) + } + c := fake.NewClientBuilder().WithScheme(s).WithObjects(current.DeepCopy()).Build() + if err := core.SyncCluster(context.Background(), c, current, desired, nil); err != nil { + t.Fatal(err) + } + got := &cnpgv1.Cluster{} + if err := c.Get(context.Background(), client.ObjectKeyFromObject(current), got); err != nil { + t.Fatal(err) + } + if core.GatewayImage(got) != "example.com/gw:2.0" { + t.Fatalf("expected the gateway image to be patched, got %q", core.GatewayImage(got)) + } +} From be0b3b82fcc864cfe575e8fd264bc37ba35a8f6b Mon Sep 17 00:00:00 2001 From: Rayhan Hossain Date: Tue, 20 Oct 2026 13:02:37 -0700 Subject: [PATCH 23/27] test(core): avoid deprecated requeue field in assertion Assert an immediate requeue through Result.IsZero and RequeueAfter while preserving the finalizer persistence check. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 33b3cb3c-1b25-4ffb-bff1-ebf0f3d04058 Signed-off-by: Rayhan Hossain --- operator/src/internal/engine/cluster_reconciler_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/operator/src/internal/engine/cluster_reconciler_test.go b/operator/src/internal/engine/cluster_reconciler_test.go index abbd0e2d..52a3b792 100644 --- a/operator/src/internal/engine/cluster_reconciler_test.go +++ b/operator/src/internal/engine/cluster_reconciler_test.go @@ -185,7 +185,7 @@ func TestClusterReconcileFinalizerLifecycle(t *testing.T) { c := newFakeClient(t, s, nil, testCluster()) r, _ := newClusterReconciler(c, s, &sqlRecorder{}) res, err := r.Reconcile(context.Background(), clusterKey) - if err != nil || !res.Requeue { + if err != nil || res.IsZero() || res.RequeueAfter != 0 { t.Fatalf("expected requeue after adding finalizer, got %v, %v", res, err) } if got := getDocumentDB(t, c); !slices.Contains(got.Finalizers, finalizer) { From 5a0596b7cc7cd4b77a57e107d01ce771e43dbeb9 Mon Sep 17 00:00:00 2001 From: Rayhan Hossain Date: Wed, 21 Oct 2026 12:54:37 -0700 Subject: [PATCH 24/27] fix(core): quote SQL names and stabilize ImageVolume probes Quote product-defined extension and replication role identifiers and catalog lookup literals with the existing PostgreSQL quoting helpers. Keep ImageVolume dry-run names independent of human-readable product names without changing the intentional fail-open policy. Add regressions for hyphenated, mixed-case and quoted SQL names, and capability enforcement and caching with product display names containing spaces, punctuation and excessive length. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 33b3cb3c-1b25-4ffb-bff1-ebf0f3d04058 Signed-off-by: Rayhan Hossain --- CHANGELOG.md | 3 + docs/designs/product-neutral-core.md | 6 ++ operator/src/go.mod | 2 +- operator/src/internal/cnpg/extension_sql.go | 16 ++-- .../src/internal/cnpg/extension_sql_test.go | 38 +++++++++- .../controller/documentdb_controller_test.go | 6 +- .../src/internal/validation/imagevolume.go | 5 +- .../internal/validation/imagevolume_test.go | 76 +++++++++++++++++++ operator/src/pkg/core/core.go | 6 +- operator/src/pkg/core/example_test.go | 2 +- operator/src/pkg/core/facade_test.go | 4 +- 11 files changed, 141 insertions(+), 23 deletions(-) create mode 100644 operator/src/internal/validation/imagevolume_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index e053dea2..f9262512 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,9 @@ ### Security - Require Go 1.26.10 across all six modules and pin Go builder images to that patch release to address [GO-2026-6809](https://pkg.go.dev/vuln/GO-2026-6809) in `crypto/x509`. Unit-test workflows now resolve Go from the operator module instead of an older hard-coded version. +### Bug Fixes +- Quote product-defined extension and replication role names in lifecycle SQL, and use a product-independent ImageVolume probe name so display names such as `Cosmos DB` do not bypass capability checks. + ### Major Features - **Product-neutral operator core**: The CNPG `Cluster` renderer, sync, PostgreSQL parameter merge, resource split/validation and extension lifecycle SQL are now driven by a `ProductProfile` and `ClusterIntent` instead of the `DocumentDB` custom resource, and are exported for other operator modules through `operator/src/pkg/core`. The sidecar injector accepts optional `gatewayContainerName`, `gatewayPort`, `gatewayArgs`, `gatewayUsernameEnv` and `gatewayPasswordEnv` parameters for non-DocumentDB gateways. The operator keeps these parameters in sync on existing clusters, and the client Service port follows the gateway port. Other products can build multi-region topologies with `core.NewReplicationContext` and name the OTel SQL health metric with `NamingProfile.HealthMetricName` (DocumentDB keeps `documentdb.postgres.up`). The cluster, certificate, persistent volume, backup and scheduled-backup reconcilers and the admission validation rules are generic over the custom resource type through a `ClusterAdapter`, with DocumentDB as the first implementation; `pkg/core` exports them too. Rendered DocumentDB clusters are byte-identical to before, CRDs and RBAC are unchanged, and existing deployments see no change. See [Product-neutral operator core](docs/designs/product-neutral-core.md). diff --git a/docs/designs/product-neutral-core.md b/docs/designs/product-neutral-core.md index 3df0a816..cdfd29da 100644 --- a/docs/designs/product-neutral-core.md +++ b/docs/designs/product-neutral-core.md @@ -75,6 +75,12 @@ unmodified. - **`internal/validation`** holds the admission rules (schema version vs. binary, image rollback, storage resize, restore compatibility, ImageVolume probe). They take `Terms` so error messages use the product's field names. + The ImageVolume probe uses a product-independent Kubernetes name prefix; + display names such as `Cosmos DB` are used only in diagnostics. + +Extension and replication role names in the profile are unquoted PostgreSQL +names. The lifecycle SQL helpers quote identifiers and catalog lookup literals, +preserving mixed case and supporting names containing hyphens or quotes. The DocumentDB controllers and webhook in `internal/controller` and `internal/webhook` are now thin wrappers over these packages. They keep their diff --git a/operator/src/go.mod b/operator/src/go.mod index 8e3fafd7..7e6fc9d6 100644 --- a/operator/src/go.mod +++ b/operator/src/go.mod @@ -76,7 +76,7 @@ require ( github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/kubernetes-csi/external-snapshotter/client/v8 v8.6.0 - github.com/lib/pq v1.12.3 // indirect + github.com/lib/pq v1.12.3 github.com/moby/spdystream v0.5.1 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect diff --git a/operator/src/internal/cnpg/extension_sql.go b/operator/src/internal/cnpg/extension_sql.go index a87e924f..61538b19 100644 --- a/operator/src/internal/cnpg/extension_sql.go +++ b/operator/src/internal/cnpg/extension_sql.go @@ -7,19 +7,19 @@ import ( "fmt" cnpgv1 "github.com/cloudnative-pg/cloudnative-pg/api/v1" + "github.com/lib/pq" util "github.com/documentdb/documentdb-operator/internal/utils" ) // The helpers below build the SQL and read the cluster state the extension -// lifecycle needs for any product. Extension and role names come from the -// product profile (operator constants, never user input), so they are -// interpolated as bare identifiers. +// lifecycle needs for any product. Product-defined names are quoted as SQL +// identifiers in statements and as string literals in catalog lookups. // ExtensionVersionSQL returns the query reporting the default (binary) and // installed versions of the named extension. func ExtensionVersionSQL(extension string) string { - return fmt.Sprintf("SELECT default_version, installed_version FROM pg_available_extensions WHERE name = '%s'", extension) + return fmt.Sprintf("SELECT default_version, installed_version FROM pg_available_extensions WHERE name = %s", pq.QuoteLiteral(extension)) } // ExtensionUpdateSQL returns the ALTER EXTENSION statement for the named @@ -27,21 +27,21 @@ func ExtensionVersionSQL(extension string) string { // targetVersion must already be validated (digits, dots, and hyphens only). func ExtensionUpdateSQL(extension, targetVersion string) string { if targetVersion == "" { - return fmt.Sprintf("ALTER EXTENSION %s UPDATE", extension) + return fmt.Sprintf("ALTER EXTENSION %s UPDATE", pq.QuoteIdentifier(extension)) } - return fmt.Sprintf("ALTER EXTENSION %s UPDATE TO '%s'", extension, targetVersion) + return fmt.Sprintf("ALTER EXTENSION %s UPDATE TO %s", pq.QuoteIdentifier(extension), pq.QuoteLiteral(targetVersion)) } // ReplicationRoleCheckSQL returns the query that yields one row when CNPG's // streaming_replica user already holds the given role. func ReplicationRoleCheckSQL(role string) string { - return fmt.Sprintf("SELECT 1 FROM pg_roles WHERE rolname = 'streaming_replica' AND pg_has_role('streaming_replica', '%s', 'USAGE');", role) + return fmt.Sprintf("SELECT 1 FROM pg_roles WHERE rolname = 'streaming_replica' AND pg_has_role('streaming_replica', %s, 'USAGE');", pq.QuoteLiteral(role)) } // ReplicationRoleGrantSQL returns the statement granting the given role to // CNPG's streaming_replica user. func ReplicationRoleGrantSQL(role string) string { - return fmt.Sprintf("GRANT %s TO streaming_replica;", role) + return fmt.Sprintf("GRANT %s TO streaming_replica;", pq.QuoteIdentifier(role)) } // ExtensionImage returns the image currently declared for the named extension diff --git a/operator/src/internal/cnpg/extension_sql_test.go b/operator/src/internal/cnpg/extension_sql_test.go index fa5da7ef..c0220d71 100644 --- a/operator/src/internal/cnpg/extension_sql_test.go +++ b/operator/src/internal/cnpg/extension_sql_test.go @@ -15,10 +15,10 @@ import ( func TestExtensionLifecycleSQLUsesProductNames(t *testing.T) { cases := map[string]string{ ExtensionVersionSQL("pgcosmos"): "SELECT default_version, installed_version FROM pg_available_extensions WHERE name = 'pgcosmos'", - ExtensionUpdateSQL("pgcosmos", ""): "ALTER EXTENSION pgcosmos UPDATE", - ExtensionUpdateSQL("pgcosmos", "1.2-0"): "ALTER EXTENSION pgcosmos UPDATE TO '1.2-0'", + ExtensionUpdateSQL("pgcosmos", ""): `ALTER EXTENSION "pgcosmos" UPDATE`, + ExtensionUpdateSQL("pgcosmos", "1.2-0"): `ALTER EXTENSION "pgcosmos" UPDATE TO '1.2-0'`, ReplicationRoleCheckSQL("cosmos_admin"): "SELECT 1 FROM pg_roles WHERE rolname = 'streaming_replica' AND pg_has_role('streaming_replica', 'cosmos_admin', 'USAGE');", - ReplicationRoleGrantSQL("cosmos_admin"): "GRANT cosmos_admin TO streaming_replica;", + ReplicationRoleGrantSQL("cosmos_admin"): `GRANT "cosmos_admin" TO streaming_replica;`, } for got, want := range cases { if got != want { @@ -27,6 +27,38 @@ func TestExtensionLifecycleSQLUsesProductNames(t *testing.T) { } } +func TestExtensionLifecycleSQLQuotesProductNames(t *testing.T) { + for _, tt := range []struct { + name string + identifier string + literal string + }{ + {name: "pg-cosmos", identifier: `"pg-cosmos"`, literal: "'pg-cosmos'"}, + {name: "PgCosmos", identifier: `"PgCosmos"`, literal: "'PgCosmos'"}, + {name: `pg"cosmos`, identifier: `"pg""cosmos"`, literal: `'pg"cosmos'`}, + {name: "pg'cosmos", identifier: `"pg'cosmos"`, literal: "'pg''cosmos'"}, + {name: "select", identifier: `"select"`, literal: "'select'"}, + } { + t.Run(tt.name, func(t *testing.T) { + if got, want := ExtensionUpdateSQL(tt.name, ""), "ALTER EXTENSION "+tt.identifier+" UPDATE"; got != want { + t.Errorf("default upgrade = %q, want %q", got, want) + } + if got, want := ExtensionUpdateSQL(tt.name, "1.2-0"), "ALTER EXTENSION "+tt.identifier+" UPDATE TO '1.2-0'"; got != want { + t.Errorf("targeted upgrade = %q, want %q", got, want) + } + if got, want := ReplicationRoleGrantSQL(tt.name), "GRANT "+tt.identifier+" TO streaming_replica;"; got != want { + t.Errorf("replication grant = %q, want %q", got, want) + } + if got, want := ExtensionVersionSQL(tt.name), "SELECT default_version, installed_version FROM pg_available_extensions WHERE name = "+tt.literal; got != want { + t.Errorf("extension lookup = %q, want %q", got, want) + } + if got, want := ReplicationRoleCheckSQL(tt.name), "SELECT 1 FROM pg_roles WHERE rolname = 'streaming_replica' AND pg_has_role('streaming_replica', "+tt.literal+", 'USAGE');"; got != want { + t.Errorf("replication role lookup = %q, want %q", got, want) + } + }) + } +} + func TestClusterImageReaders(t *testing.T) { cluster := &cnpgv1.Cluster{Spec: cnpgv1.ClusterSpec{ PostgresConfiguration: cnpgv1.PostgresConfiguration{Extensions: []cnpgv1.ExtensionConfiguration{ diff --git a/operator/src/internal/controller/documentdb_controller_test.go b/operator/src/internal/controller/documentdb_controller_test.go index ee46ac21..0fd9d066 100644 --- a/operator/src/internal/controller/documentdb_controller_test.go +++ b/operator/src/internal/controller/documentdb_controller_test.go @@ -854,7 +854,7 @@ var _ = Describe("DocumentDB Controller", func() { // Verify both SQL calls were made Expect(sqlCalls).To(HaveLen(2)) Expect(sqlCalls[0]).To(ContainSubstring("pg_available_extensions")) - Expect(sqlCalls[1]).To(Equal("ALTER EXTENSION documentdb UPDATE")) + Expect(sqlCalls[1]).To(Equal(`ALTER EXTENSION "documentdb" UPDATE`)) // Status should reflect the upgraded version (default version as semver) updatedDB := &dbpreview.DocumentDB{} @@ -1361,7 +1361,7 @@ var _ = Describe("DocumentDB Controller", func() { // Both version-check and ALTER EXTENSION should have been called Expect(sqlCalls).To(HaveLen(2)) Expect(sqlCalls[0]).To(ContainSubstring("pg_available_extensions")) - Expect(sqlCalls[1]).To(Equal("ALTER EXTENSION documentdb UPDATE")) + Expect(sqlCalls[1]).To(Equal(`ALTER EXTENSION "documentdb" UPDATE`)) // Status should reflect the upgraded version updatedDB := &dbpreview.DocumentDB{} @@ -1433,7 +1433,7 @@ var _ = Describe("DocumentDB Controller", func() { // Should run ALTER EXTENSION UPDATE TO specific version Expect(sqlCalls).To(HaveLen(2)) Expect(sqlCalls[0]).To(ContainSubstring("pg_available_extensions")) - Expect(sqlCalls[1]).To(Equal("ALTER EXTENSION documentdb UPDATE TO '0.110-0'")) + Expect(sqlCalls[1]).To(Equal(`ALTER EXTENSION "documentdb" UPDATE TO '0.110-0'`)) // Status should reflect the explicit version updatedDB := &dbpreview.DocumentDB{} diff --git a/operator/src/internal/validation/imagevolume.go b/operator/src/internal/validation/imagevolume.go index b728398a..fb132d17 100644 --- a/operator/src/internal/validation/imagevolume.go +++ b/operator/src/internal/validation/imagevolume.go @@ -6,7 +6,6 @@ package validation import ( "context" "fmt" - "strings" "sync/atomic" "github.com/go-logr/logr" @@ -35,7 +34,7 @@ type ImageVolumeProbe struct { // administrator enables the feature gate later the next create re-probes and // succeeds without restarting the operator. Confirmed *atomic.Bool - // Product names the operator and extension in the error message. + // Product is a display name used only in the error message. Product string Log logr.Logger } @@ -143,7 +142,7 @@ func hasImageVolume(pod *corev1.Pod) bool { func (v *ImageVolumeProbe) dryRunProbePod(ctx context.Context, namespace string, volume corev1.Volume) (*corev1.Pod, error) { pod := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ - GenerateName: strings.ToLower(v.Product) + "-imagevolume-preflight-", + GenerateName: "imagevolume-preflight-", Namespace: namespace, }, Spec: corev1.PodSpec{ diff --git a/operator/src/internal/validation/imagevolume_test.go b/operator/src/internal/validation/imagevolume_test.go new file mode 100644 index 00000000..d55a0679 --- /dev/null +++ b/operator/src/internal/validation/imagevolume_test.go @@ -0,0 +1,76 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package validation + +import ( + "context" + "fmt" + "strings" + "sync/atomic" + "testing" + + "github.com/go-logr/logr" + corev1 "k8s.io/api/core/v1" + apivalidation "k8s.io/apimachinery/pkg/api/validation" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" +) + +func TestImageVolumeProbeProductDisplayNames(t *testing.T) { + for _, product := range []string{"Cosmos DB", "Cosmos/DB (Preview)", strings.Repeat("Long Product ", 30)} { + for _, behavior := range []string{"enabled", "rejected", "pruned"} { + t.Run(product+"/"+behavior, func(t *testing.T) { + scheme := runtime.NewScheme() + if err := corev1.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + var creates int + c := fake.NewClientBuilder().WithScheme(scheme).WithInterceptorFuncs(interceptor.Funcs{ + Create: func(_ context.Context, _ client.WithWatch, obj client.Object, opts ...client.CreateOption) error { + creates++ + pod := obj.(*corev1.Pod) + if errs := apivalidation.NameIsDNSSubdomain(pod.GenerateName, true); len(errs) != 0 { + return fmt.Errorf("invalid metadata.generateName: %v", errs) + } + options := (&client.CreateOptions{}).ApplyOptions(opts) + if len(options.DryRun) != 1 || options.DryRun[0] != "All" { + t.Fatal("capability probe must be a server-side dry run") + } + for i := range pod.Spec.Volumes { + if pod.Spec.Volumes[i].Image == nil { + continue + } + switch behavior { + case "rejected": + return fmt.Errorf("ImageVolume feature disabled") + case "pruned": + pod.Spec.Volumes[i].Image = nil + } + } + return nil + }, + }).Build() + var confirmed atomic.Bool + probe := &ImageVolumeProbe{Client: c, Confirmed: &confirmed, Product: product, Log: logr.Discard()} + err := probe.Ensure(context.Background(), "default") + if behavior == "enabled" { + if err != nil || !confirmed.Load() { + t.Fatalf("expected supported feature to be cached, got error %v, cached %v", err, confirmed.Load()) + } + if err := probe.Ensure(context.Background(), "default"); err != nil { + t.Fatal(err) + } + } else if err == nil || !strings.Contains(err.Error(), "ImageVolume feature is not enabled") || + !strings.Contains(err.Error(), product) || confirmed.Load() { + t.Fatalf("expected uncached capability rejection naming %q, got error %v, cached %v", product, err, confirmed.Load()) + } + if creates != 2 { + t.Fatalf("expected a valid baseline and image probe, got %d creates", creates) + } + }) + } + } +} diff --git a/operator/src/pkg/core/core.go b/operator/src/pkg/core/core.go index b6d32708..3b49a959 100644 --- a/operator/src/pkg/core/core.go +++ b/operator/src/pkg/core/core.go @@ -98,7 +98,8 @@ func ValidateResources(res Resources, monitoring bool, base *field.Path, cfg Spl // ExtensionVersionSQL queries the default and installed versions of extension. func ExtensionVersionSQL(extension string) string { return cnpg.ExtensionVersionSQL(extension) } -// ExtensionUpdateSQL upgrades extension, to targetVersion when set. +// ExtensionUpdateSQL upgrades extension, to targetVersion when set. Pass the +// unquoted extension name; it is quoted as a SQL identifier. func ExtensionUpdateSQL(extension, targetVersion string) string { return cnpg.ExtensionUpdateSQL(extension, targetVersion) } @@ -106,7 +107,8 @@ func ExtensionUpdateSQL(extension, targetVersion string) string { // ReplicationRoleCheckSQL checks whether streaming_replica is a member of role. func ReplicationRoleCheckSQL(role string) string { return cnpg.ReplicationRoleCheckSQL(role) } -// ReplicationRoleGrantSQL grants role to streaming_replica. +// ReplicationRoleGrantSQL grants role to streaming_replica. Pass the unquoted +// role name; it is quoted as a SQL identifier. func ReplicationRoleGrantSQL(role string) string { return cnpg.ReplicationRoleGrantSQL(role) } // ExtensionImage returns the image currently mounted for extension in cluster. diff --git a/operator/src/pkg/core/example_test.go b/operator/src/pkg/core/example_test.go index 3260adfd..bb98f2b7 100644 --- a/operator/src/pkg/core/example_test.go +++ b/operator/src/pkg/core/example_test.go @@ -40,7 +40,7 @@ func Example() { // Output: // pgcosmos // operator - // ALTER EXTENSION pgcosmos UPDATE + // ALTER EXTENSION "pgcosmos" UPDATE } // ExampleNewReplicationContext builds the contexts a product's diff --git a/operator/src/pkg/core/facade_test.go b/operator/src/pkg/core/facade_test.go index e6aa7533..44a784a9 100644 --- a/operator/src/pkg/core/facade_test.go +++ b/operator/src/pkg/core/facade_test.go @@ -122,10 +122,10 @@ func TestFacadeExtensionLifecycleHelpers(t *testing.T) { if got := core.ExtensionVersionSQL("widget"); !strings.Contains(got, "name = 'widget'") { t.Fatalf("ExtensionVersionSQL = %q", got) } - if got := core.ExtensionUpdateSQL("widget", "1.2-3"); got != "ALTER EXTENSION widget UPDATE TO '1.2-3'" { + if got := core.ExtensionUpdateSQL("widget", "1.2-3"); got != `ALTER EXTENSION "widget" UPDATE TO '1.2-3'` { t.Fatalf("ExtensionUpdateSQL = %q", got) } - if got := core.ReplicationRoleGrantSQL("widget_admin"); got != "GRANT widget_admin TO streaming_replica;" { + if got := core.ReplicationRoleGrantSQL("widget_admin"); got != `GRANT "widget_admin" TO streaming_replica;` { t.Fatalf("ReplicationRoleGrantSQL = %q", got) } From 4f4015ab60a05f94d787f46fe430b9fd0c43f70a Mon Sep 17 00:00:00 2001 From: Rayhan Hossain Date: Wed, 21 Oct 2026 13:30:15 -0700 Subject: [PATCH 25/27] fix(core): align gateway naming and verify second-product lifecycle Keep truncated Service names DNS-valid and derive certificate SANs from the actual Service name in both certificate modes. Cover both naming boundaries with focused regressions. Reuse a realistic public Cosmos profile across cluster rendering and reconciliation, gateway contract convergence, credentials and TLS injection, admission, extension upgrades, and custom Backup/PV restore metadata. Verify the public fixture from an independent consumer module, and document destination-specific naming and plugin identity constraints. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 33b3cb3c-1b25-4ffb-bff1-ebf0f3d04058 Signed-off-by: Rayhan Hossain --- CHANGELOG.md | 1 + docs/designs/product-neutral-core.md | 46 +- .../internal/lifecycle/lifecycle_test.go | 31 +- operator/src/internal/engine/certificate.go | 4 +- operator/src/internal/product/cluster.go | 6 +- operator/src/internal/product/cluster_test.go | 12 + operator/src/internal/product/profile.go | 4 +- operator/src/pkg/core/engine_test.go | 22 +- operator/src/pkg/core/replication_test.go | 3 +- operator/src/pkg/core/second_product_test.go | 544 ++++++++++++++++++ 10 files changed, 649 insertions(+), 24 deletions(-) create mode 100644 operator/src/pkg/core/second_product_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index f9262512..322de4a7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ - Require Go 1.26.10 across all six modules and pin Go builder images to that patch release to address [GO-2026-6809](https://pkg.go.dev/vuln/GO-2026-6809) in `crypto/x509`. Unit-test workflows now resolve Go from the operator module instead of an older hard-coded version. ### Bug Fixes +- Keep truncated gateway Service names DNS-valid and gateway certificate DNS names aligned with the actual Service name. - Quote product-defined extension and replication role names in lifecycle SQL, and use a product-independent ImageVolume probe name so display names such as `Cosmos DB` do not bypass capability checks. ### Major Features diff --git a/docs/designs/product-neutral-core.md b/docs/designs/product-neutral-core.md index cdfd29da..3a553a5b 100644 --- a/docs/designs/product-neutral-core.md +++ b/docs/designs/product-neutral-core.md @@ -78,9 +78,19 @@ unmodified. The ImageVolume probe uses a product-independent Kubernetes name prefix; display names such as `Cosmos DB` are used only in diagnostics. -Extension and replication role names in the profile are unquoted PostgreSQL -names. The lifecycle SQL helpers quote identifiers and catalog lookup literals, -preserving mixed case and supporting names containing hyphens or quotes. +Profiles are trusted operator configuration, not human-readable input to be +automatically sanitized. Machine identifiers must satisfy their destination's +rules: Kubernetes domains, Service/container/Secret names, environment variable +names, PostgreSQL parameters, and OTel metric names have different constraints. +`ProductProfile.Name` is a display name, not a machine identifier. + +Extension and replication role names are supplied without SQL quoting. The +lifecycle SQL helpers quote identifiers and catalog lookup literals, preserving +mixed case and supporting names containing hyphens or quotes. However, +`ExtensionSpec.Name` is also the CNPG extension mount identifier: CNPG limits it +to 1-59 characters matching `^[a-z0-9]([-a-z0-9_]*[a-z0-9])?$`. SQL quoting does +not relax that separate constraint. Bootstrap SQL is product-authored SQL and +is passed through verbatim. The DocumentDB controllers and webhook in `internal/controller` and `internal/webhook` are now thin wrappers over these packages. They keep their @@ -127,6 +137,18 @@ port changes, the client Service's ports follow it; the Service keeps its assigned cluster IPs and node ports. See [the plugin configuration guide](../developer-guides/sidecar-injector-plugin-configuration.md). +The configured plugin name must match the deployed plugin server's advertised +identity. The shared injector binary advertises +`cnpg-i-sidecar-injector.documentdb.io`; another product can reuse that identity +and binary with its own gateway parameters. Merely changing the profile's +plugin name does not rename the plugin server. + +Gateway certificate DNS names use the same `NamingProfile.ServiceName` helper +as the gateway Service, including its 63-character truncation and removal of +trailing hyphens at the truncation boundary. Truncation preserves the legacy +naming convention, not uniqueness: product CRD name validation should reserve +room for the Service prefix to avoid collisions between truncated names. + ## Wire contracts kept on purpose These names are visible to running clusters or users. Renaming them would @@ -154,6 +176,17 @@ helpers (`ValidateSchemaVersion`, `ValidateImageRollback`, `ValidateStorageResize`, `ValidateRestoreCompatibility`, `NewImageVolumeProbe`). It adds no behavior of its own. +For a checked-out or mirrored source tree, the consuming module can use: + +```go +require github.com/documentdb/documentdb-operator v0.0.0 + +replace github.com/documentdb/documentdb-operator => ../documentdb-kubernetes-operator/operator/src +``` + +The replacement points at `operator/src`, the Go module, not the repository +root. Pin the checkout or mirror to the reviewed source commit. + A second product: 1. Defines its own CRD and a `core.ClusterAdapter[*MyCR]` that returns its @@ -161,6 +194,8 @@ A second product: `core.ClusterIntent`, `core.ClusterSpec`, and `core.ClusterStatus`. Backup support adds a `core.BackupAdapter[*MyBackup]` and `core.ScheduledBackupAdapter[*MySchedule]`. + `Intent.Product` must match `Profile()`, and overlapping operational inputs + such as monitoring must agree between `Intent` and `Spec`. 2. Wires the generic reconcilers in its `main.go`: ```go @@ -210,6 +245,11 @@ CNPG `Cluster` creation with the second product's extension, and status, using only `pkg/core`. `pkg/core/replication_test.go` drives the cluster and backup reconcilers with replica, not-present, and promotion-pending contexts built with `core.NewReplicationContext`. +`pkg/core/second_product_test.go` reuses a `Cosmos DB` profile across rendering, +cluster reconciliation, gateway contract updates/removal, TLS certificates, +ImageVolume admission, and backup/PV restore metadata. It includes a separate +`AccountBackup` custom resource, a hyphenated extension name, a mixed-case +replication role, and Kubernetes name validation at probe and Service boundaries. `pkg/core/example_test.go` renders a non-DocumentDB cluster through the facade. `internal/cnpg/second_product_test.go` renders a synthetic `pgcosmos` product end to end and fails if any DocumentDB string other than the documented diff --git a/operator/cnpg-plugins/sidecar-injector/internal/lifecycle/lifecycle_test.go b/operator/cnpg-plugins/sidecar-injector/internal/lifecycle/lifecycle_test.go index 2e3474d6..1dea50ec 100644 --- a/operator/cnpg-plugins/sidecar-injector/internal/lifecycle/lifecycle_test.go +++ b/operator/cnpg-plugins/sidecar-injector/internal/lifecycle/lifecycle_test.go @@ -486,14 +486,16 @@ func TestLifecycleHookDefaultGatewayContract(t *testing.T) { func TestLifecycleHookCustomGatewayContract(t *testing.T) { containers := runCreateHook(t, map[string]string{ - "gatewayImage": "cosmos-gw:1", - "gatewayContainerName": "pgcosmos-gateway", - "gatewayPort": "8081", - "gatewayArgs": `["--port","8081"]`, - "gatewayUsernameEnv": "COSMOS_USER", - "gatewayPasswordEnv": "COSMOS_KEY", - "otelCollectorImage": "otel:1", - "otelConfigMapName": "otel-config", + "gatewayImage": "cosmos-gw:1", + "gatewayContainerName": "pgcosmos-gateway", + "gatewayPort": "8081", + "gatewayArgs": `["--port","8081"]`, + "gatewayUsernameEnv": "COSMOS_USER", + "gatewayPasswordEnv": "COSMOS_KEY", + "documentDbCredentialSecret": "cosmos-credentials", + "gatewayTLSSecret": "cosmos-gateway-tls", + "otelCollectorImage": "otel:1", + "otelConfigMapName": "otel-config", }) if _, ok := containers[gatewayContainerName]; ok { t.Fatal("default gateway container name leaked") @@ -512,5 +514,18 @@ func TestLifecycleHookCustomGatewayContract(t *testing.T) { if names[0] != "COSMOS_USER" || names[1] != "COSMOS_KEY" { t.Errorf("env = %v", names) } + for i, key := range []string{"username", "password"} { + source := gateway.Env[i].ValueFrom + if source == nil || source.SecretKeyRef == nil || source.SecretKeyRef.Name != "cosmos-credentials" || + source.SecretKeyRef.Key != key { + t.Errorf("%s does not reference the product credential Secret key %s", gateway.Env[i].Name, key) + } + } + if len(gateway.VolumeMounts) != 1 || gateway.VolumeMounts[0].Name != "gateway-tls" || + gateway.VolumeMounts[0].MountPath != "/tls" || !gateway.VolumeMounts[0].ReadOnly { + t.Errorf("gateway TLS mount = %+v", gateway.VolumeMounts) + } + assertEnvValue(t, gateway.Env, "CERT_PATH", "/tls/tls.crt") + assertEnvValue(t, gateway.Env, "KEY_FILE", "/tls/tls.key") assertEnvValue(t, gateway.Env, "OTEL_METRICS_ENABLED", "true") } diff --git a/operator/src/internal/engine/certificate.go b/operator/src/internal/engine/certificate.go index 270b9374..d59144b0 100644 --- a/operator/src/internal/engine/certificate.go +++ b/operator/src/internal/engine/certificate.go @@ -184,7 +184,7 @@ func (r *CertificateReconciler[T]) ensureCertManagerManagedCert(ctx context.Cont secretName = obj.GetName() + "-gateway-cert-tls" } - serviceBase := r.Adapter.Profile().Naming.ServicePrefix + obj.GetName() + serviceBase := r.Adapter.Profile().Naming.ServiceName(obj.GetName()) baseDNS := []string{serviceBase, serviceBase + "." + obj.GetNamespace(), serviceBase + "." + obj.GetNamespace() + ".svc"} dnsSet := map[string]struct{}{} finalDNS := []string{} @@ -284,7 +284,7 @@ func (r *CertificateReconciler[T]) ensureSelfSignedCert(ctx context.Context, obj } } - serviceBase := r.Adapter.Profile().Naming.ServicePrefix + obj.GetName() + serviceBase := r.Adapter.Profile().Naming.ServiceName(obj.GetName()) dnsNames := []string{ serviceBase, serviceBase + "." + namespace, diff --git a/operator/src/internal/product/cluster.go b/operator/src/internal/product/cluster.go index f061b924..559f6a56 100644 --- a/operator/src/internal/product/cluster.go +++ b/operator/src/internal/product/cluster.go @@ -4,6 +4,8 @@ package product import ( + "strings" + "sigs.k8s.io/controller-runtime/pkg/client" dbpreview "github.com/documentdb/documentdb-operator/api/preview" @@ -48,11 +50,11 @@ func (n NamingProfile) SchemaVersionAnnotation() string { return n.Domain + "/sc func (n NamingProfile) Finalizer() string { return n.Domain + "/pv-retention-finalizer" } // ServiceName is the gateway Service name for a custom resource, truncated to -// the 63-character Kubernetes name limit. +// the 63-character Kubernetes name limit without leaving a trailing hyphen. func (n NamingProfile) ServiceName(clusterName string) string { name := n.ServicePrefix + clusterName if len(name) > 63 { - name = name[:63] + name = strings.TrimRight(name[:63], "-") } return name } diff --git a/operator/src/internal/product/cluster_test.go b/operator/src/internal/product/cluster_test.go index 789dd741..5540d996 100644 --- a/operator/src/internal/product/cluster_test.go +++ b/operator/src/internal/product/cluster_test.go @@ -8,6 +8,7 @@ import ( "testing" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/validation" dbpreview "github.com/documentdb/documentdb-operator/api/preview" util "github.com/documentdb/documentdb-operator/internal/utils" @@ -33,6 +34,17 @@ func TestDocumentDBNamingMatchesWireContract(t *testing.T) { } } +func TestServiceNameTruncationKeepsValidDNSLabel(t *testing.T) { + n := NamingProfile{ServicePrefix: "cosmos-gateway-"} + got := n.ServiceName(strings.Repeat("a", 47) + "-a") + if errs := validation.IsDNS1035Label(got); len(errs) != 0 { + t.Fatalf("truncating a valid cluster name produced invalid Service name %q: %v", got, errs) + } + if want := n.ServicePrefix + strings.Repeat("a", 47); got != want { + t.Fatalf("ServiceName = %q, want %q", got, want) + } +} + func TestDocumentDBAdapterStatusRoundTrip(t *testing.T) { a := DocumentDBAdapter{} db := &dbpreview.DocumentDB{} diff --git a/operator/src/internal/product/profile.go b/operator/src/internal/product/profile.go index 182c02c2..0e24c533 100644 --- a/operator/src/internal/product/profile.go +++ b/operator/src/internal/product/profile.go @@ -69,7 +69,9 @@ type GatewayProfile struct { // extension image through CNPG's ImageVolume extension support. type ExtensionSpec struct { // Name is the CNPG extension entry name and the PostgreSQL extension name - // used for CREATE/ALTER EXTENSION and version discovery. + // used for CREATE/ALTER EXTENSION and version discovery. It must satisfy + // CNPG's extension-name constraints (1-59 characters, lowercase letters, + // digits, internal hyphens/underscores, and alphanumeric ends). Name string // DynamicLibraryPath, ExtensionControlPath, and LdLibraryPath are the // image-relative paths CNPG adds to the PostgreSQL search paths. diff --git a/operator/src/pkg/core/engine_test.go b/operator/src/pkg/core/engine_test.go index 35563c78..4edf03e5 100644 --- a/operator/src/pkg/core/engine_test.go +++ b/operator/src/pkg/core/engine_test.go @@ -40,16 +40,20 @@ type accountSpec struct { } type accountStatus struct { - State string `json:"state,omitempty"` - Endpoint string `json:"endpoint,omitempty"` - SchemaVersion string `json:"schemaVersion,omitempty"` - Engine string `json:"engine,omitempty"` - Gateway string `json:"gateway,omitempty"` + State string `json:"state,omitempty"` + Endpoint string `json:"endpoint,omitempty"` + SchemaVersion string `json:"schemaVersion,omitempty"` + Engine string `json:"engine,omitempty"` + Gateway string `json:"gateway,omitempty"` + TLS *dbpreview.TLSStatus `json:"tls,omitempty"` } func (a *account) DeepCopyObject() runtime.Object { out := *a a.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + if a.Status.TLS != nil { + out.Status.TLS = a.Status.TLS.DeepCopy() + } return &out } @@ -114,6 +118,7 @@ func (accountAdapter) Status(obj *account) core.ClusterStatus { SchemaVersion: obj.Status.SchemaVersion, ExtensionImage: obj.Status.Engine, GatewayImage: obj.Status.Gateway, + TLS: obj.Status.TLS, } } @@ -123,6 +128,7 @@ func (accountAdapter) SetStatus(obj *account, s core.ClusterStatus) { obj.Status.SchemaVersion = s.SchemaVersion obj.Status.Engine = s.ExtensionImage obj.Status.Gateway = s.GatewayImage + obj.Status.TLS = s.TLS } func (accountAdapter) ConnectionString(obj *account, host string, port int32, _ bool) string { @@ -140,7 +146,8 @@ func TestClusterReconcilerSecondProduct(t *testing.T) { if err := cnpgv1.AddToScheme(scheme); err != nil { t.Fatal(err) } - scheme.AddKnownTypes(accountGV, &account{}, &accountList{}) + scheme.AddKnownTypeWithName(accountGV.WithKind("Account"), &account{}) + scheme.AddKnownTypeWithName(accountGV.WithKind("AccountList"), &accountList{}) metav1.AddToGroupVersion(scheme, accountGV) acct := &account{ @@ -251,7 +258,8 @@ func TestSecondProductMonitoringConfig(t *testing.T) { if err := clientgoscheme.AddToScheme(scheme); err != nil { t.Fatal(err) } - scheme.AddKnownTypes(accountGV, &account{}, &accountList{}) + scheme.AddKnownTypeWithName(accountGV.WithKind("Account"), &account{}) + scheme.AddKnownTypeWithName(accountGV.WithKind("AccountList"), &accountList{}) metav1.AddToGroupVersion(scheme, accountGV) acct := &account{ObjectMeta: metav1.ObjectMeta{Name: "acct", Namespace: "ns", UID: "uid-1"}} c := fake.NewClientBuilder().WithScheme(scheme).Build() diff --git a/operator/src/pkg/core/replication_test.go b/operator/src/pkg/core/replication_test.go index a9f23eee..3f021405 100644 --- a/operator/src/pkg/core/replication_test.go +++ b/operator/src/pkg/core/replication_test.go @@ -91,7 +91,8 @@ func replicationScheme(t *testing.T) *runtime.Scheme { t.Fatal(err) } } - scheme.AddKnownTypes(accountGV, &account{}, &accountList{}) + scheme.AddKnownTypeWithName(accountGV.WithKind("Account"), &account{}) + scheme.AddKnownTypeWithName(accountGV.WithKind("AccountList"), &accountList{}) metav1.AddToGroupVersion(scheme, accountGV) return scheme } diff --git a/operator/src/pkg/core/second_product_test.go b/operator/src/pkg/core/second_product_test.go new file mode 100644 index 00000000..2bb026e5 --- /dev/null +++ b/operator/src/pkg/core/second_product_test.go @@ -0,0 +1,544 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package core_test + +import ( + "context" + "encoding/json" + "fmt" + "slices" + "strings" + "sync/atomic" + "testing" + + cmapi "github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1" + cnpgv1 "github.com/cloudnative-pg/cloudnative-pg/api/v1" + "github.com/go-logr/logr" + snapshotv1 "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" + corev1 "k8s.io/api/core/v1" + apivalidation "k8s.io/apimachinery/pkg/api/validation" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/apimachinery/pkg/util/validation/field" + "k8s.io/client-go/tools/record" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" + + dbpreview "github.com/documentdb/documentdb-operator/api/preview" + "github.com/documentdb/documentdb-operator/pkg/core" +) + +// cosmosAdapter reuses one realistic profile across the public rendering, +// reconciliation, TLS, and admission surfaces. +type cosmosAdapter struct { + accountAdapter + profile core.ProductProfile + spec core.ClusterSpec +} + +func newCosmosAdapter() *cosmosAdapter { + return &cosmosAdapter{ + profile: core.ProductProfile{ + Name: "Cosmos DB", DefaultTag: "1.3.0", + ExtensionImageRepo: "registry.example.com/cosmos/pgcosmos", + GatewayImageRepo: "registry.example.com/cosmos/gateway", + DefaultCredentialSecret: "cosmos-credentials", + SidecarInjectorPlugin: core.DocumentDBProfile().SidecarInjectorPlugin, + Postgres: core.PostgresProfile{ + Extension: core.ExtensionSpec{ + Name: "pg-cosmos", DynamicLibraryPath: []string{"lib"}, + ExtensionControlPath: []string{"share"}, LdLibraryPath: []string{"lib", "system"}, + }, + AdditionalLibraries: []string{"pgcosmos_core"}, + BootstrapSQL: []string{`CREATE EXTENSION "pg-cosmos" CASCADE`}, + PgHBA: []string{"host all all localhost trust", "hostssl replication streaming_replica all cert"}, + ProtectedParameters: map[string]string{"pgcosmos.locked": "operator"}, + ReplicationAdminRole: "Cosmos-Admin", + }, + Gateway: core.GatewayProfile{ + ContainerName: "pgcosmos-gateway", Port: 8081, Args: []string{"--port", "8081"}, + UsernameEnv: "COSMOS_USER", PasswordEnv: "COSMOS_KEY", + }, + Naming: core.NamingProfile{ + Domain: "cosmos.example.com", ServicePrefix: "cosmos-gateway-", Kind: "Account", + MetricClusterAttribute: "cosmos.cluster", HealthMetricName: "cosmos.postgres.up", + OperatorName: "cosmos-operator", + }, + }, + spec: core.ClusterSpec{ + ServiceType: "LoadBalancer", StorageClass: "onprem-csi", + Monitoring: &dbpreview.MonitoringSpec{Enabled: true}, + }, + } +} + +func (a *cosmosAdapter) Profile() core.ProductProfile { return a.profile } +func (a *cosmosAdapter) Spec(*account) core.ClusterSpec { return a.spec } + +func (a *cosmosAdapter) Intent(obj *account) core.ClusterIntent { + in := a.accountAdapter.Intent(obj) + in.Product = a.Profile() + in.Images = core.Images{ + PostgresExtension: a.profile.ExtensionImageRepo + ":" + a.profile.DefaultTag, + Gateway: a.profile.GatewayImageRepo + ":" + a.profile.DefaultTag, + Postgres: "registry.example.com/postgres:18", + PullSecrets: []corev1.LocalObjectReference{{Name: "registry-pull"}}, + } + in.CredentialSecret = a.profile.DefaultCredentialSecret + in.SidecarInjectorPlugin = a.profile.SidecarInjectorPlugin + in.Resources = core.Resources{Memory: "4Gi", CPU: "2"} + in.Monitoring = a.spec.Monitoring + in.Postgres = core.Postgres{ + PostInitSQL: []string{"SELECT 1"}, + Parameters: map[string]string{"pgcosmos.locked": "user", "work_mem": "8MB"}, + } + in.TLS.Postgres = &cnpgv1.CertificatesConfiguration{ + ServerTLSSecret: "cosmos-pg-server", ServerCASecret: "cosmos-pg-ca", + ReplicationTLSSecret: "cosmos-pg-replication", ClientCASecret: "cosmos-pg-ca", + } + if obj.Status.TLS != nil && obj.Status.TLS.Ready { + in.TLS.GatewaySecret = obj.Status.TLS.SecretName + } + return in +} + +func (*cosmosAdapter) ConnectionString(_ *account, host string, port int32, _ bool) string { + return fmt.Sprintf("AccountEndpoint=https://%s:%d/;", host, port) +} + +func TestCosmosPublicLifecycle(t *testing.T) { + t.Setenv("GATEWAY_PORT", "10260") + ctx := context.Background() + a := newCosmosAdapter() + acct := &account{ + ObjectMeta: metav1.ObjectMeta{Name: "acct", Namespace: "ns", UID: "uid-1"}, + Spec: accountSpec{Instances: 3, PvcSize: "20Gi"}, + Status: accountStatus{TLS: &dbpreview.TLSStatus{Ready: true, SecretName: "cosmos-gateway-tls"}}, + } + controlled := true + svc := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: a.profile.Naming.ServiceName(acct.Name), Namespace: acct.Namespace, + OwnerReferences: []metav1.OwnerReference{{ + APIVersion: accountGV.String(), Kind: "Account", Name: acct.Name, + UID: acct.UID, Controller: &controlled, + }}, + }, + Spec: corev1.ServiceSpec{ + Type: corev1.ServiceTypeLoadBalancer, ClusterIP: "10.0.0.10", + Ports: []corev1.ServicePort{{Name: "gateway", Protocol: corev1.ProtocolTCP, Port: 8081, TargetPort: intstr.FromInt32(8081), NodePort: 30081}}, + }, + Status: corev1.ServiceStatus{LoadBalancer: corev1.LoadBalancerStatus{Ingress: []corev1.LoadBalancerIngress{{IP: "203.0.113.10"}}}}, + } + scheme := replicationScheme(t) + c := fake.NewClientBuilder().WithScheme(scheme).WithObjects(acct, svc). + WithStatusSubresource(&account{}, &cnpgv1.Cluster{}).Build() + installed, granted := "1.2-3", false + var sql []string + r := &core.ClusterReconciler[*account]{ + Client: c, Scheme: scheme, Adapter: a, Recorder: record.NewFakeRecorder(20), + SQLExecutor: func(_ context.Context, _ *cnpgv1.Cluster, cmd string) (string, error) { + sql = append(sql, cmd) + switch cmd { + case "SELECT default_version, installed_version FROM pg_available_extensions WHERE name = 'pg-cosmos'": + return "default_version | installed_version\n---+---\n1.3-0 | " + installed, nil + case "SELECT 1 FROM pg_roles WHERE rolname = 'streaming_replica' AND pg_has_role('streaming_replica', 'Cosmos-Admin', 'USAGE');": + if granted { + return "(1 row)", nil + } + return "(0 rows)", nil + case `GRANT "Cosmos-Admin" TO streaming_replica;`: + granted = true + return "", nil + case `ALTER EXTENSION "pg-cosmos" UPDATE`: + installed = "1.3-0" + return "", nil + default: + return "", fmt.Errorf("unexpected SQL: %s", cmd) + } + }, + } + req := ctrl.Request{NamespacedName: types.NamespacedName{Name: acct.Name, Namespace: acct.Namespace}} + reconcile := func() { + t.Helper() + if result, err := r.Reconcile(ctx, req); err != nil || result.RequeueAfter == core.RequeueAfterShort { + t.Fatalf("reconcile = %+v, %v", result, err) + } + } + for range 2 { + reconcile() + } + cluster := &cnpgv1.Cluster{} + if err := c.Get(ctx, req.NamespacedName, cluster); err != nil { + t.Fatal(err) + } + pg := cluster.Spec.PostgresConfiguration + if len(pg.Extensions) != 1 || pg.Extensions[0].Name != "pg-cosmos" || + core.ExtensionImage(cluster, "pg-cosmos") != a.profile.ExtensionImageRepo+":1.3.0" { + t.Fatalf("extensions = %+v", pg.Extensions) + } + if !slices.Equal(pg.AdditionalLibraries, a.profile.Postgres.AdditionalLibraries) || + !slices.Equal(pg.PgHBA, a.profile.Postgres.PgHBA) || + pg.Parameters["pgcosmos.locked"] != "operator" || pg.Parameters["work_mem"] != "8MB" { + t.Fatalf("PostgreSQL contract = %+v", pg) + } + pgJSON, err := json.Marshal(pg) + if err != nil { + t.Fatal(err) + } + if strings.Contains(strings.ToLower(string(pgJSON)), "documentdb") { + t.Fatalf("DocumentDB PostgreSQL settings leaked: %s", pgJSON) + } + if got := cluster.Spec.Bootstrap.InitDB.PostInitSQL; !slices.Equal(got, []string{`CREATE EXTENSION "pg-cosmos" CASCADE`, "SELECT 1"}) { + t.Fatalf("bootstrap SQL = %v", got) + } + if got := cluster.Spec.Certificates; got == nil || got.ServerTLSSecret != "cosmos-pg-server" || got.ReplicationTLSSecret != "cosmos-pg-replication" { + t.Fatalf("PostgreSQL TLS = %+v", got) + } + params := cluster.Spec.Plugins[0].Parameters + if params["documentDbCredentialSecret"] != "cosmos-credentials" || params["gatewayTLSSecret"] != "cosmos-gateway-tls" { + t.Fatalf("credential/TLS references = %v", params) + } + cm := &corev1.ConfigMap{} + if err := c.Get(ctx, client.ObjectKey{Name: "acct-otel-config", Namespace: "ns"}, cm); err != nil { + t.Fatal(err) + } + if !strings.Contains(cm.Data["static.yaml"], "metric_name: cosmos.postgres.up\n") || + !strings.Contains(cm.Data["dynamic.yaml"], "key: cosmos.cluster") { + t.Fatalf("monitoring config = %v", cm.Data) + } + cluster.Status.CurrentPrimary = "acct-1" + cluster.Status.Phase = "Cluster in healthy state" + cluster.Status.InstancesStatus = map[cnpgv1.PodStatus][]string{cnpgv1.PodHealthy: {"acct-1"}} + if err := c.Status().Update(ctx, cluster); err != nil { + t.Fatal(err) + } + reconcile() + if !granted || installed != "1.2-3" { + t.Fatalf("replication grant or two-phase guard failed: granted %t, schema %s, SQL %q", granted, installed, sql) + } + a.spec.SchemaVersion = "auto" + reconcile() + if installed != "1.3-0" { + t.Fatalf("automatic schema upgrade did not execute: %q", sql) + } + got := &account{} + if err := c.Get(ctx, req.NamespacedName, got); err != nil { + t.Fatal(err) + } + if got.Status.SchemaVersion != "1.3.0" || got.Status.Engine != a.profile.ExtensionImageRepo+":1.3.0" || + !slices.Contains(got.Finalizers, "cosmos.example.com/pv-retention-finalizer") { + t.Fatalf("product status/identity = %+v", got) + } + + for _, gateway := range []core.GatewayProfile{ + {ContainerName: "pgcosmos-gateway-v2", Port: 8082, Args: []string{}, UsernameEnv: "COSMOS_USER2", PasswordEnv: "COSMOS_KEY2"}, + {}, + } { + a.profile.Gateway = gateway + if err := c.Get(ctx, req.NamespacedName, cluster); err != nil { + t.Fatal(err) + } + previousRestart := cluster.Annotations["kubectl.kubernetes.io/restartedAt"] + reconcile() + if err := c.Get(ctx, req.NamespacedName, cluster); err != nil { + t.Fatal(err) + } + if restart := cluster.Annotations["kubectl.kubernetes.io/restartedAt"]; restart == "" || restart == previousRestart { + t.Fatal("gateway contract change did not request a restart") + } + desired := core.RenderCluster(req, a.Intent(got), acct.Name, "onprem-csi", true, logr.Discard()) + for _, key := range []string{"gatewayContainerName", "gatewayPort", "gatewayArgs", "gatewayUsernameEnv", "gatewayPasswordEnv"} { + actual, exists := cluster.Spec.Plugins[0].Parameters[key] + want, wantExists := desired.Spec.Plugins[0].Parameters[key] + if actual != want || exists != wantExists { + t.Errorf("%s = %q (set %t), want %q (set %t)", key, actual, exists, want, wantExists) + } + } + if err := c.Get(ctx, client.ObjectKeyFromObject(svc), svc); err != nil { + t.Fatal(err) + } + port := core.GatewayPort(a.profile) + if svc.Spec.Ports[0].Port != port || svc.Spec.Ports[0].TargetPort.IntVal != port || + svc.Spec.Ports[0].NodePort != 30081 || svc.Spec.ClusterIP != "10.0.0.10" { + t.Fatalf("Service did not converge while preserving allocation: %+v", svc.Spec) + } + if err := c.Get(ctx, req.NamespacedName, got); err != nil { + t.Fatal(err) + } + if want := fmt.Sprintf("AccountEndpoint=https://203.0.113.10:%d/;", port); got.Status.Endpoint != want { + t.Fatalf("endpoint = %q, want %q", got.Status.Endpoint, want) + } + version := cluster.ResourceVersion + reconcile() + if err := c.Get(ctx, req.NamespacedName, cluster); err != nil { + t.Fatal(err) + } + if cluster.ResourceVersion != version { + t.Fatal("unchanged desired state repeatedly patched/restarted the CNPG cluster") + } + } + in := a.Intent(got) + if errs := core.ValidateResources(in.Resources, true, field.NewPath("spec", "compute"), core.DefaultSplitConfig()); len(errs) != 0 { + t.Fatalf("valid resource envelope rejected: %v", errs) + } + if errs := core.ValidateImageRollback(got.Status.SchemaVersion, "1.2.3"); len(errs) == 0 { + t.Fatal("binary rollback below the migrated Cosmos schema was allowed") + } +} + +func TestCosmosPublicImageVolume(t *testing.T) { + for _, behavior := range []string{"supported", "rejected", "pruned", "baseline-blocked"} { + t.Run(behavior, func(t *testing.T) { + a := newCosmosAdapter() + creates := 0 + c := fake.NewClientBuilder().WithScheme(replicationScheme(t)). + WithInterceptorFuncs(interceptor.Funcs{ + Create: func(_ context.Context, _ client.WithWatch, obj client.Object, opts ...client.CreateOption) error { + creates++ + pod := obj.(*corev1.Pod) + if errs := apivalidation.NameIsDNSSubdomain(pod.GenerateName, true); len(errs) != 0 { + t.Fatalf("invalid probe name: %v", errs) + } + options := (&client.CreateOptions{}).ApplyOptions(opts) + if !slices.Equal(options.DryRun, []string{metav1.DryRunAll}) { + t.Fatal("probe was not a server-side dry run") + } + if behavior == "baseline-blocked" { + return fmt.Errorf("namespace admission policy rejected baseline") + } + for i := range pod.Spec.Volumes { + if pod.Spec.Volumes[i].Image == nil { + continue + } + if behavior == "rejected" { + return fmt.Errorf("ImageVolume disabled") + } + if behavior == "pruned" { + pod.Spec.Volumes[i].Image = nil + } + } + return nil + }, + }).Build() + var confirmed atomic.Bool + probe := core.NewImageVolumeProbe(c, &confirmed, a.profile.Name, logr.Discard()) + err := probe.Ensure(context.Background(), "ns") + switch behavior { + case "supported": + if err != nil || !confirmed.Load() { + t.Fatalf("support was not cached: %v", err) + } + if err := probe.Ensure(context.Background(), "ns"); err != nil { + t.Fatal(err) + } + case "baseline-blocked": + if err != nil || confirmed.Load() || creates != 1 { + t.Fatalf("genuine admission rejection must remain inconclusive and uncached: %v", err) + } + return + default: + if err == nil || !strings.Contains(err.Error(), a.profile.Name) || confirmed.Load() { + t.Fatalf("unsupported ImageVolume was not rejected for %s: %v", a.profile.Name, err) + } + } + if creates != 2 { + t.Fatalf("probe creates = %d, want 2", creates) + } + }) + } +} + +func TestCosmosCertificateServiceDNS(t *testing.T) { + for _, mode := range []string{"SelfSigned", "CertManager"} { + for _, name := range []string{"acct", strings.Repeat("a", 50), strings.Repeat("a", 47) + "-a"} { + t.Run(mode+"/"+name, func(t *testing.T) { + ctx := context.Background() + a := newCosmosAdapter() + a.spec.GatewayTLS = &dbpreview.GatewayTLS{ + Mode: mode, + CertManager: &dbpreview.CertManagerTLS{IssuerRef: dbpreview.IssuerRef{Name: "corporate-ca"}}, + } + scheme := replicationScheme(t) + if err := cmapi.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + acct := &account{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: "ns", UID: "uid-1"}} + c := fake.NewClientBuilder().WithScheme(scheme).WithObjects(acct).WithStatusSubresource(&account{}).Build() + r := &core.CertificateReconciler[*account]{Client: c, Scheme: scheme, Adapter: a} + if _, err := r.ReconcileCertificates(ctx, acct); err != nil { + t.Fatal(err) + } + cert := &cmapi.Certificate{} + if err := c.Get(ctx, client.ObjectKey{Name: name + "-gateway-cert", Namespace: "ns"}, cert); err != nil { + t.Fatal(err) + } + service := a.profile.Naming.ServiceName(name) + if errs := apivalidation.NameIsDNS1035Label(service, false); len(errs) != 0 { + t.Fatalf("invalid Service DNS name %q: %v", service, errs) + } + for _, dns := range []string{service, service + ".ns", service + ".ns.svc"} { + if !slices.Contains(cert.Spec.DNSNames, dns) { + t.Errorf("certificate DNS names %v do not cover actual Service DNS %q", cert.Spec.DNSNames, dns) + } + } + }) + } + } +} + +type accountBackup struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + Spec dbpreview.BackupSpec `json:"spec,omitempty"` + Status dbpreview.BackupStatus `json:"status,omitempty"` +} + +func (b *accountBackup) DeepCopyObject() runtime.Object { + out := *b + b.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + b.Spec.DeepCopyInto(&out.Spec) + b.Status.DeepCopyInto(&out.Status) + return &out +} + +type accountBackupList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []accountBackup `json:"items"` +} + +func (l *accountBackupList) DeepCopyObject() runtime.Object { + out := *l + l.ListMeta.DeepCopyInto(&out.ListMeta) + out.Items = make([]accountBackup, len(l.Items)) + for i := range l.Items { + out.Items[i] = *l.Items[i].DeepCopyObject().(*accountBackup) + } + return &out +} + +type accountBackupAdapter struct{} + +func (accountBackupAdapter) New() *accountBackup { return &accountBackup{} } +func (accountBackupAdapter) NewList() client.ObjectList { return &accountBackupList{} } +func (accountBackupAdapter) Spec(b *accountBackup) *dbpreview.BackupSpec { return &b.Spec } +func (accountBackupAdapter) Status(b *accountBackup) *dbpreview.BackupStatus { return &b.Status } +func (accountBackupAdapter) Items(list client.ObjectList) []*accountBackup { + items := list.(*accountBackupList).Items + result := make([]*accountBackup, len(items)) + for i := range items { + result[i] = &items[i] + } + return result +} + +func TestCosmosBackupAndRestoreMetadata(t *testing.T) { + ctx := context.Background() + a := newCosmosAdapter() + scheme := replicationScheme(t) + scheme.AddKnownTypeWithName(accountGV.WithKind("AccountBackup"), &accountBackup{}) + scheme.AddKnownTypeWithName(accountGV.WithKind("AccountBackupList"), &accountBackupList{}) + acct := &account{ + ObjectMeta: metav1.ObjectMeta{Name: "acct", Namespace: "ns", UID: "uid-1"}, + Spec: accountSpec{Instances: 1, PvcSize: "20Gi"}, + Status: accountStatus{SchemaVersion: "1.2.3"}, + } + backup := &accountBackup{ + ObjectMeta: metav1.ObjectMeta{Name: "nightly", Namespace: "ns", UID: "backup-uid"}, + Spec: dbpreview.BackupSpec{Cluster: cnpgv1.LocalObjectReference{Name: acct.Name}}, + } + vsc := &snapshotv1.VolumeSnapshotClass{ + ObjectMeta: metav1.ObjectMeta{Name: "onprem", Annotations: map[string]string{"snapshot.storage.kubernetes.io/is-default-class": "true"}}, + Driver: "csi.example.com", + } + c := fake.NewClientBuilder().WithScheme(scheme).WithObjects(acct, backup, vsc). + WithStatusSubresource(&account{}, &accountBackup{}, &cnpgv1.Backup{}).Build() + r := &core.BackupReconciler[*account, *accountBackup]{ + Client: c, Scheme: scheme, Recorder: record.NewFakeRecorder(10), + Cluster: a, Backups: accountBackupAdapter{}, + } + req := ctrl.Request{NamespacedName: client.ObjectKeyFromObject(backup)} + if _, err := r.Reconcile(ctx, req); err != nil { + t.Fatal(err) + } + cnpgBackup := &cnpgv1.Backup{} + if err := c.Get(ctx, req.NamespacedName, cnpgBackup); err != nil { + t.Fatal(err) + } + owner := metav1.GetControllerOf(cnpgBackup) + if owner == nil || owner.APIVersion != accountGV.String() || owner.Kind != "AccountBackup" || + owner.UID != backup.UID || cnpgBackup.Spec.Cluster.Name != acct.Name { + t.Fatalf("backup ownership/target = %+v", cnpgBackup) + } + cnpgBackup.Status.Phase = cnpgv1.BackupPhaseCompleted + completed := metav1.Now() + cnpgBackup.Status.StoppedAt = &completed + if err := c.Status().Update(ctx, cnpgBackup); err != nil { + t.Fatal(err) + } + if _, err := r.Reconcile(ctx, req); err != nil { + t.Fatal(err) + } + if err := c.Get(ctx, req.NamespacedName, backup); err != nil { + t.Fatal(err) + } + if backup.Status.SchemaVersion != "1.2.3" || backup.Status.Phase != cnpgv1.BackupPhaseCompleted { + t.Fatalf("product backup status = %+v", backup.Status) + } + + pv := &corev1.PersistentVolume{ObjectMeta: metav1.ObjectMeta{Name: "retained-cosmos-pv"}} + pvReconciler := &core.PersistentVolumeReconciler[*account]{Client: c, Adapter: a} + if !pvReconciler.ApplyDesiredPVConfiguration(ctx, pv, acct) { + t.Fatal("PV ownership/schema metadata was not applied") + } + if pv.Labels["cosmos.example.com/cluster"] != acct.Name || pv.Labels["cosmos.example.com/namespace"] != acct.Namespace || + pv.Annotations["cosmos.example.com/schema-version"] != "1.2.3" { + t.Fatalf("product PV metadata = %+v", pv.ObjectMeta) + } + if err := c.Create(ctx, pv); err != nil { + t.Fatal(err) + } + base := field.NewPath("spec", "restore") + terms := core.ValidationTerms{Product: a.profile.Name, VersionField: "spec.engineVersion", ImageField: "spec.image.engine"} + for _, src := range []core.RestoreSource{ + core.BackupRestoreSource(base, backup.Name), + core.PVRestoreSource(base, pv.Name), + } { + schema, warnings := core.RestoreSourceSchemaVersion(ctx, c, src, a.profile.Naming, func() (string, error) { + return backup.Status.SchemaVersion, nil + }) + if schema != "1.2.3" || len(warnings) != 0 { + t.Fatalf("restore source %s: schema %q, warnings %v", src.Kind, schema, warnings) + } + if warnings, errs := core.ValidateRestoreCompatibility("1.2.3", schema, src, terms); len(warnings) != 0 || len(errs) != 0 { + t.Fatalf("compatible restore rejected: %v, %v", warnings, errs) + } + _, errs := core.ValidateRestoreCompatibility("1.1.0", schema, src, terms) + if len(errs) != 1 || !strings.Contains(errs[0].Detail, "Cosmos DB") || !strings.HasPrefix(errs[0].Field, "spec.restore.") { + t.Fatalf("unsafe restore was not rejected with product-specific diagnostics: %v", errs) + } + } + for _, recovery := range []*core.Recovery{{BackupName: backup.Name}, {PersistentVolumeName: pv.Name}} { + in := a.Intent(acct) + in.Bootstrap.Recovery = recovery + rendered := core.RenderCluster(ctrl.Request{NamespacedName: client.ObjectKeyFromObject(acct)}, in, acct.Name, "onprem-csi", true, logr.Discard()) + got := rendered.Spec.Bootstrap.Recovery + if got == nil { + t.Fatal("recovery intent rendered default initdb") + } + if recovery.BackupName != "" { + if got.Backup == nil || got.Backup.Name != backup.Name { + t.Fatalf("backup recovery = %+v", got) + } + } else if got.VolumeSnapshots == nil || + got.VolumeSnapshots.Storage.Kind != "PersistentVolumeClaim" || got.VolumeSnapshots.Storage.Name == "" { + t.Fatalf("retained PV recovery = %+v", got) + } + } +} From cd3071ea8f5ee409e963ef87584f84832c6b8e50 Mon Sep 17 00:00:00 2001 From: Rayhan Hossain Date: Mon, 26 Oct 2026 10:14:49 -0700 Subject: [PATCH 26/27] fix(core): keep long gateway Service names unique Plain truncation to 63 characters made cluster names that differ only past the cut share one gateway Service. UpsertService returned the other owner's Service as success, so the second cluster published the first cluster's endpoint. Shorten long names with a stable FNV-32a hash of the cluster name instead. A cluster that already controls a plainly truncated Service from an earlier release keeps it, so existing endpoints and certificate DNS names do not change. UpsertService now refuses a same-named Service controlled by another owner, and certificate DNS names follow the resolved Service name. Align the consumers that compute or look up the name: the e2e port-forward helper mirrors the hashed name, and kubectl documentdb status matches the Service by its controlling DocumentDB instead of a name substring that could select another cluster's Service. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 33b3cb3c-1b25-4ffb-bff1-ebf0f3d04058 Signed-off-by: Rayhan Hossain --- CHANGELOG.md | 2 +- docs/designs/product-neutral-core.md | 17 +- .../connecting-to-documentdb.md | 2 +- documentdb-kubectl-plugin/cmd/status.go | 12 +- .../cmd/status_run_test.go | 31 ++++ operator/src/internal/engine/certificate.go | 10 +- .../src/internal/engine/cluster_reconciler.go | 8 +- operator/src/internal/engine/service_name.go | 39 ++++ .../src/internal/engine/service_name_test.go | 166 ++++++++++++++++++ operator/src/internal/product/cluster.go | 24 ++- operator/src/internal/product/cluster_test.go | 46 ++++- operator/src/internal/utils/util.go | 65 ++++++- operator/src/internal/utils/util_test.go | 35 +++- .../pkg/e2eutils/portforward/portforward.go | 17 +- .../e2eutils/portforward/portforward_test.go | 12 +- 15 files changed, 437 insertions(+), 49 deletions(-) create mode 100644 operator/src/internal/engine/service_name.go create mode 100644 operator/src/internal/engine/service_name_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 322de4a7..63c8cb87 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,7 @@ - Require Go 1.26.10 across all six modules and pin Go builder images to that patch release to address [GO-2026-6809](https://pkg.go.dev/vuln/GO-2026-6809) in `crypto/x509`. Unit-test workflows now resolve Go from the operator module instead of an older hard-coded version. ### Bug Fixes -- Keep truncated gateway Service names DNS-valid and gateway certificate DNS names aligned with the actual Service name. +- Give each cluster its own gateway Service when `documentdb-service-` exceeds 63 characters: long names are shortened with a stable hash of the cluster name instead of plain truncation, which could make two clusters share one Service and publish the other cluster's endpoint. Clusters that already own a plainly truncated Service keep it. The operator now refuses a same-named Service controlled by another resource, gateway certificate DNS names follow the actual Service name, and `kubectl documentdb status` finds the Service by its owning DocumentDB instead of a name substring. - Quote product-defined extension and replication role names in lifecycle SQL, and use a product-independent ImageVolume probe name so display names such as `Cosmos DB` do not bypass capability checks. ### Major Features diff --git a/docs/designs/product-neutral-core.md b/docs/designs/product-neutral-core.md index 3a553a5b..3b166666 100644 --- a/docs/designs/product-neutral-core.md +++ b/docs/designs/product-neutral-core.md @@ -143,11 +143,18 @@ identity. The shared injector binary advertises and binary with its own gateway parameters. Merely changing the profile's plugin name does not rename the plugin server. -Gateway certificate DNS names use the same `NamingProfile.ServiceName` helper -as the gateway Service, including its 63-character truncation and removal of -trailing hyphens at the truncation boundary. Truncation preserves the legacy -naming convention, not uniqueness: product CRD name validation should reserve -room for the Service prefix to avoid collisions between truncated names. +Gateway certificate DNS names use the same name as the gateway Service. +`NamingProfile.ServiceName` returns `ServicePrefix + name` when it fits the +63-character limit. Otherwise it keeps the first 54 characters, trims a +trailing hyphen, and appends `-` plus the 8-hex-digit FNV-32a hash of the +custom resource name, so names that differ only past the cut still get +distinct Services. Keep `ServicePrefix` under 55 characters so shortened names +retain it and the Service watch predicate still matches. Earlier releases +plainly truncated long names; a cluster that already controls such a Service +keeps using it (and its certificate DNS names) so its endpoint does not change. +`UpsertService` refuses a same-named Service controlled by another owner +instead of returning it, so a cluster never publishes another cluster's +endpoint. ## Wire contracts kept on purpose diff --git a/docs/operator-public-documentation/preview/getting-started/connecting-to-documentdb.md b/docs/operator-public-documentation/preview/getting-started/connecting-to-documentdb.md index dc8a45fc..eecb21c6 100644 --- a/docs/operator-public-documentation/preview/getting-started/connecting-to-documentdb.md +++ b/docs/operator-public-documentation/preview/getting-started/connecting-to-documentdb.md @@ -80,7 +80,7 @@ kubectl get svc documentdb-service-my-documentdb -n documentdb-ns \ -o jsonpath='{.status.loadBalancer.ingress[0].ip}' ``` -The service name follows the pattern `documentdb-service-` (truncated to 63 characters). +The service name follows the pattern `documentdb-service-`. If that exceeds the 63-character Kubernetes limit, the operator shortens it and appends a hash of the DocumentDB name so every cluster gets its own Service (for example `documentdb-service--<8-hex-digit-hash>`). Clusters created by earlier releases keep their existing Service name. Run `kubectl get svc -n ` to see the exact name. !!! note Set `spec.environment` to `aks`, `eks`, or `gke` to apply cloud-specific load balancer annotations automatically. See [Networking](../configuration/networking.md) for details. diff --git a/documentdb-kubectl-plugin/cmd/status.go b/documentdb-kubectl-plugin/cmd/status.go index b7545825..9aff892f 100644 --- a/documentdb-kubectl-plugin/cmd/status.go +++ b/documentdb-kubectl-plugin/cmd/status.go @@ -249,9 +249,15 @@ func findDocumentDBServiceEndpoint(ctx context.Context, clientset kubernetes.Int if err != nil { return "", fmt.Errorf("list services: %w", err) } - for _, svc := range services.Items { - if strings.HasPrefix(svc.Name, documentdbServicePrefix) && (strings.Contains(svc.Name, clusterName) || strings.Contains(svc.Name, documentName)) { - return renderServiceEndpoint(&svc), nil + // The operator shortens long Service names with a hash suffix, so match + // the Service by its controlling DocumentDB rather than by name; a name + // substring could select another cluster's Service. + for idx := range services.Items { + svc := &services.Items[idx] + owner := metav1.GetControllerOf(svc) + if strings.HasPrefix(svc.Name, documentdbServicePrefix) && owner != nil && + owner.Kind == "DocumentDB" && owner.Name == documentName { + return renderServiceEndpoint(svc), nil } } diff --git a/documentdb-kubectl-plugin/cmd/status_run_test.go b/documentdb-kubectl-plugin/cmd/status_run_test.go index a91bd03f..8568407c 100644 --- a/documentdb-kubectl-plugin/cmd/status_run_test.go +++ b/documentdb-kubectl-plugin/cmd/status_run_test.go @@ -11,6 +11,7 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/dynamic" "k8s.io/client-go/kubernetes" kubefake "k8s.io/client-go/kubernetes/fake" @@ -179,3 +180,33 @@ func TestStatusRunRendersClusterTable(t *testing.T) { } } } + +func TestFindDocumentDBServiceEndpointMatchesOwnerNotSubstring(t *testing.T) { + const namespace = "ns" + controlled := func(name, owner, ip string) *corev1.Service { + return &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, Namespace: namespace, + OwnerReferences: []metav1.OwnerReference{{ + APIVersion: "documentdb.io/preview", Kind: "DocumentDB", Name: owner, + UID: types.UID("uid-" + owner), Controller: &[]bool{true}[0], + }}, + }, + Spec: corev1.ServiceSpec{ClusterIP: ip}, + } + } + long := strings.Repeat("a", 44) + "-east" + client := kubefake.NewSimpleClientset( + // Another cluster whose name contains "db"; it must not be reported for "db". + controlled(documentdbServicePrefix+"db2", "db2", "10.0.0.9"), + // The operator shortened this long name with a hash suffix. + controlled(documentdbServicePrefix+strings.Repeat("a", 35)+"-2f5a7991", long, "10.0.0.5"), + ) + + if got, err := findDocumentDBServiceEndpoint(context.Background(), client, namespace, "member", long); err != nil || got != "10.0.0.5" { + t.Fatalf("long name: got %q, %v; want the owned hashed Service", got, err) + } + if got, err := findDocumentDBServiceEndpoint(context.Background(), client, namespace, "member", "db"); err == nil { + t.Fatalf("expected no endpoint for db, got another cluster's %q", got) + } +} diff --git a/operator/src/internal/engine/certificate.go b/operator/src/internal/engine/certificate.go index d59144b0..74bab2a0 100644 --- a/operator/src/internal/engine/certificate.go +++ b/operator/src/internal/engine/certificate.go @@ -184,7 +184,10 @@ func (r *CertificateReconciler[T]) ensureCertManagerManagedCert(ctx context.Cont secretName = obj.GetName() + "-gateway-cert-tls" } - serviceBase := r.Adapter.Profile().Naming.ServiceName(obj.GetName()) + serviceBase, err := gatewayServiceName(ctx, r.Client, r.Adapter.Profile().Naming, obj) + if err != nil { + return ctrl.Result{}, err + } baseDNS := []string{serviceBase, serviceBase + "." + obj.GetNamespace(), serviceBase + "." + obj.GetNamespace() + ".svc"} dnsSet := map[string]struct{}{} finalDNS := []string{} @@ -284,7 +287,10 @@ func (r *CertificateReconciler[T]) ensureSelfSignedCert(ctx context.Context, obj } } - serviceBase := r.Adapter.Profile().Naming.ServiceName(obj.GetName()) + serviceBase, err := gatewayServiceName(ctx, r.Client, r.Adapter.Profile().Naming, obj) + if err != nil { + return ctrl.Result{}, err + } dnsNames := []string{ serviceBase, serviceBase + "." + namespace, diff --git a/operator/src/internal/engine/cluster_reconciler.go b/operator/src/internal/engine/cluster_reconciler.go index 01e401bb..d87e067c 100644 --- a/operator/src/internal/engine/cluster_reconciler.go +++ b/operator/src/internal/engine/cluster_reconciler.go @@ -162,9 +162,15 @@ func (r *ClusterReconciler[T]) Reconcile(ctx context.Context, req ctrl.Request) serviceType = corev1.ServiceTypeLoadBalancer // Public LoadBalancer service } + serviceName, err := gatewayServiceName(ctx, r.Client, p.Naming, obj) + if err != nil { + logger.Error(err, "Failed to resolve "+p.Name+" Service name; Requeuing.") + return ctrl.Result{RequeueAfter: RequeueAfterShort}, nil + } + apiVersion, kind := obj.GetObjectKind().GroupVersionKind().ToAPIVersionAndKind() svc := util.BuildGatewayService(util.GatewayServiceOptions{ - Name: p.Naming.ServiceName(obj.GetName()), + Name: serviceName, Namespace: req.Namespace, ClusterName: obj.GetName(), OwnerAPIVersion: apiVersion, diff --git a/operator/src/internal/engine/service_name.go b/operator/src/internal/engine/service_name.go new file mode 100644 index 00000000..ebc08744 --- /dev/null +++ b/operator/src/internal/engine/service_name.go @@ -0,0 +1,39 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package engine + +import ( + "context" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/documentdb/documentdb-operator/internal/product" +) + +// gatewayServiceName returns the gateway Service name for obj. A cluster whose +// long name an earlier release plainly truncated keeps that existing Service, +// and with it its LoadBalancer endpoint and certificate names, but only when +// the Service is controlled by obj; any other cluster uses the hashed name. +func gatewayServiceName(ctx context.Context, c client.Reader, naming product.NamingProfile, obj metav1.Object) (string, error) { + name := naming.ServiceName(obj.GetName()) + legacy, ok := naming.LegacyServiceName(obj.GetName()) + if !ok { + return name, nil + } + svc := &corev1.Service{} + if err := c.Get(ctx, types.NamespacedName{Name: legacy, Namespace: obj.GetNamespace()}, svc); err != nil { + if apierrors.IsNotFound(err) { + return name, nil + } + return "", err + } + if owner := metav1.GetControllerOf(svc); owner != nil && owner.UID == obj.GetUID() { + return legacy, nil + } + return name, nil +} diff --git a/operator/src/internal/engine/service_name_test.go b/operator/src/internal/engine/service_name_test.go new file mode 100644 index 00000000..88b9e83c --- /dev/null +++ b/operator/src/internal/engine/service_name_test.go @@ -0,0 +1,166 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package engine + +import ( + "context" + "slices" + "strings" + "testing" + + cmapi "github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1" + cnpgv1 "github.com/cloudnative-pg/cloudnative-pg/api/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" + + dbpreview "github.com/documentdb/documentdb-operator/api/preview" + "github.com/documentdb/documentdb-operator/internal/product" +) + +// sharedPrefix makes "documentdb-service-"+name exceed 63 characters only in +// the trailing "-east"/"-west", which plain truncation used to discard. +var sharedPrefix = strings.Repeat("a", 44) + +func longCluster(suffix string, uid types.UID) *dbpreview.DocumentDB { + db := reconcilableCluster() + db.Name = sharedPrefix + suffix + db.UID = uid + db.Spec.ExposeViaService.ServiceType = "ClusterIP" + return db +} + +func ownedService(name, ip string, owner *dbpreview.DocumentDB) *corev1.Service { + svc := gatewayService(ip) + svc.Name = name + svc.OwnerReferences = []metav1.OwnerReference{{ + APIVersion: "documentdb.io/preview", Kind: "DocumentDB", Name: owner.Name, UID: owner.UID, + Controller: &[]bool{true}[0], + }} + return svc +} + +func TestGatewayServiceNameKeepsOnlyOwnLegacyService(t *testing.T) { + naming := product.DocumentDBProfile().Naming + east, west := longCluster("-east", "east-uid"), longCluster("-west", "west-uid") + legacy, ok := naming.LegacyServiceName(east.Name) + if !ok || legacy != naming.ServicePrefix+sharedPrefix { + t.Fatalf("unexpected legacy name %q, %v", legacy, ok) + } + + s := newTestScheme(t) + ctx := context.Background() + c := newFakeClient(t, s, nil, ownedService(legacy, "10.0.0.1", east)) + for _, tt := range []struct { + name string + obj *dbpreview.DocumentDB + want string + }{ + {name: "short name needs no lookup", obj: testCluster(), want: naming.ServicePrefix + "db"}, + {name: "owner of the legacy Service keeps it", obj: east, want: legacy}, + {name: "other cluster gets its hashed name", obj: west, want: naming.ServiceName(west.Name)}, + {name: "no legacy Service uses the hashed name", obj: longCluster("-north", "north-uid"), want: naming.ServiceName(sharedPrefix + "-north")}, + } { + t.Run(tt.name, func(t *testing.T) { + got, err := gatewayServiceName(ctx, c, naming, tt.obj) + if err != nil || got != tt.want { + t.Fatalf("gatewayServiceName = %q, %v; want %q", got, err, tt.want) + } + }) + } + + failing := newFakeClient(t, s, &interceptor.Funcs{Get: func(context.Context, client.WithWatch, client.ObjectKey, client.Object, ...client.GetOption) error { + return errInjected + }}) + if _, err := gatewayServiceName(ctx, failing, naming, west); err != errInjected { + t.Fatalf("expected lookup error, got %v", err) + } +} + +// TestClusterReconcileLongNamesDoNotShareService reproduces two clusters whose +// names differ only past the 63-character cut: each must publish its own +// Service endpoint, and the pre-existing cluster keeps its legacy Service. +func TestClusterReconcileLongNamesDoNotShareService(t *testing.T) { + naming := product.DocumentDBProfile().Naming + east, west := longCluster("-east", "east-uid"), longCluster("-west", "west-uid") + legacy, _ := naming.LegacyServiceName(east.Name) + westService := naming.ServiceName(west.Name) + if westService == legacy || westService == naming.ServiceName(east.Name) { + t.Fatalf("expected a distinct Service name for %q, got %q", west.Name, westService) + } + + s := newTestScheme(t) + ctx := context.Background() + c := newFakeClient(t, s, nil, east, west, + ownedService(legacy, "10.0.0.1", east), ownedService(westService, "10.0.0.2", west)) + r, _ := newClusterReconciler(c, s, &sqlRecorder{roleOut: "(1 row)"}) + + for _, tt := range []struct { + db *dbpreview.DocumentDB + ip string + }{{db: west, ip: "10.0.0.2"}, {db: east, ip: "10.0.0.1"}} { + key := ctrl.Request{NamespacedName: types.NamespacedName{Name: tt.db.Name, Namespace: testNamespace}} + if _, err := r.Reconcile(ctx, key); err != nil { + t.Fatalf("%s: create reconcile: %v", tt.db.Name, err) + } + cluster := &cnpgv1.Cluster{} + if err := c.Get(ctx, key.NamespacedName, cluster); err != nil { + t.Fatal(err) + } + cluster.Status = cnpgv1.ClusterStatus{ + Phase: CNPGClusterHealthyPhase, + CurrentPrimary: tt.db.Name + "-1", + InstancesStatus: map[cnpgv1.PodStatus][]string{cnpgv1.PodHealthy: {tt.db.Name + "-1"}}, + } + if err := c.Update(ctx, cluster); err != nil { + t.Fatal(err) + } + if _, err := r.Reconcile(ctx, key); err != nil { + t.Fatalf("%s: converge reconcile: %v", tt.db.Name, err) + } + got := &dbpreview.DocumentDB{} + if err := c.Get(ctx, key.NamespacedName, got); err != nil { + t.Fatal(err) + } + if !strings.Contains(got.Status.ConnectionString, tt.ip) { + t.Fatalf("%s published %q, want its own Service IP %s", tt.db.Name, got.Status.ConnectionString, tt.ip) + } + } + + if err := c.Get(ctx, types.NamespacedName{Name: naming.ServiceName(east.Name), Namespace: testNamespace}, &corev1.Service{}); !apierrors.IsNotFound(err) { + t.Fatalf("existing cluster must keep its legacy Service instead of creating a hashed one, got %v", err) + } +} + +func TestCertificateNamesFollowResolvedService(t *testing.T) { + naming := product.DocumentDBProfile().Naming + east, west := longCluster("-east", "east-uid"), longCluster("-west", "west-uid") + legacy, _ := naming.LegacyServiceName(east.Name) + + for _, tt := range []struct { + db *dbpreview.DocumentDB + want string + }{{db: east, want: legacy}, {db: west, want: naming.ServiceName(west.Name)}} { + t.Run(tt.db.Name, func(t *testing.T) { + s := newTestScheme(t) + c := newFakeClient(t, s, nil, tt.db, ownedService(legacy, "10.0.0.1", east)) + key := ctrl.Request{NamespacedName: types.NamespacedName{Name: tt.db.Name, Namespace: testNamespace}} + if _, err := newCertReconciler(c, s).Reconcile(context.Background(), key); err != nil { + t.Fatal(err) + } + cert := &cmapi.Certificate{} + if err := c.Get(context.Background(), types.NamespacedName{Name: tt.db.Name + "-gateway-cert", Namespace: testNamespace}, cert); err != nil { + t.Fatal(err) + } + want := []string{tt.want, tt.want + "." + testNamespace, tt.want + "." + testNamespace + ".svc"} + if !slices.Equal(cert.Spec.DNSNames, want) { + t.Fatalf("DNSNames = %v, want %v", cert.Spec.DNSNames, want) + } + }) + } +} diff --git a/operator/src/internal/product/cluster.go b/operator/src/internal/product/cluster.go index 559f6a56..f30298b3 100644 --- a/operator/src/internal/product/cluster.go +++ b/operator/src/internal/product/cluster.go @@ -4,11 +4,10 @@ package product import ( - "strings" - "sigs.k8s.io/controller-runtime/pkg/client" dbpreview "github.com/documentdb/documentdb-operator/api/preview" + util "github.com/documentdb/documentdb-operator/internal/utils" ) // NamingProfile carries the product's Kubernetes naming conventions: the domain @@ -49,14 +48,21 @@ func (n NamingProfile) SchemaVersionAnnotation() string { return n.Domain + "/sc // Finalizer is the custom resource finalizer used to warn about retained PVs. func (n NamingProfile) Finalizer() string { return n.Domain + "/pv-retention-finalizer" } -// ServiceName is the gateway Service name for a custom resource, truncated to -// the 63-character Kubernetes name limit without leaving a trailing hyphen. +// ServiceName is the gateway Service name for a custom resource: +// ServicePrefix+clusterName, or, when that exceeds the 63-character Kubernetes +// name limit, a truncated form ending in a hash of clusterName so distinct +// clusters never share a Service. ServicePrefix must stay under 55 characters +// so long names keep it. func (n NamingProfile) ServiceName(clusterName string) string { - name := n.ServicePrefix + clusterName - if len(name) > 63 { - name = strings.TrimRight(name[:63], "-") - } - return name + return util.GatewayServiceName(n.ServicePrefix, clusterName) +} + +// LegacyServiceName is the plainly truncated gateway Service name that earlier +// releases created for clusterName, reported only when it differs from the +// current scheme; existing clusters keep that Service so their endpoint does +// not change. +func (n NamingProfile) LegacyServiceName(clusterName string) (string, bool) { + return util.LegacyGatewayServiceName(n.ServicePrefix, clusterName) } // ClusterSpec is the product-neutral, read-only view of the operational inputs diff --git a/operator/src/internal/product/cluster_test.go b/operator/src/internal/product/cluster_test.go index 5540d996..b8f28309 100644 --- a/operator/src/internal/product/cluster_test.go +++ b/operator/src/internal/product/cluster_test.go @@ -34,14 +34,44 @@ func TestDocumentDBNamingMatchesWireContract(t *testing.T) { } } -func TestServiceNameTruncationKeepsValidDNSLabel(t *testing.T) { - n := NamingProfile{ServicePrefix: "cosmos-gateway-"} - got := n.ServiceName(strings.Repeat("a", 47) + "-a") - if errs := validation.IsDNS1035Label(got); len(errs) != 0 { - t.Fatalf("truncating a valid cluster name produced invalid Service name %q: %v", got, errs) - } - if want := n.ServicePrefix + strings.Repeat("a", 47); got != want { - t.Fatalf("ServiceName = %q, want %q", got, want) +func TestServiceNameKeepsLongClusterNamesDistinct(t *testing.T) { + for _, n := range []NamingProfile{DocumentDBProfile().Naming, {ServicePrefix: "cosmos-gateway-"}} { + short := n.ServiceName("db") + if short != n.ServicePrefix+"db" { + t.Fatalf("short names must be unchanged, got %q", short) + } + if _, ok := n.LegacyServiceName("db"); ok { + t.Fatal("a name that fits has no legacy form") + } + fits := strings.Repeat("f", 63-len(n.ServicePrefix)) + if got := n.ServiceName(fits); got != n.ServicePrefix+fits { + t.Fatalf("a 63-character name must be unchanged, got %q", got) + } + + // The names differ only after the 63-character cut, and the cut + // lands on a hyphen. + base := strings.Repeat("a", 63-len(n.ServicePrefix)-1) + east, west := n.ServiceName(base+"-east"), n.ServiceName(base+"-west") + if east == west { + t.Fatalf("distinct clusters share Service %q", east) + } + for _, got := range []string{east, west} { + if errs := validation.IsDNS1035Label(got); len(errs) != 0 { + t.Fatalf("invalid Service name %q: %v", got, errs) + } + if !strings.HasPrefix(got, n.ServicePrefix) { + t.Fatalf("Service name %q lost prefix %q", got, n.ServicePrefix) + } + } + if again := n.ServiceName(base + "-east"); again != east { + t.Fatalf("Service name is not stable: %q vs %q", east, again) + } + if _, ok := n.LegacyServiceName(base + "-east"); ok { + t.Fatal("plain truncation ending in a hyphen was never a valid Service name") + } + if legacy, ok := n.LegacyServiceName(base + "x-east"); !ok || legacy != (n.ServicePrefix+base+"x") { + t.Fatalf("unexpected legacy name %q, %v", legacy, ok) + } } } diff --git a/operator/src/internal/utils/util.go b/operator/src/internal/utils/util.go index 14b92977..a852a815 100644 --- a/operator/src/internal/utils/util.go +++ b/operator/src/internal/utils/util.go @@ -6,6 +6,7 @@ package util import ( "context" "fmt" + "hash/fnv" "os" "strconv" "strings" @@ -30,13 +31,8 @@ import ( // GetDocumentDBServiceDefinition returns the LoadBalancer Service definition for a given DocumentDB instance func GetDocumentDBServiceDefinition(documentdb *dbpreview.DocumentDB, replicationContext *ReplicationContext, namespace string, serviceType corev1.ServiceType) *corev1.Service { - // Ensure service name doesn't exceed 63 characters (Kubernetes limit) - serviceName := DOCUMENTDB_SERVICE_PREFIX + documentdb.Name - if len(serviceName) > 63 { - serviceName = serviceName[:63] - } return BuildGatewayService(GatewayServiceOptions{ - Name: serviceName, + Name: GatewayServiceName(DOCUMENTDB_SERVICE_PREFIX, documentdb.Name), Namespace: namespace, ClusterName: documentdb.Name, OwnerAPIVersion: documentdb.APIVersion, @@ -49,6 +45,40 @@ func GetDocumentDBServiceDefinition(documentdb *dbpreview.DocumentDB, replicatio }) } +// maxServiceNameLength is the Kubernetes DNS-1035 label limit for Service names. +const maxServiceNameLength = 63 + +// GatewayServiceName returns prefix+clusterName when it fits the 63-character +// Service name limit. A longer name keeps as much of that readable prefix as +// fits and ends in a hash of clusterName, so cluster names that differ only +// past the cut still get distinct Services. +func GatewayServiceName(prefix, clusterName string) string { + name := prefix + clusterName + if len(name) <= maxServiceNameLength { + return name + } + h := fnv.New32a() + _, _ = h.Write([]byte(clusterName)) + suffix := fmt.Sprintf("-%08x", h.Sum32()) + return strings.TrimRight(name[:maxServiceNameLength-len(suffix)], "-") + suffix +} + +// LegacyGatewayServiceName returns the plainly truncated Service name that +// releases before GatewayServiceName gave clusters whose prefixed name exceeded +// 63 characters. It reports false when no such Service could exist: the name +// fits, or truncation leaves an invalid trailing hyphen. +func LegacyGatewayServiceName(prefix, clusterName string) (string, bool) { + name := prefix + clusterName + if len(name) <= maxServiceNameLength { + return "", false + } + name = name[:maxServiceNameLength] + if strings.HasSuffix(name, "-") { + return "", false + } + return name, true +} + // GatewayServiceOptions are the product-neutral inputs of the gateway Service. type GatewayServiceOptions struct { // Name and Namespace of the Service. @@ -187,7 +217,9 @@ func EnsureServiceIP(ctx context.Context, service *corev1.Service) (string, erro // An existing Service controlled by the desired Service's controller owner // has its ports converged to the desired ports (for example after a gateway // port change); server-assigned fields such as ClusterIP(s) and NodePorts and -// all other spec and metadata fields are preserved. +// all other spec and metadata fields are preserved. A same-named Service +// controlled by a different owner is left untouched and reported as an error, +// so a caller never adopts and publishes another cluster's endpoint. func UpsertService(ctx context.Context, c client.Client, service *corev1.Service) (*corev1.Service, error) { log := log.FromContext(ctx) foundService := &corev1.Service{} @@ -203,10 +235,16 @@ func UpsertService(ctx context.Context, c client.Client, service *corev1.Service if err := c.Get(ctx, types.NamespacedName{Name: service.Name, Namespace: service.Namespace}, foundService); err != nil { return nil, err } + if err := checkServiceController(foundService, service); err != nil { + return nil, err + } } else { return nil, err } } else { + if err := checkServiceController(foundService, service); err != nil { + return nil, err + } if sameController(foundService, service) { foundService.Spec.Ports = convergeServicePorts(foundService.Spec.Ports, service.Spec.Ports) } @@ -217,6 +255,19 @@ func UpsertService(ctx context.Context, c client.Client, service *corev1.Service return foundService, nil } +// checkServiceController returns an error when found is controlled by an owner +// other than desired's controller owner. A Service with no controller is +// accepted, as before, so a pre-created Service keeps working. +func checkServiceController(found, desired *corev1.Service) error { + owner := metav1.GetControllerOf(desired) + current := metav1.GetControllerOf(found) + if owner == nil || current == nil || current.UID == owner.UID { + return nil + } + return fmt.Errorf("service %s/%s is controlled by %s %q (uid %s), not %s %q (uid %s)", + found.Namespace, found.Name, current.Kind, current.Name, current.UID, owner.Kind, owner.Name, owner.UID) +} + // sameController reports whether found is controlled by desired's controller // owner, so UpsertService never rewrites a Service it does not own. func sameController(found, desired *corev1.Service) bool { diff --git a/operator/src/internal/utils/util_test.go b/operator/src/internal/utils/util_test.go index 4a6ac1b4..642b1eed 100644 --- a/operator/src/internal/utils/util_test.go +++ b/operator/src/internal/utils/util_test.go @@ -1023,8 +1023,8 @@ func TestBuildGatewayService_ProductNeutral(t *testing.T) { } // TestUpsertServiceConvergesPorts updates an existing gateway Service to a new -// gateway port, keeping server-assigned and unrelated fields, and leaves a -// Service controlled by another owner untouched. +// gateway port, keeping server-assigned and unrelated fields, and refuses a +// Service controlled by another owner without rewriting it. func TestUpsertServiceConvergesPorts(t *testing.T) { ctx := context.Background() opts := GatewayServiceOptions{ @@ -1066,8 +1066,9 @@ func TestUpsertServiceConvergesPorts(t *testing.T) { foreignOpts := opts foreignOpts.Name = "foreign" - if _, err := UpsertService(ctx, c, BuildGatewayService(foreignOpts)); err != nil { - t.Fatal(err) + if svc, err := UpsertService(ctx, c, BuildGatewayService(foreignOpts)); err == nil || svc != nil || + !strings.Contains(err.Error(), "other-uid") { + t.Fatalf("expected a foreign-controller error and no Service, got %v, %v", svc, err) } if err := c.Get(ctx, client.ObjectKeyFromObject(foreign), got); err != nil { t.Fatal(err) @@ -1075,4 +1076,30 @@ func TestUpsertServiceConvergesPorts(t *testing.T) { if got.Spec.Ports[0].Port != 10260 { t.Errorf("a Service controlled by another owner was rewritten: %+v", got.Spec.Ports) } + + unowned := existing.DeepCopy() + unowned.Name, unowned.OwnerReferences, unowned.ResourceVersion = "unowned", nil, "" + if err := c.Create(ctx, unowned); err != nil { + t.Fatal(err) + } + unownedOpts := opts + unownedOpts.Name = "unowned" + if _, err := UpsertService(ctx, c, BuildGatewayService(unownedOpts)); err != nil { + t.Fatalf("a pre-created Service without a controller must still be usable: %v", err) + } +} + +func TestGatewayServiceNameHashesOnlyLongNames(t *testing.T) { + if got := GatewayServiceName(DOCUMENTDB_SERVICE_PREFIX, "db"); got != "documentdb-service-db" { + t.Fatalf("short name changed: %q", got) + } + long := strings.Repeat("a", 44) + east := GatewayServiceName(DOCUMENTDB_SERVICE_PREFIX, long+"-east") + // Pin the format: a stable hash must not drift between releases. + if want := DOCUMENTDB_SERVICE_PREFIX + strings.Repeat("a", 35) + "-2f5a7991"; east != want { + t.Fatalf("GatewayServiceName = %q, want %q", east, want) + } + if east == GatewayServiceName(DOCUMENTDB_SERVICE_PREFIX, long+"-west") { + t.Fatal("names differing past the cut must not share a Service") + } } diff --git a/test/e2e/pkg/e2eutils/portforward/portforward.go b/test/e2e/pkg/e2eutils/portforward/portforward.go index f0323dfa..037215c5 100644 --- a/test/e2e/pkg/e2eutils/portforward/portforward.go +++ b/test/e2e/pkg/e2eutils/portforward/portforward.go @@ -22,7 +22,9 @@ import ( "context" "errors" "fmt" + "hash/fnv" "io" + "strings" "github.com/cloudnative-pg/cloudnative-pg/tests/utils/environment" "github.com/cloudnative-pg/cloudnative-pg/tests/utils/forwardconnection" @@ -37,8 +39,10 @@ import ( const GatewayPort = 10260 // ServiceNamePrefix mirrors DOCUMENTDB_SERVICE_PREFIX from the operator. -// The fully-qualified service name is ServiceNamePrefix + dd.Name, -// truncated to 63 characters to honour the Kubernetes DNS limit. +// The Service name is ServiceNamePrefix + dd.Name; when that exceeds 63 +// characters the operator keeps the first 54 and appends "-" plus the +// FNV-32a hash of dd.Name (see utils.GatewayServiceName). E2E clusters are +// created fresh, so the operator's legacy plainly truncated name never applies. const ServiceNamePrefix = "documentdb-service-" // GatewayServiceName returns the Service name the operator creates for @@ -48,10 +52,13 @@ func GatewayServiceName(dd *previewv1.DocumentDB) string { return "" } name := ServiceNamePrefix + dd.Name - if len(name) > 63 { - name = name[:63] + if len(name) <= 63 { + return name } - return name + h := fnv.New32a() + _, _ = h.Write([]byte(dd.Name)) + suffix := fmt.Sprintf("-%08x", h.Sum32()) + return strings.TrimRight(name[:63-len(suffix)], "-") + suffix } // OpenWithErr establishes a port-forward from localPort on the caller's diff --git a/test/e2e/pkg/e2eutils/portforward/portforward_test.go b/test/e2e/pkg/e2eutils/portforward/portforward_test.go index 8eaa78e7..6bb6a8c6 100644 --- a/test/e2e/pkg/e2eutils/portforward/portforward_test.go +++ b/test/e2e/pkg/e2eutils/portforward/portforward_test.go @@ -25,11 +25,17 @@ func TestGatewayServiceName(t *testing.T) { "documentdb-service-my-dd", }, { - "truncated", - &previewv1.DocumentDB{ObjectMeta: metav1.ObjectMeta{Name: strings.Repeat("x", 80)}}, - // 19 (prefix) + 44 xs = 63 + "exactly 63", + &previewv1.DocumentDB{ObjectMeta: metav1.ObjectMeta{Name: strings.Repeat("x", 44)}}, + // 19 (prefix) + 44 xs = 63: unchanged "documentdb-service-" + strings.Repeat("x", 44), }, + { + // Same vector as the operator's TestGatewayServiceNameHashesOnlyLongNames. + "hashed", + &previewv1.DocumentDB{ObjectMeta: metav1.ObjectMeta{Name: strings.Repeat("a", 44) + "-east"}}, + "documentdb-service-" + strings.Repeat("a", 35) + "-2f5a7991", + }, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { From f22ad88b46a76fc5bf6a47a9407c841a55fe43d8 Mon Sep 17 00:00:00 2001 From: Rayhan Hossain Date: Wed, 28 Oct 2026 09:01:36 -0700 Subject: [PATCH 27/27] fix(core): keep gateway Service names and certificates stable across recreation An upgraded long-name cluster that kept its legacy Service switched to the hashed name when that Service was deleted, but managed certificates were create-only, so the new hostname was missing from the certificate and strict TLS clients failed. Choose a long-name cluster's Service name once and record it in the /gateway-service-name annotation (optimistic-lock patch), so recreating the Service keeps its hostname. The legacy name is chosen only when this cluster controls a Service or gateway Certificate using it and no other resource controls that Service; a colliding upgraded cluster moves to its hashed name. Certificate evidence is read as unstructured so core consumers need not register cert-manager types. Managed gateway Certificates now gain any missing Service DNS names (add-only, both managed modes), TLS readiness requires a Ready condition for the current certificate generation, and a certificate controlled by a different resource (a deleted CR of the same name) is neither changed nor used. Helm RBAC grants patch on cert-manager Certificates. The kubectl plugin and e2e helper honor the recorded name, and the plugin ignores Services of a different owner UID. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 33b3cb3c-1b25-4ffb-bff1-ebf0f3d04058 Signed-off-by: Rayhan Hossain --- CHANGELOG.md | 2 +- docs/designs/product-neutral-core.md | 24 +- .../connecting-to-documentdb.md | 2 +- documentdb-kubectl-plugin/cmd/status.go | 38 ++- .../cmd/status_run_test.go | 41 ++- .../templates/05_clusterrole.yaml | 14 +- .../tests/05_clusterrole_test.yaml | 8 +- operator/src/config/rbac/role.yaml | 1 + .../controller/certificate_controller.go | 2 +- operator/src/internal/engine/certificate.go | 71 +++- operator/src/internal/engine/service_name.go | 110 ++++++- .../src/internal/engine/service_name_test.go | 304 ++++++++++++++++-- operator/src/internal/product/cluster.go | 5 + .../pkg/e2eutils/portforward/portforward.go | 15 +- .../e2eutils/portforward/portforward_test.go | 16 + 15 files changed, 584 insertions(+), 69 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 63c8cb87..4d1b8eea 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,7 @@ - Require Go 1.26.10 across all six modules and pin Go builder images to that patch release to address [GO-2026-6809](https://pkg.go.dev/vuln/GO-2026-6809) in `crypto/x509`. Unit-test workflows now resolve Go from the operator module instead of an older hard-coded version. ### Bug Fixes -- Give each cluster its own gateway Service when `documentdb-service-` exceeds 63 characters: long names are shortened with a stable hash of the cluster name instead of plain truncation, which could make two clusters share one Service and publish the other cluster's endpoint. Clusters that already own a plainly truncated Service keep it. The operator now refuses a same-named Service controlled by another resource, gateway certificate DNS names follow the actual Service name, and `kubectl documentdb status` finds the Service by its owning DocumentDB instead of a name substring. +- Give each cluster its own gateway Service when `documentdb-service-` exceeds 63 characters: long names are shortened with a stable hash of the cluster name instead of plain truncation, which could make two clusters share one Service and publish the other cluster's endpoint. Clusters that already own a plainly truncated Service or gateway certificate keep that name. The operator records the chosen name in the `documentdb.io/gateway-service-name` annotation, so recreating the Service keeps the same hostname, and adds any missing Service DNS names to its managed gateway certificate (the operator ClusterRole now includes `patch` on cert-manager Certificates). Rolling the operator back to an earlier release is not supported for these long-name clusters. The operator now refuses a same-named Service controlled by another resource, gateway certificate DNS names follow the actual Service name, and `kubectl documentdb status` finds the Service by its owning DocumentDB instead of a name substring. - Quote product-defined extension and replication role names in lifecycle SQL, and use a product-independent ImageVolume probe name so display names such as `Cosmos DB` do not bypass capability checks. ### Major Features diff --git a/docs/designs/product-neutral-core.md b/docs/designs/product-neutral-core.md index 3b166666..7c21ac49 100644 --- a/docs/designs/product-neutral-core.md +++ b/docs/designs/product-neutral-core.md @@ -150,11 +150,25 @@ trailing hyphen, and appends `-` plus the 8-hex-digit FNV-32a hash of the custom resource name, so names that differ only past the cut still get distinct Services. Keep `ServicePrefix` under 55 characters so shortened names retain it and the Service watch predicate still matches. Earlier releases -plainly truncated long names; a cluster that already controls such a Service -keeps using it (and its certificate DNS names) so its endpoint does not change. -`UpsertService` refuses a same-named Service controlled by another owner -instead of returning it, so a cluster never publishes another cluster's -endpoint. +plainly truncated long names. On the first reconcile of a long-name cluster, +the operator keeps that legacy name if the cluster still controls a Service or +managed gateway Certificate using it, and otherwise uses the hashed name. It +records the choice in the `/gateway-service-name` annotation +(`NamingProfile.ServiceNameAnnotation`) on the custom resource and uses it from +then on, so deleting and recreating the Service never changes the hostname +that clients and certificate DNS names rely on. The annotation is ignored and +rewritten unless it holds one of those two names. The operator therefore needs +`patch` on its custom resource. When the chosen name changes (for example, an +upgraded cluster whose legacy name another cluster's Service holds), the +operator adds the missing Service DNS names to its existing managed gateway +Certificate (self-signed and cert-manager modes) and keeps the names already +there, so clients using either hostname keep valid TLS. This requires `patch` +on `certificates.cert-manager.io`. `UpsertService` refuses a same-named Service +controlled by another owner instead of returning it, so a cluster never +publishes another cluster's endpoint. Rolling the operator back to a release +that plainly truncates names is not supported for clusters whose prefixed name +exceeds 63 characters: that release would move them back to the truncated +Service name. ## Wire contracts kept on purpose diff --git a/docs/operator-public-documentation/preview/getting-started/connecting-to-documentdb.md b/docs/operator-public-documentation/preview/getting-started/connecting-to-documentdb.md index eecb21c6..c6d0ae49 100644 --- a/docs/operator-public-documentation/preview/getting-started/connecting-to-documentdb.md +++ b/docs/operator-public-documentation/preview/getting-started/connecting-to-documentdb.md @@ -80,7 +80,7 @@ kubectl get svc documentdb-service-my-documentdb -n documentdb-ns \ -o jsonpath='{.status.loadBalancer.ingress[0].ip}' ``` -The service name follows the pattern `documentdb-service-`. If that exceeds the 63-character Kubernetes limit, the operator shortens it and appends a hash of the DocumentDB name so every cluster gets its own Service (for example `documentdb-service--<8-hex-digit-hash>`). Clusters created by earlier releases keep their existing Service name. Run `kubectl get svc -n ` to see the exact name. +The service name follows the pattern `documentdb-service-`. If that exceeds the 63-character Kubernetes limit, the operator shortens it and appends a hash of the DocumentDB name so every cluster gets its own Service (for example `documentdb-service--<8-hex-digit-hash>`). Clusters created by earlier releases keep their existing Service name. For long names, the operator records the chosen name in the `documentdb.io/gateway-service-name` annotation on the DocumentDB resource and keeps it even if the Service is recreated. Run `kubectl get svc -n ` to see the exact name. !!! note Set `spec.environment` to `aks`, `eks`, or `gke` to apply cloud-specific load balancer annotations automatically. See [Networking](../configuration/networking.md) for details. diff --git a/documentdb-kubectl-plugin/cmd/status.go b/documentdb-kubectl-plugin/cmd/status.go index 9aff892f..0ee77f77 100644 --- a/documentdb-kubectl-plugin/cmd/status.go +++ b/documentdb-kubectl-plugin/cmd/status.go @@ -13,12 +13,17 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" ) const documentdbServicePrefix = "documentdb-service-" +// gatewayServiceNameAnnotation records the gateway Service name the operator +// chose for a DocumentDB whose prefixed name exceeds 63 characters. +const gatewayServiceNameAnnotation = "documentdb.io/gateway-service-name" + type statusOptions struct { documentDBName string namespace string @@ -221,7 +226,8 @@ func (o *statusOptions) populateClusterStatus(ctx context.Context, st *clusterSt } } - serviceIP, err := findDocumentDBServiceEndpoint(ctx, clientset, o.namespace, st.Cluster, o.documentDBName) + serviceIP, err := findDocumentDBServiceEndpoint(ctx, clientset, o.namespace, st.Cluster, o.documentDBName, + document.GetUID(), document.GetAnnotations()[gatewayServiceNameAnnotation]) if err == nil { st.ServiceIP = serviceIP } @@ -229,11 +235,29 @@ func (o *statusOptions) populateClusterStatus(ctx context.Context, st *clusterSt return nil } -func findDocumentDBServiceEndpoint(ctx context.Context, clientset kubernetes.Interface, namespace, clusterName, documentName string) (string, error) { +// findDocumentDBServiceEndpoint returns the endpoint of the gateway Service +// controlled by the DocumentDB with uid. It tries the operator-recorded Service +// name, then prefix+cluster/document name, then any prefixed Service the +// DocumentDB controls (long names are shortened with a hash suffix). A Service +// controlled by another resource, such as one left by a deleted DocumentDB of +// the same name, is never reported. +func findDocumentDBServiceEndpoint(ctx context.Context, clientset kubernetes.Interface, namespace, clusterName, documentName string, uid types.UID, recordedName string) (string, error) { + controlledByDocument := func(svc *corev1.Service) bool { + owner := metav1.GetControllerOf(svc) + if owner == nil { + // Services from releases that predate owner references. + return uid == "" + } + return owner.Kind == "DocumentDB" && owner.Name == documentName && (uid == "" || owner.UID == uid) + } + candidateNames := []string{ documentdbServicePrefix + clusterName, documentdbServicePrefix + documentName, } + if recordedName != "" { + candidateNames = append([]string{recordedName}, candidateNames...) + } for _, name := range candidateNames { svc, err := clientset.CoreV1().Services(namespace).Get(ctx, name, metav1.GetOptions{}) if err != nil { @@ -242,6 +266,9 @@ func findDocumentDBServiceEndpoint(ctx context.Context, clientset kubernetes.Int } return "", fmt.Errorf("get service %s: %w", name, err) } + if metav1.GetControllerOf(svc) != nil && !controlledByDocument(svc) { + continue + } return renderServiceEndpoint(svc), nil } @@ -249,14 +276,9 @@ func findDocumentDBServiceEndpoint(ctx context.Context, clientset kubernetes.Int if err != nil { return "", fmt.Errorf("list services: %w", err) } - // The operator shortens long Service names with a hash suffix, so match - // the Service by its controlling DocumentDB rather than by name; a name - // substring could select another cluster's Service. for idx := range services.Items { svc := &services.Items[idx] - owner := metav1.GetControllerOf(svc) - if strings.HasPrefix(svc.Name, documentdbServicePrefix) && owner != nil && - owner.Kind == "DocumentDB" && owner.Name == documentName { + if strings.HasPrefix(svc.Name, documentdbServicePrefix) && metav1.GetControllerOf(svc) != nil && controlledByDocument(svc) { return renderServiceEndpoint(svc), nil } } diff --git a/documentdb-kubectl-plugin/cmd/status_run_test.go b/documentdb-kubectl-plugin/cmd/status_run_test.go index 8568407c..10e4df25 100644 --- a/documentdb-kubectl-plugin/cmd/status_run_test.go +++ b/documentdb-kubectl-plugin/cmd/status_run_test.go @@ -183,30 +183,53 @@ func TestStatusRunRendersClusterTable(t *testing.T) { func TestFindDocumentDBServiceEndpointMatchesOwnerNotSubstring(t *testing.T) { const namespace = "ns" - controlled := func(name, owner, ip string) *corev1.Service { + controlled := func(name, owner string, uid types.UID, ip string) *corev1.Service { return &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ Name: name, Namespace: namespace, OwnerReferences: []metav1.OwnerReference{{ APIVersion: "documentdb.io/preview", Kind: "DocumentDB", Name: owner, - UID: types.UID("uid-" + owner), Controller: &[]bool{true}[0], + UID: uid, Controller: &[]bool{true}[0], }}, }, Spec: corev1.ServiceSpec{ClusterIP: ip}, } } long := strings.Repeat("a", 44) + "-east" + legacy := documentdbServicePrefix + strings.Repeat("a", 44) client := kubefake.NewSimpleClientset( // Another cluster whose name contains "db"; it must not be reported for "db". - controlled(documentdbServicePrefix+"db2", "db2", "10.0.0.9"), + controlled(documentdbServicePrefix+"db2", "db2", "uid-db2", "10.0.0.9"), // The operator shortened this long name with a hash suffix. - controlled(documentdbServicePrefix+strings.Repeat("a", 35)+"-2f5a7991", long, "10.0.0.5"), + controlled(documentdbServicePrefix+strings.Repeat("a", 35)+"-2f5a7991", long, "uid-long", "10.0.0.5"), + // A Service left by a deleted DocumentDB with the same name. + controlled(documentdbServicePrefix+"old", "old", "uid-deleted", "10.0.0.7"), + // An upgraded cluster that kept its legacy truncated name. + controlled(legacy, long+"-x", "uid-legacy", "10.0.0.6"), ) + ctx := context.Background() - if got, err := findDocumentDBServiceEndpoint(context.Background(), client, namespace, "member", long); err != nil || got != "10.0.0.5" { - t.Fatalf("long name: got %q, %v; want the owned hashed Service", got, err) - } - if got, err := findDocumentDBServiceEndpoint(context.Background(), client, namespace, "member", "db"); err == nil { - t.Fatalf("expected no endpoint for db, got another cluster's %q", got) + for _, tt := range []struct { + name, doc, recorded string + uid types.UID + want string + }{ + {name: "hashed long name found by owner", doc: long, uid: "uid-long", want: "10.0.0.5"}, + {name: "recorded legacy name", doc: long + "-x", uid: "uid-legacy", recorded: legacy, want: "10.0.0.6"}, + {name: "substring match is not an owner", doc: "db", uid: "uid-db"}, + {name: "stale Service of a deleted DocumentDB", doc: "old", uid: "uid-new"}, + } { + t.Run(tt.name, func(t *testing.T) { + got, err := findDocumentDBServiceEndpoint(ctx, client, namespace, "member", tt.doc, tt.uid, tt.recorded) + if tt.want == "" { + if err == nil { + t.Fatalf("expected no endpoint, got %q", got) + } + return + } + if err != nil || got != tt.want { + t.Fatalf("got %q, %v; want %q", got, err, tt.want) + } + }) } } diff --git a/operator/documentdb-helm-chart/templates/05_clusterrole.yaml b/operator/documentdb-helm-chart/templates/05_clusterrole.yaml index b94fff5b..b3c96738 100644 --- a/operator/documentdb-helm-chart/templates/05_clusterrole.yaml +++ b/operator/documentdb-helm-chart/templates/05_clusterrole.yaml @@ -38,12 +38,16 @@ rules: - apiGroups: ["postgresql.cnpg.io"] resources: ["clusters", "publications", "subscriptions", "clusters/status"] verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] -# cert-manager: certificate_controller only Get/Create namespaced Certificates -# and Issuers, and Owns() them (which requires watch). It never Updates, -# Patches, Deletes, or sets Finalizers on cert-manager objects. ClusterIssuer -# is referenced by name in CR spec but resolved by cert-manager itself. +# cert-manager: certificate_controller Gets/Creates namespaced Certificates +# and Issuers and Owns() them (which requires watch). It patches only its own +# gateway Certificates, to add gateway Service DNS names they lack. It never +# Deletes or sets Finalizers on cert-manager objects. ClusterIssuer is +# referenced by name in CR spec but resolved by cert-manager itself. - apiGroups: ["cert-manager.io"] - resources: ["certificates", "issuers"] + resources: ["certificates"] + verbs: ["get", "list", "watch", "create", "patch"] +- apiGroups: ["cert-manager.io"] + resources: ["issuers"] verbs: ["get", "list", "watch", "create"] # Backup permissions diff --git a/operator/documentdb-helm-chart/tests/05_clusterrole_test.yaml b/operator/documentdb-helm-chart/tests/05_clusterrole_test.yaml index 6512b062..29892400 100644 --- a/operator/documentdb-helm-chart/tests/05_clusterrole_test.yaml +++ b/operator/documentdb-helm-chart/tests/05_clusterrole_test.yaml @@ -39,7 +39,13 @@ tests: path: rules content: apiGroups: ["cert-manager.io"] - resources: ["certificates", "issuers"] + resources: ["certificates"] + verbs: ["get", "list", "watch", "create", "patch"] + - contains: + path: rules + content: + apiGroups: ["cert-manager.io"] + resources: ["issuers"] verbs: ["get", "list", "watch", "create"] - it: should include backup-related permissions diff --git a/operator/src/config/rbac/role.yaml b/operator/src/config/rbac/role.yaml index 38de72b7..590430ca 100644 --- a/operator/src/config/rbac/role.yaml +++ b/operator/src/config/rbac/role.yaml @@ -58,6 +58,7 @@ rules: verbs: - get - list + - patch - watch - apiGroups: - documentdb.io diff --git a/operator/src/internal/controller/certificate_controller.go b/operator/src/internal/controller/certificate_controller.go index 9fa91014..57dfefe6 100644 --- a/operator/src/internal/controller/certificate_controller.go +++ b/operator/src/internal/controller/certificate_controller.go @@ -23,7 +23,7 @@ type CertificateReconciler struct { Scheme *runtime.Scheme } -// +kubebuilder:rbac:groups=documentdb.io,resources=dbs,verbs=get;list;watch +// +kubebuilder:rbac:groups=documentdb.io,resources=dbs,verbs=get;list;watch;patch // +kubebuilder:rbac:groups=documentdb.io,resources=dbs/status,verbs=get;update;patch // +kubebuilder:rbac:groups=cert-manager.io,resources=certificates;issuers,verbs=get;list;watch;create;update;patch // +kubebuilder:rbac:groups=cert-manager.io,resources=certificates/status;issuers/status,verbs=get diff --git a/operator/src/internal/engine/certificate.go b/operator/src/internal/engine/certificate.go index 74bab2a0..b3a46523 100644 --- a/operator/src/internal/engine/certificate.go +++ b/operator/src/internal/engine/certificate.go @@ -5,6 +5,7 @@ package engine import ( "context" + "slices" "time" cmapi "github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1" @@ -237,9 +238,12 @@ func (r *CertificateReconciler[T]) ensureCertManagerManagedCert(ctx context.Cont } return ctrl.Result{RequeueAfter: RequeueAfterShort}, nil } + if res, done, err := r.ensureServiceDNSNames(ctx, obj, cert, baseDNS); done { + return res, err + } for _, cond := range cert.Status.Conditions { - if cond.Type == cmapi.CertificateConditionReady && cond.Status == cmmeta.ConditionTrue { + if certificateReady(cert, cond) { if !r.Adapter.Status(obj).TLS.Ready { if err := r.updateTLSStatus(ctx, obj, func(status *dbpreview.TLSStatus) { status.Ready = true @@ -329,9 +333,12 @@ func (r *CertificateReconciler[T]) ensureSelfSignedCert(ctx context.Context, obj } return ctrl.Result{RequeueAfter: RequeueAfterShort}, nil } + if res, done, err := r.ensureServiceDNSNames(ctx, obj, cert, dnsNames); done { + return res, err + } for _, cond := range cert.Status.Conditions { - if cond.Type == cmapi.CertificateConditionReady && cond.Status == cmmeta.ConditionTrue { + if certificateReady(cert, cond) { if !r.Adapter.Status(obj).TLS.Ready { if err := r.updateTLSStatus(ctx, obj, func(status *dbpreview.TLSStatus) { status.Ready = true @@ -355,6 +362,62 @@ func (r *CertificateReconciler[T]) ensureSelfSignedCert(ctx context.Context, obj return ctrl.Result{RequeueAfter: RequeueAfterShort}, nil } +// ensureServiceDNSNames adds to an existing managed Certificate any gateway +// Service DNS name it lacks, for example after the Service name changes. It +// only adds names, so hostnames clients already use stay valid. A Certificate +// controlled by another resource is neither modified nor used; one with no +// controller is used as before. done reports that the caller should return +// with the given result. Readiness checks require a Ready condition observed +// for the Certificate's current generation, so the added names are issued +// before TLS is reported ready again. +func (r *CertificateReconciler[T]) ensureServiceDNSNames(ctx context.Context, obj T, cert *cmapi.Certificate, serviceDNS []string) (ctrl.Result, bool, error) { + if ref := metav1.GetControllerOf(cert); ref != nil && ref.UID != obj.GetUID() { + // Left by a deleted custom resource of the same name; wait for + // garbage collection instead of using its status or Secret. + if err := r.updateTLSStatus(ctx, obj, func(status *dbpreview.TLSStatus) { + status.Ready = false + status.Message = "Waiting for a gateway certificate owned by another resource to be removed" + }); err != nil { + return ctrl.Result{}, true, err + } + return ctrl.Result{RequeueAfter: RequeueAfterShort}, true, nil + } + if !controlledBy(cert, obj) { + return ctrl.Result{}, false, nil + } + var missing []string + for _, n := range serviceDNS { + if !slices.Contains(cert.Spec.DNSNames, n) { + missing = append(missing, n) + } + } + if len(missing) == 0 { + return ctrl.Result{}, false, nil + } + before := cert.DeepCopy() + cert.Spec.DNSNames = append(cert.Spec.DNSNames, missing...) + if err := r.Patch(ctx, cert, client.MergeFrom(before)); err != nil { + return ctrl.Result{}, true, err + } + if err := r.updateTLSStatus(ctx, obj, func(status *dbpreview.TLSStatus) { + status.Ready = false + status.SecretName = cert.Spec.SecretName + status.Message = "Updating gateway certificate DNS names" + }); err != nil { + return ctrl.Result{}, true, err + } + return ctrl.Result{RequeueAfter: RequeueAfterShort}, true, nil +} + +// certificateReady reports whether cond is a Ready=True condition for cert's +// current generation, so a spec change (such as added DNS names) is issued +// before TLS is reported ready. A condition without an observed generation is +// accepted as before. +func certificateReady(cert *cmapi.Certificate, cond cmapi.CertificateCondition) bool { + return cond.Type == cmapi.CertificateConditionReady && cond.Status == cmmeta.ConditionTrue && + (cond.ObservedGeneration == 0 || cond.ObservedGeneration >= cert.Generation) +} + func (r *CertificateReconciler[T]) updateTLSStatus(ctx context.Context, obj T, mutate func(*dbpreview.TLSStatus)) error { key := types.NamespacedName{Name: obj.GetName(), Namespace: obj.GetNamespace()} return retry.RetryOnConflict(retry.DefaultRetry, func() error { @@ -372,6 +435,10 @@ func (r *CertificateReconciler[T]) updateTLSStatus(ctx context.Context, obj T, m return err } r.Adapter.SetStatus(obj, r.Adapter.Status(current)) + // Keep obj current so later writes in this reconcile, such as the + // optimistic-lock patch that records the gateway Service name, do not + // conflict with this status update. + obj.SetResourceVersion(current.GetResourceVersion()) return nil }) } diff --git a/operator/src/internal/engine/service_name.go b/operator/src/internal/engine/service_name.go index ebc08744..8fdb36fc 100644 --- a/operator/src/internal/engine/service_name.go +++ b/operator/src/internal/engine/service_name.go @@ -5,35 +5,115 @@ package engine import ( "context" + "slices" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" "github.com/documentdb/documentdb-operator/internal/product" ) -// gatewayServiceName returns the gateway Service name for obj. A cluster whose -// long name an earlier release plainly truncated keeps that existing Service, -// and with it its LoadBalancer endpoint and certificate names, but only when -// the Service is controlled by obj; any other cluster uses the hashed name. -func gatewayServiceName(ctx context.Context, c client.Reader, naming product.NamingProfile, obj metav1.Object) (string, error) { - name := naming.ServiceName(obj.GetName()) - legacy, ok := naming.LegacyServiceName(obj.GetName()) +var certificateGVK = schema.GroupVersionKind{Group: "cert-manager.io", Version: "v1", Kind: "Certificate"} + +// gatewayServiceName returns the gateway Service name for obj and keeps it +// stable for the life of the custom resource. +// +// A name that fits 63 characters is always prefix+name. A longer name is +// chosen once and recorded in the ServiceNameAnnotation, so deleting and +// recreating the Service (or the operator restarting) does not change the +// hostname clients use. The first choice keeps the plainly truncated name an +// earlier release used when this cluster still shows it in use (a Service or +// managed gateway Certificate it controls) and no other resource controls a +// Service by that name; otherwise it is the hashed name. A recorded legacy +// name is dropped in favor of the hashed name once another resource controls +// that Service. Managed certificates follow the returned name (see +// CertificateReconciler), so a change never leaves TLS without a matching SAN. +func gatewayServiceName(ctx context.Context, c client.Client, naming product.NamingProfile, obj client.Object) (string, error) { + hashed := naming.ServiceName(obj.GetName()) + if len(naming.ServicePrefix+obj.GetName()) <= 63 { + return hashed, nil + } + legacy, hasLegacy := naming.LegacyServiceName(obj.GetName()) + key := naming.ServiceNameAnnotation() + recorded := obj.GetAnnotations()[key] + if recorded == hashed { + return hashed, nil + } + + chosen := hashed + if hasLegacy { + available, inUse, err := legacyServiceNameState(ctx, c, obj, legacy) + if err != nil { + return "", err + } + if available && (recorded == legacy || inUse) { + chosen = legacy + } + } + if recorded == chosen { + return chosen, nil + } + + // Record the choice. Values other than the two valid names, and a legacy + // name another resource now holds, are replaced here. The optimistic lock + // makes concurrent reconcilers agree: a loser requeues and reads the + // winner's choice. + before, ok := obj.DeepCopyObject().(client.Object) if !ok { - return name, nil + return chosen, nil + } + annotations := obj.GetAnnotations() + if annotations == nil { + annotations = map[string]string{} } + annotations[key] = chosen + obj.SetAnnotations(annotations) + if err := c.Patch(ctx, obj, client.MergeFromWithOptions(before, client.MergeFromWithOptimisticLock{})); err != nil { + return "", err + } + return chosen, nil +} + +// legacyServiceNameState reports whether the legacy Service name is available +// to obj (no other resource controls a Service by that name) and whether obj +// shows it in use: it controls that Service, or a managed gateway Certificate +// listing it. The Certificate is read as unstructured so callers need not +// register cert-manager types, and a missing cert-manager CRD counts as none. +func legacyServiceNameState(ctx context.Context, c client.Reader, obj client.Object, legacy string) (available, inUse bool, err error) { + ns := obj.GetNamespace() svc := &corev1.Service{} - if err := c.Get(ctx, types.NamespacedName{Name: legacy, Namespace: obj.GetNamespace()}, svc); err != nil { - if apierrors.IsNotFound(err) { - return name, nil + switch err := c.Get(ctx, types.NamespacedName{Name: legacy, Namespace: ns}, svc); { + case err == nil: + if ref := metav1.GetControllerOf(svc); ref != nil && ref.UID != obj.GetUID() { + return false, false, nil } - return "", err + if controlledBy(svc, obj) { + return true, true, nil + } + case !apierrors.IsNotFound(err): + return false, false, err } - if owner := metav1.GetControllerOf(svc); owner != nil && owner.UID == obj.GetUID() { - return legacy, nil + + cert := &unstructured.Unstructured{} + cert.SetGroupVersionKind(certificateGVK) + switch err := c.Get(ctx, types.NamespacedName{Name: obj.GetName() + "-gateway-cert", Namespace: ns}, cert); { + case err == nil: + dnsNames, _, _ := unstructured.NestedStringSlice(cert.Object, "spec", "dnsNames") + return true, controlledBy(cert, obj) && slices.Contains(dnsNames, legacy), nil + case apierrors.IsNotFound(err) || meta.IsNoMatchError(err): + return true, false, nil + default: + return false, false, err } - return name, nil +} + +func controlledBy(o, owner metav1.Object) bool { + ref := metav1.GetControllerOf(o) + return ref != nil && ref.UID == owner.GetUID() } diff --git a/operator/src/internal/engine/service_name_test.go b/operator/src/internal/engine/service_name_test.go index 88b9e83c..6690654b 100644 --- a/operator/src/internal/engine/service_name_test.go +++ b/operator/src/internal/engine/service_name_test.go @@ -10,13 +10,17 @@ import ( "testing" cmapi "github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1" + cmmeta "github.com/cert-manager/cert-manager/pkg/apis/meta/v1" cnpgv1 "github.com/cloudnative-pg/cloudnative-pg/api/v1" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" "sigs.k8s.io/controller-runtime/pkg/client/interceptor" dbpreview "github.com/documentdb/documentdb-operator/api/preview" @@ -45,40 +49,174 @@ func ownedService(name, ip string, owner *dbpreview.DocumentDB) *corev1.Service return svc } -func TestGatewayServiceNameKeepsOnlyOwnLegacyService(t *testing.T) { +func legacyCert(owner *dbpreview.DocumentDB, dnsNames ...string) *cmapi.Certificate { + return &cmapi.Certificate{ + ObjectMeta: metav1.ObjectMeta{ + Name: owner.Name + "-gateway-cert", Namespace: testNamespace, + OwnerReferences: []metav1.OwnerReference{{ + APIVersion: "documentdb.io/preview", Kind: "DocumentDB", Name: owner.Name, UID: owner.UID, + Controller: &[]bool{true}[0], + }}, + }, + Spec: cmapi.CertificateSpec{SecretName: owner.Name + "-gateway-cert-tls", DNSNames: dnsNames}, + } +} + +func TestGatewayServiceNameChoosesOnceAndRecordsIt(t *testing.T) { naming := product.DocumentDBProfile().Naming - east, west := longCluster("-east", "east-uid"), longCluster("-west", "west-uid") + key := naming.ServiceNameAnnotation() + east := longCluster("-east", "east-uid") legacy, ok := naming.LegacyServiceName(east.Name) if !ok || legacy != naming.ServicePrefix+sharedPrefix { t.Fatalf("unexpected legacy name %q, %v", legacy, ok) } - - s := newTestScheme(t) + hashed := naming.ServiceName(east.Name) + otherUID := longCluster("-east", "someone-else") ctx := context.Background() - c := newFakeClient(t, s, nil, ownedService(legacy, "10.0.0.1", east)) + for _, tt := range []struct { - name string - obj *dbpreview.DocumentDB - want string + name string + recorded string + objs []client.Object + want string }{ - {name: "short name needs no lookup", obj: testCluster(), want: naming.ServicePrefix + "db"}, - {name: "owner of the legacy Service keeps it", obj: east, want: legacy}, - {name: "other cluster gets its hashed name", obj: west, want: naming.ServiceName(west.Name)}, - {name: "no legacy Service uses the hashed name", obj: longCluster("-north", "north-uid"), want: naming.ServiceName(sharedPrefix + "-north")}, + {name: "owned legacy Service keeps the legacy name", objs: []client.Object{ownedService(legacy, "10.0.0.1", east)}, want: legacy}, + {name: "owned certificate keeps the legacy name after the Service is gone", objs: []client.Object{legacyCert(east, legacy, legacy+".default")}, want: legacy}, + {name: "legacy Service of another owner is not adopted", objs: []client.Object{ownedService(legacy, "10.0.0.1", otherUID)}, want: hashed}, + {name: "certificate of another owner is not evidence", objs: []client.Object{legacyCert(otherUID, legacy)}, want: hashed}, + {name: "certificate without the legacy name is not evidence", objs: []client.Object{legacyCert(east, hashed)}, want: hashed}, + {name: "no evidence uses the hashed name", want: hashed}, + {name: "recorded legacy name is kept without evidence", recorded: legacy, want: legacy}, + {name: "recorded hashed name is kept despite legacy evidence", recorded: hashed, objs: []client.Object{ownedService(legacy, "10.0.0.1", east)}, want: hashed}, + {name: "unrecognized recorded value is replaced", recorded: "documentdb-service-other", want: hashed}, + {name: "owned certificate does not claim a legacy Service another owner controls", objs: []client.Object{ownedService(legacy, "10.0.0.1", otherUID), legacyCert(east, legacy)}, want: hashed}, + {name: "recorded legacy name is dropped once another owner controls that Service", recorded: legacy, objs: []client.Object{ownedService(legacy, "10.0.0.1", otherUID)}, want: hashed}, } { t.Run(tt.name, func(t *testing.T) { - got, err := gatewayServiceName(ctx, c, naming, tt.obj) + s := newTestScheme(t) + db := east.DeepCopy() + if tt.recorded != "" { + db.Annotations = map[string]string{key: tt.recorded} + } + c := newFakeClient(t, s, nil, append([]client.Object{db}, tt.objs...)...) + got, err := gatewayServiceName(ctx, c, naming, db) if err != nil || got != tt.want { t.Fatalf("gatewayServiceName = %q, %v; want %q", got, err, tt.want) } + stored := &dbpreview.DocumentDB{} + if err := c.Get(ctx, client.ObjectKeyFromObject(db), stored); err != nil { + t.Fatal(err) + } + if stored.Annotations[key] != tt.want { + t.Fatalf("recorded %q = %q, want %q", key, stored.Annotations[key], tt.want) + } }) } - failing := newFakeClient(t, s, &interceptor.Funcs{Get: func(context.Context, client.WithWatch, client.ObjectKey, client.Object, ...client.GetOption) error { - return errInjected - }}) - if _, err := gatewayServiceName(ctx, failing, naming, west); err != errInjected { - t.Fatalf("expected lookup error, got %v", err) + t.Run("short name is neither looked up nor recorded", func(t *testing.T) { + s := newTestScheme(t) + db := testCluster() + c := newFakeClient(t, s, &interceptor.Funcs{Get: func(context.Context, client.WithWatch, client.ObjectKey, client.Object, ...client.GetOption) error { + return errInjected + }}, db) + if got, err := gatewayServiceName(ctx, c, naming, db); err != nil || got != naming.ServicePrefix+"db" { + t.Fatalf("gatewayServiceName = %q, %v", got, err) + } + if _, ok := db.Annotations[key]; ok { + t.Fatal("short names must not be recorded") + } + }) + + t.Run("cert-manager types need not be registered", func(t *testing.T) { + s := runtime.NewScheme() + for _, add := range []func(*runtime.Scheme) error{clientgoscheme.AddToScheme, dbpreview.AddToScheme} { + if err := add(s); err != nil { + t.Fatal(err) + } + } + db := east.DeepCopy() + c := fake.NewClientBuilder().WithScheme(s).WithObjects(db).Build() + if got, err := gatewayServiceName(ctx, c, naming, db); err != nil || got != hashed { + t.Fatalf("gatewayServiceName = %q, %v; want %q", got, err, hashed) + } + }) + + t.Run("concurrent choices do not overwrite each other", func(t *testing.T) { + s := newTestScheme(t) + db := east.DeepCopy() + c := newFakeClient(t, s, nil, db) + stale := &dbpreview.DocumentDB{} + if err := c.Get(ctx, client.ObjectKeyFromObject(db), stale); err != nil { + t.Fatal(err) + } + // Another reconciler records first. + if _, err := gatewayServiceName(ctx, c, naming, db); err != nil { + t.Fatal(err) + } + if _, err := gatewayServiceName(ctx, c, naming, stale); !apierrors.IsConflict(err) { + t.Fatalf("expected a conflict for the stale copy, got %v", err) + } + }) + + t.Run("lookup and record errors propagate", func(t *testing.T) { + s := newTestScheme(t) + failGet := newFakeClient(t, s, &interceptor.Funcs{Get: func(context.Context, client.WithWatch, client.ObjectKey, client.Object, ...client.GetOption) error { + return errInjected + }}, east.DeepCopy()) + if _, err := gatewayServiceName(ctx, failGet, naming, east.DeepCopy()); err != errInjected { + t.Fatalf("expected lookup error, got %v", err) + } + failPatch := newFakeClient(t, s, &interceptor.Funcs{Patch: func(context.Context, client.WithWatch, client.Object, client.Patch, ...client.PatchOption) error { + return errInjected + }}, east.DeepCopy()) + if _, err := gatewayServiceName(ctx, failPatch, naming, east.DeepCopy()); err != errInjected { + t.Fatalf("expected record error, got %v", err) + } + }) +} + +// TestLegacyServiceRecreationKeepsCertificateValid is the reviewer's +// scenario: an upgraded long-name cluster has a legacy Service and a managed +// certificate; deleting the Service must not move the cluster to a hostname +// the certificate does not cover. +func TestLegacyServiceRecreationKeepsCertificateValid(t *testing.T) { + naming := product.DocumentDBProfile().Naming + east := longCluster("-east", "east-uid") + legacy, _ := naming.LegacyServiceName(east.Name) + sans := []string{legacy, legacy + "." + testNamespace, legacy + "." + testNamespace + ".svc"} + + s := newTestScheme(t) + ctx := context.Background() + c := newFakeClient(t, s, nil, east, ownedService(legacy, "10.0.0.1", east), legacyCert(east, sans...)) + key := ctrl.Request{NamespacedName: types.NamespacedName{Name: east.Name, Namespace: testNamespace}} + r, _ := newClusterReconciler(c, s, &sqlRecorder{roleOut: "(1 row)"}) + if _, err := r.Reconcile(ctx, key); err != nil { + t.Fatal(err) + } + + // The Service is deleted (by a user or a cleanup tool) and recreated. + if err := c.Delete(ctx, &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: legacy, Namespace: testNamespace}}); err != nil { + t.Fatal(err) + } + if _, err := newCertReconciler(c, s).Reconcile(ctx, key); err != nil { + t.Fatal(err) + } + if _, err := r.Reconcile(ctx, key); err != nil { + t.Fatal(err) + } + + if err := c.Get(ctx, types.NamespacedName{Name: legacy, Namespace: testNamespace}, &corev1.Service{}); err != nil { + t.Fatalf("expected the Service to be recreated under its recorded name %q: %v", legacy, err) + } + if err := c.Get(ctx, types.NamespacedName{Name: naming.ServiceName(east.Name), Namespace: testNamespace}, &corev1.Service{}); !apierrors.IsNotFound(err) { + t.Fatalf("the hostname must not move to the hashed name, got %v", err) + } + cert := &cmapi.Certificate{} + if err := c.Get(ctx, types.NamespacedName{Name: east.Name + "-gateway-cert", Namespace: testNamespace}, cert); err != nil { + t.Fatal(err) + } + if !slices.Contains(cert.Spec.DNSNames, legacy+"."+testNamespace+".svc") { + t.Fatalf("certificate %v does not cover the Service hostname", cert.Spec.DNSNames) } } @@ -164,3 +302,133 @@ func TestCertificateNamesFollowResolvedService(t *testing.T) { }) } } + +// TestCollidingUpgradedClusterMovesToItsOwnServiceAndCertificate covers two +// long-name clusters that collided on one legacy Service before the upgrade: +// the cluster that does not own that Service moves to its hashed Service and +// its certificate gains the new hostname while keeping the old names. +func TestCollidingUpgradedClusterMovesToItsOwnServiceAndCertificate(t *testing.T) { + naming := product.DocumentDBProfile().Naming + east, west := longCluster("-east", "east-uid"), longCluster("-west", "west-uid") + legacy, _ := naming.LegacyServiceName(west.Name) + hashed := naming.ServiceName(west.Name) + legacySANs := []string{legacy, legacy + "." + testNamespace, legacy + "." + testNamespace + ".svc"} + + for _, mode := range []*dbpreview.GatewayTLS{nil, {Mode: "CertManager", CertManager: &dbpreview.CertManagerTLS{IssuerRef: dbpreview.IssuerRef{Name: "ca"}}}} { + name := "self-signed" + if mode != nil { + name = "cert-manager" + } + t.Run(name, func(t *testing.T) { + s := newTestScheme(t) + ctx := context.Background() + db := west.DeepCopy() + if mode != nil { + db.Spec.TLS = &dbpreview.TLSConfiguration{Gateway: mode} + } + cert := legacyCert(db, legacySANs...) + cert.Status.Conditions = []cmapi.CertificateCondition{{Type: cmapi.CertificateConditionReady, Status: cmmeta.ConditionTrue}} + c := newFakeClient(t, s, nil, east, db, ownedService(legacy, "10.0.0.1", east), ownedService(hashed, "10.0.0.2", db), cert) + key := ctrl.Request{NamespacedName: types.NamespacedName{Name: db.Name, Namespace: testNamespace}} + + res, err := newCertReconciler(c, s).Reconcile(ctx, key) + if err != nil || res.RequeueAfter != RequeueAfterShort { + t.Fatalf("expected a short requeue after updating DNS names, got %v, %v", res, err) + } + got := &cmapi.Certificate{} + if err := c.Get(ctx, client.ObjectKeyFromObject(cert), got); err != nil { + t.Fatal(err) + } + want := append(append([]string{}, legacySANs...), hashed, hashed+"."+testNamespace, hashed+"."+testNamespace+".svc") + if !slices.Equal(got.Spec.DNSNames, want) { + t.Fatalf("DNSNames = %v, want %v", got.Spec.DNSNames, want) + } + if st := getDocumentDBByName(t, c, db.Name).Status.TLS; st == nil || st.Ready || st.Message != "Updating gateway certificate DNS names" { + t.Fatalf("unexpected TLS status %+v", st) + } + + // The next pass finds nothing to add. + if _, err := newCertReconciler(c, s).Reconcile(ctx, key); err != nil { + t.Fatal(err) + } + again := &cmapi.Certificate{} + if err := c.Get(ctx, client.ObjectKeyFromObject(cert), again); err != nil || !slices.Equal(again.Spec.DNSNames, want) { + t.Fatalf("DNS names changed again: %v, %v", again.Spec.DNSNames, err) + } + + r, _ := newClusterReconciler(c, s, &sqlRecorder{roleOut: "(1 row)"}) + if _, err := r.Reconcile(ctx, key); err != nil { + t.Fatal(err) + } + if got := getDocumentDBByName(t, c, db.Name); got.Annotations[naming.ServiceNameAnnotation()] != hashed { + t.Fatalf("expected %s to record %q, got %v", db.Name, hashed, got.Annotations) + } + }) + } +} + +func TestCertificateOfAnotherResourceIsNeitherChangedNorUsed(t *testing.T) { + naming := product.DocumentDBProfile().Naming + west := longCluster("-west", "west-uid") + legacy, _ := naming.LegacyServiceName(west.Name) + // Left by a deleted DocumentDB of the same name and still Ready. + cert := legacyCert(longCluster("-west", "deleted-uid"), legacy) + cert.Status.Conditions = []cmapi.CertificateCondition{{Type: cmapi.CertificateConditionReady, Status: cmmeta.ConditionTrue}} + s := newTestScheme(t) + c := newFakeClient(t, s, nil, west, cert) + key := ctrl.Request{NamespacedName: types.NamespacedName{Name: west.Name, Namespace: testNamespace}} + res, err := newCertReconciler(c, s).Reconcile(context.Background(), key) + if err != nil || res.RequeueAfter != RequeueAfterShort { + t.Fatalf("expected a short requeue, got %v, %v", res, err) + } + got := &cmapi.Certificate{} + if err := c.Get(context.Background(), client.ObjectKeyFromObject(cert), got); err != nil || !slices.Equal(got.Spec.DNSNames, []string{legacy}) { + t.Fatalf("a certificate this cluster does not control was modified: %v, %v", got.Spec.DNSNames, err) + } + if st := getDocumentDBByName(t, c, west.Name).Status.TLS; st == nil || st.Ready || st.SecretName != "" { + t.Fatalf("another resource's certificate must not be reported ready or used: %+v", st) + } +} + +func TestCertificateReadyMustMatchCurrentGeneration(t *testing.T) { + s := newTestScheme(t) + db := testCluster() + svc := product.DocumentDBProfile().Naming.ServiceName(db.Name) + cert := legacyCert(db, svc, svc+"."+testNamespace, svc+"."+testNamespace+".svc") + cert.Generation = 2 + // cert-manager has not yet observed the latest spec (new DNS names). + cert.Status.Conditions = []cmapi.CertificateCondition{{Type: cmapi.CertificateConditionReady, Status: cmmeta.ConditionTrue, ObservedGeneration: 1}} + c := newFakeClient(t, s, nil, db, cert) + r := newCertReconciler(c, s) + + if res, err := r.Reconcile(context.Background(), certKey); err != nil || res.RequeueAfter != RequeueAfterShort { + t.Fatalf("expected to wait for the current generation, got %v, %v", res, err) + } + if st := getTLSStatus(t, c); st.Ready { + t.Fatalf("a Ready condition from an older generation must not report TLS ready: %+v", st) + } + + got := &cmapi.Certificate{} + if err := c.Get(context.Background(), client.ObjectKeyFromObject(cert), got); err != nil { + t.Fatal(err) + } + got.Status.Conditions[0].ObservedGeneration = 2 + if err := c.Update(context.Background(), got); err != nil { + t.Fatal(err) + } + if res, err := r.Reconcile(context.Background(), certKey); err != nil || res != (ctrl.Result{}) { + t.Fatalf("expected ready, got %v, %v", res, err) + } + if st := getTLSStatus(t, c); !st.Ready { + t.Fatalf("expected TLS ready once the current generation is observed: %+v", st) + } +} + +func getDocumentDBByName(t *testing.T, c client.Client, name string) *dbpreview.DocumentDB { + t.Helper() + db := &dbpreview.DocumentDB{} + if err := c.Get(context.Background(), types.NamespacedName{Name: name, Namespace: testNamespace}, db); err != nil { + t.Fatal(err) + } + return db +} diff --git a/operator/src/internal/product/cluster.go b/operator/src/internal/product/cluster.go index f30298b3..57b5889b 100644 --- a/operator/src/internal/product/cluster.go +++ b/operator/src/internal/product/cluster.go @@ -48,6 +48,11 @@ func (n NamingProfile) SchemaVersionAnnotation() string { return n.Domain + "/sc // Finalizer is the custom resource finalizer used to warn about retained PVs. func (n NamingProfile) Finalizer() string { return n.Domain + "/pv-retention-finalizer" } +// ServiceNameAnnotation records, on a custom resource whose prefixed name +// exceeds 63 characters, the gateway Service name the operator chose, so the +// name stays fixed when the Service is deleted and recreated. +func (n NamingProfile) ServiceNameAnnotation() string { return n.Domain + "/gateway-service-name" } + // ServiceName is the gateway Service name for a custom resource: // ServicePrefix+clusterName, or, when that exceeds the 63-character Kubernetes // name limit, a truncated form ending in a hash of clusterName so distinct diff --git a/test/e2e/pkg/e2eutils/portforward/portforward.go b/test/e2e/pkg/e2eutils/portforward/portforward.go index 037215c5..ffe236cc 100644 --- a/test/e2e/pkg/e2eutils/portforward/portforward.go +++ b/test/e2e/pkg/e2eutils/portforward/portforward.go @@ -41,10 +41,15 @@ const GatewayPort = 10260 // ServiceNamePrefix mirrors DOCUMENTDB_SERVICE_PREFIX from the operator. // The Service name is ServiceNamePrefix + dd.Name; when that exceeds 63 // characters the operator keeps the first 54 and appends "-" plus the -// FNV-32a hash of dd.Name (see utils.GatewayServiceName). E2E clusters are -// created fresh, so the operator's legacy plainly truncated name never applies. +// FNV-32a hash of dd.Name (see utils.GatewayServiceName), unless it recorded +// another valid choice (the plainly truncated name of an upgraded cluster) in +// ServiceNameAnnotation. const ServiceNamePrefix = "documentdb-service-" +// ServiceNameAnnotation mirrors NamingProfile.ServiceNameAnnotation for +// DocumentDB. +const ServiceNameAnnotation = "documentdb.io/gateway-service-name" + // GatewayServiceName returns the Service name the operator creates for // the given DocumentDB CR. func GatewayServiceName(dd *previewv1.DocumentDB) string { @@ -58,7 +63,11 @@ func GatewayServiceName(dd *previewv1.DocumentDB) string { h := fnv.New32a() _, _ = h.Write([]byte(dd.Name)) suffix := fmt.Sprintf("-%08x", h.Sum32()) - return strings.TrimRight(name[:63-len(suffix)], "-") + suffix + hashed := strings.TrimRight(name[:63-len(suffix)], "-") + suffix + if recorded := dd.Annotations[ServiceNameAnnotation]; recorded == name[:63] && !strings.HasSuffix(recorded, "-") { + return recorded + } + return hashed } // OpenWithErr establishes a port-forward from localPort on the caller's diff --git a/test/e2e/pkg/e2eutils/portforward/portforward_test.go b/test/e2e/pkg/e2eutils/portforward/portforward_test.go index 6bb6a8c6..48beca71 100644 --- a/test/e2e/pkg/e2eutils/portforward/portforward_test.go +++ b/test/e2e/pkg/e2eutils/portforward/portforward_test.go @@ -36,6 +36,22 @@ func TestGatewayServiceName(t *testing.T) { &previewv1.DocumentDB{ObjectMeta: metav1.ObjectMeta{Name: strings.Repeat("a", 44) + "-east"}}, "documentdb-service-" + strings.Repeat("a", 35) + "-2f5a7991", }, + { + "recorded legacy name", + &previewv1.DocumentDB{ObjectMeta: metav1.ObjectMeta{ + Name: strings.Repeat("a", 44) + "x-east", + Annotations: map[string]string{ServiceNameAnnotation: "documentdb-service-" + strings.Repeat("a", 44)}, + }}, + "documentdb-service-" + strings.Repeat("a", 44), + }, + { + "unrecognized recorded name is ignored", + &previewv1.DocumentDB{ObjectMeta: metav1.ObjectMeta{ + Name: strings.Repeat("a", 44) + "-east", + Annotations: map[string]string{ServiceNameAnnotation: "documentdb-service-other"}, + }}, + "documentdb-service-" + strings.Repeat("a", 35) + "-2f5a7991", + }, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) {